Removed deprecated URLField.verify_exists.

The deprecation schedule was slightly accelerated because of possible security ramifications.



git-svn-id: http://code.djangoproject.com/svn/django/trunk@17847 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/conf/global_settings.py

@@ -403,11 +403,6 @@ THOUSAND_SEPARATOR = ','
 # Hint: you really don't!
 TRANSACTIONS_MANAGED = False
 
-# The User-Agent string to use when checking for URL validity through the
-# isExistingURL validator.
-from django import get_version
-URL_VALIDATOR_USER_AGENT = "Django/%s (https://www.djangoproject.com)" % get_version()
-
 # The tablespaces to use for each model when not specified otherwise.
 DEFAULT_TABLESPACE = ''
 DEFAULT_INDEX_TABLESPACE = ''

django/core/validators.py

@@ -1,6 +1,4 @@
 import re
-import urllib
-import urllib2
 import urlparse
 
 from django.core.exceptions import ValidationError
@@ -11,13 +9,6 @@ from django.utils.ipv6 import is_valid_ipv6_address
 # These values, if given to validate(), will trigger the self.required check.
 EMPTY_VALUES = (None, '', [], (), {})
 
-try:
-    from django.conf import settings
-    URL_VALIDATOR_USER_AGENT = settings.URL_VALIDATOR_USER_AGENT
-except ImportError:
-    # It's OK if Django settings aren't configured.
-    URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)'
-
 class RegexValidator(object):
     regex = ''
     message = _(u'Enter a valid value.')
@@ -51,11 +42,8 @@ class URLValidator(RegexValidator):
         r'(?::\d+)?' # optional port
         r'(?:/?|[/?]\S+)$', re.IGNORECASE)
 
-    def __init__(self, verify_exists=False,
+    def __init__(self):
-                 validator_user_agent=URL_VALIDATOR_USER_AGENT):
         super(URLValidator, self).__init__()
-        self.verify_exists = verify_exists
-        self.user_agent = validator_user_agent
 
     def __call__(self, value):
         try:
@@ -76,58 +64,6 @@ class URLValidator(RegexValidator):
         else:
             url = value
 
-        if self.verify_exists:
-            import warnings
-            warnings.warn(
-                "The URLField verify_exists argument has intractable security "
-                "and performance issues. Accordingly, it has been deprecated.",
-                DeprecationWarning
-                )
-
-            headers = {
-                "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
-                "Accept-Language": "en-us,en;q=0.5",
-                "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
-                "Connection": "close",
-                "User-Agent": self.user_agent,
-            }
-            url = url.encode('utf-8')
-            # Quote characters from the unreserved set, refs #16812
-            url = urllib.quote(url, "!*'();:@&=+$,/?#[]")
-            broken_error = ValidationError(
-                _(u'This URL appears to be a broken link.'), code='invalid_link')
-            try:
-                req = urllib2.Request(url, None, headers)
-                req.get_method = lambda: 'HEAD'
-                #Create an opener that does not support local file access
-                opener = urllib2.OpenerDirector()
-
-                #Don't follow redirects, but don't treat them as errors either
-                error_nop = lambda *args, **kwargs: True
-                http_error_processor = urllib2.HTTPErrorProcessor()
-                http_error_processor.http_error_301 = error_nop
-                http_error_processor.http_error_302 = error_nop
-                http_error_processor.http_error_307 = error_nop
-
-                handlers = [urllib2.UnknownHandler(),
-                            urllib2.HTTPHandler(),
-                            urllib2.HTTPDefaultErrorHandler(),
-                            urllib2.FTPHandler(),
-                            http_error_processor]
-                try:
-                    import ssl
-                except ImportError:
-                    # Python isn't compiled with SSL support
-                    pass
-                else:
-                    handlers.append(urllib2.HTTPSHandler())
-                map(opener.add_handler, handlers)
-                opener.open(req, timeout=10)
-            except ValueError:
-                raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
-            except: # urllib2.URLError, httplib.InvalidURL, etc.
-                raise broken_error
-
 
 def validate_integer(value):
     try:

django/db/models/fields/__init__.py

@@ -1251,12 +1251,10 @@ class TimeField(Field):
 class URLField(CharField):
     description = _("URL")
 
-    def __init__(self, verbose_name=None, name=None, verify_exists=False,
+    def __init__(self, verbose_name=None, name=None, **kwargs):
-                 **kwargs):
         kwargs['max_length'] = kwargs.get('max_length', 200)
         CharField.__init__(self, verbose_name, name, **kwargs)
-        self.validators.append(
+        self.validators.append(validators.URLValidator())
-            validators.URLValidator(verify_exists=verify_exists))
 
     def formfield(self, **kwargs):
         # As with CharField, this will cause URL validation to be performed

django/forms/fields.py

@@ -598,14 +598,11 @@ class ImageField(FileField):
 class URLField(CharField):
     default_error_messages = {
         'invalid': _(u'Enter a valid URL.'),
-        'invalid_link': _(u'This URL appears to be a broken link.'),
     }
 
-    def __init__(self, max_length=None, min_length=None, verify_exists=False,
+    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
-            validator_user_agent=validators.URL_VALIDATOR_USER_AGENT, *args, **kwargs):
+        super(URLField, self).__init__(max_length, min_length, *args, **kwargs)
-        super(URLField, self).__init__(max_length, min_length, *args,
+        self.validators.append(validators.URLValidator())
-                                       **kwargs)
-        self.validators.append(validators.URLValidator(verify_exists=verify_exists, validator_user_agent=validator_user_agent))
 
     def to_python(self, value):
 

docs/ref/forms/fields.txt

@@ -789,7 +789,7 @@ For each field, we describe the default widget used if you don't specify
     * Empty value: ``''`` (an empty string)
     * Normalizes to: A Unicode object.
     * Validates that the given value is a valid URL.
-    * Error message keys: ``required``, ``invalid``, ``invalid_link``
+    * Error message keys: ``required``, ``invalid``
 
     Takes the following optional arguments:
 
@@ -798,20 +798,6 @@ For each field, we describe the default widget used if you don't specify
 
     These are the same as ``CharField.max_length`` and ``CharField.min_length``.
 
-    .. attribute:: verify_exists
-
-        If ``True``, the validator will attempt to load the given URL, raising
-        ``ValidationError`` if the page gives a 404. Defaults to ``False``.
-
-        .. deprecated:: 1.4
-            ``verify_exists`` was deprecated for security reasons and will be removed in
-            Django 1.5. This deprecation also removes ``validator_user_agent``.
-
-    .. attribute:: validator_user_agent
-
-        String used as the user-agent used when checking for a URL's existence.
-        Defaults to the value of the :setting:`URL_VALIDATOR_USER_AGENT` setting.
-
 .. versionchanged:: 1.2
    The URLField previously did not recognize URLs as valid that contained an IDN
    (Internationalized Domain Name; a domain name containing unicode characters)

docs/ref/models/fields.txt

@@ -887,23 +887,9 @@ shortcuts.
 ``URLField``
 ------------
 
-.. class:: URLField([verify_exists=False, max_length=200, **options])
+.. class:: URLField([max_length=200, **options])
 
-A :class:`CharField` for a URL. Has one extra optional argument:
+A :class:`CharField` for a URL.
-
-.. deprecated:: 1.4
-   ``verify_exists`` is deprecated for security reasons as of 1.4 and will be
-   removed in Django 1.5. Prior to 1.3.1, the default value was ``True``.
-
-.. attribute:: URLField.verify_exists
-
-    If ``True``, the URL given will be checked for existence (i.e.,
-    the URL actually loads and doesn't give a 404 response) using a
-    ``HEAD`` request. Redirects are allowed, but will not be followed.
-
-    Note that when you're using the single-threaded development server,
-    validating a URL being served by the same server will hang. This should not
-    be a problem for multithreaded servers.
 
 The admin represents this as an ``<input type="text">`` (a single-line input).
 

docs/ref/settings.txt

@@ -2148,18 +2148,6 @@ to ensure your processes are running in the correct environment.
 
 .. _pytz: http://pytz.sourceforge.net/
 
-.. setting:: URL_VALIDATOR_USER_AGENT
-
-URL_VALIDATOR_USER_AGENT
-------------------------
-
-Default: ``Django/<version> (https://www.djangoproject.com/)``
-
-The string to use as the ``User-Agent`` header when checking to see if
-URLs exist (see the ``verify_exists`` option on
-:class:`~django.db.models.URLField`). This setting was deprecated in
-1.3.1 along with ``verify_exists`` and will be removed in 1.4.
-
 .. setting:: USE_ETAGS
 
 USE_ETAGS
@@ -2344,3 +2332,12 @@ IGNORABLE_404_STARTS
 
 .. deprecated:: 1.4
    This setting has been superseded by :setting:`IGNORABLE_404_URLS`.
+
+.. setting:: URL_VALIDATOR_USER_AGENT
+
+URL_VALIDATOR_USER_AGENT
+------------------------
+
+.. deprecated:: 1.5
+    This value was used as the ``User-Agent`` header when checking if a URL
+    exists, a feature that was removed due to security and performance issues.

docs/ref/validators.txt

@@ -88,26 +88,10 @@ to, or in lieu of custom ``field.clean()`` methods.
 
 ``URLValidator``
 ----------------
-.. class:: URLValidator([verify_exists=False, validator_user_agent=URL_VALIDATOR_USER_AGENT])
+.. class:: URLValidator()
 
-    A :class:`RegexValidator` that ensures a value looks like a URL and
+    A :class:`RegexValidator` that ensures a value looks like a URL, and raises
-    optionally verifies that the URL actually exists (i.e., doesn't return a
+    an error code of ``'invalid'`` if it doesn't.
-    404 status code). Raises an error code of ``'invalid'`` if it doesn't look
-    like a URL, and a code of ``'invalid_link'`` if it doesn't exist.
-
-    :param verify_exists: Sets :attr:`verify_exists`. Defaults to ``False``.
-    :param validator_user_agent: Sets :attr:`validator_user_agent`. Defaults to
-        :setting:`URL_VALIDATOR_USER_AGENT` or, if that setting is set to a
-        null value, ``"Django (https://www.djangoproject.com/)"``.
-
-    .. attribute:: verify_exists
-
-        If set to ``True``, this validator checks that the URL actually exists.
-
-    .. attribute:: validator_user_agent
-
-        If :attr:`verify_exists` is ``True``, Django uses this value as the
-        "User-agent" for the request.
 
 ``validate_email``
 ------------------

docs/topics/forms/modelforms.txt

@@ -106,8 +106,7 @@ Model field                      Form field
 
 ``TimeField``                    ``TimeField``
 
-``URLField``                     ``URLField`` with ``verify_exists`` set
+``URLField``                     ``URLField``
-                                 to the model field's ``verify_exists``
 ===============================  ========================================
 
 .. versionadded:: 1.2

tests/modeltests/validation/models.py

@@ -15,7 +15,6 @@ class ModelToValidate(models.Model):
     parent = models.ForeignKey('self', blank=True, null=True, limit_choices_to={'number': 10})
     email = models.EmailField(blank=True)
     url = models.URLField(blank=True)
-    url_verify = models.URLField(blank=True, verify_exists=True)
     f_with_custom_validator = models.IntegerField(blank=True, null=True, validators=[validate_answer_to_universe])
 
     def clean(self):

tests/modeltests/validation/tests.py

@@ -1,14 +1,9 @@
 from __future__ import absolute_import
 
-import warnings
-
 from django import forms
 from django.core.exceptions import NON_FIELD_ERRORS
 from django.test import TestCase
 
-# Import the verify_exists_urls from the 'forms' test app
-from regressiontests.forms.tests.fields import verify_exists_urls
-
 from . import ValidationTestCase
 from .models import (Author, Article, ModelToValidate,
     GenericIPAddressTestModel, GenericIPAddrUnpackUniqueTest)
@@ -22,14 +17,6 @@ from .validators import TestModelsWithValidators
 
 class BaseModelValidationTests(ValidationTestCase):
 
-    def setUp(self):
-        self.save_warnings_state()
-        warnings.filterwarnings('ignore', category=DeprecationWarning,
-                                module='django.core.validators')
-
-    def tearDown(self):
-        self.restore_warnings_state()
-
     def test_missing_required_field_raises_error(self):
         mtv = ModelToValidate(f_with_custom_validator=42)
         self.assertFailsValidation(mtv.full_clean, ['name', 'number'])
@@ -70,25 +57,6 @@ class BaseModelValidationTests(ValidationTestCase):
         mtv = ModelToValidate(number=10, name='Some Name', url='not a url')
         self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'Enter a valid value.'])
 
-    #The tests below which use url_verify are deprecated
-    def test_correct_url_but_nonexisting_gives_404(self):
-        mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=404')
-        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
-
-    @verify_exists_urls(existing_urls=('http://www.google.com/',))
-    def test_correct_url_value_passes(self):
-        mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://www.google.com/')
-        self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
-
-    @verify_exists_urls(existing_urls=('http://qa-dev.w3.org/link-testsuite/http.php?code=301',))
-    def test_correct_url_with_redirect(self):
-        mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=301') #example.com is a redirect to iana.org now
-        self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
-
-    def test_correct_https_url_but_nonexisting(self):
-        mtv = ModelToValidate(number=10, name='Some Name', url_verify='https://www.example.com/')
-        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
-
     def test_text_greater_that_charfields_max_length_raises_erros(self):
         mtv = ModelToValidate(number=10, name='Some Name'*100)
         self.assertFailsValidation(mtv.full_clean, ['name',])

tests/regressiontests/forms/tests/error_messages.py

@@ -7,8 +7,6 @@ from django.forms import *
 from django.test import TestCase
 from django.utils.safestring import mark_safe
 
-from .fields import verify_exists_urls
-
 
 class AssertFormErrorsMixin(object):
     def assertFormErrors(self, expected, the_callable, *args, **kwargs):
@@ -143,17 +141,14 @@ class FormsErrorMessagesTestCase(TestCase, AssertFormErrorsMixin):
         self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', None))
         self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', ''))
 
-    @verify_exists_urls()
     def test_urlfield(self):
         e = {
             'required': 'REQUIRED',
             'invalid': 'INVALID',
-            'invalid_link': 'INVALID LINK',
         }
-        f = URLField(verify_exists=True, error_messages=e)
+        f = URLField(error_messages=e)
         self.assertFormErrors([u'REQUIRED'], f.clean, '')
         self.assertFormErrors([u'INVALID'], f.clean, 'abc.c')
-        self.assertFormErrors([u'INVALID LINK'], f.clean, 'http://www.broken.djangoproject.com')
 
     def test_booleanfield(self):
         e = {

tests/regressiontests/forms/tests/fields.py

@@ -27,10 +27,8 @@ __init__(). For example, CharField has a max_length option.
 import datetime
 import re
 import os
-import urllib2
 import warnings
 from decimal import Decimal
-from functools import wraps
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.forms import *
@@ -48,32 +46,6 @@ def fix_os_paths(x):
         return x
 
 
-def verify_exists_urls(existing_urls=()):
-    """
-    Patches urllib to simulate the availability of some urls even when there
-    is no Internet connection. This hack should be removed alongside with
-    `URLField.verify_exists` in Django 1.5.
-    """
-    def decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            from django.core import validators
-            # patch urllib2.OpenerDirector
-            original_open = validators.urllib2.OpenerDirector.open
-            def custom_open(self, req, data=None, timeout=None):
-                if req.get_full_url() in existing_urls:
-                    return True
-                raise Exception()
-            try:
-                urllib2.OpenerDirector.open = custom_open
-                func(*args, **kwargs)
-            finally:
-                # unpatch urllib2.OpenerDirector
-                validators.urllib2.OpenerDirector.open = original_open
-        return wrapper
-    return decorator
-
-
 class FieldsTests(SimpleTestCase):
 
     def setUp(self):
@@ -651,30 +623,6 @@ class FieldsTests(SimpleTestCase):
         self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example.')
         self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://.com')
 
-    @verify_exists_urls(('http://www.google.com/',))
-    def test_urlfield_3(self):
-        f = URLField(verify_exists=True)
-        self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com'))
-        self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example')
-        self.assertRaises(ValidationError, f.clean, 'http://www.broken.djangoproject.com') # bad domain
-        self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=405') # Method not allowed
-        try:
-            f.clean('http://www.broken.djangoproject.com') # bad domain
-        except ValidationError, e:
-            self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
-        self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=400') # good domain, bad page
-        try:
-            f.clean('http://google.com/we-love-microsoft.html') # good domain, bad page
-        except ValidationError, e:
-            self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
-
-    @verify_exists_urls(('http://www.google.com/',))
-    def test_urlfield_4(self):
-        f = URLField(verify_exists=True, required=False)
-        self.assertEqual(u'', f.clean(''))
-        self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com'))
-
-    @verify_exists_urls(('http://example.com/',))
     def test_urlfield_5(self):
         f = URLField(min_length=15, max_length=20)
         self.assertRaisesMessage(ValidationError, "[u'Ensure this value has at least 15 characters (it has 13).']", f.clean, 'http://f.com')
@@ -698,7 +646,7 @@ class FieldsTests(SimpleTestCase):
         self.assertEqual(u'http://example.com/?some_param=some_value', f.clean('http://example.com?some_param=some_value'))
 
     def test_urlfield_9(self):
-        f = URLField(verify_exists=False)
+        f = URLField()
         urls = (
             u'http://עברית.idn.icann.org/',
             u'http://sãopaulo.com/',
@@ -722,13 +670,6 @@ class FieldsTests(SimpleTestCase):
         except ValidationError, e:
             self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
 
-    @verify_exists_urls((u'http://xn--hxargifdar.idn.icann.org/%CE%91%CF%81%CF%87%CE%B9%CE%BA%CE%AE_%CF%83%CE%B5%CE%BB%CE%AF%CE%B4%CE%B1',))
-    def test_urlfield_10(self):
-        # UTF-8 in the domain.
-        f = URLField(verify_exists=True)
-        url = u'http://\u03b5\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac.idn.icann.org/\u0391\u03c1\u03c7\u03b9\u03ba\u03ae_\u03c3\u03b5\u03bb\u03af\u03b4\u03b1'
-        self.assertEqual(url, f.clean(url))
-
     def test_urlfield_not_string(self):
         f = URLField(required=False)
         self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 23)

tests/regressiontests/generic_inline_admin/models.py

@@ -16,7 +16,7 @@ class Media(models.Model):
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
     content_object = generic.GenericForeignKey()
-    url = models.URLField(verify_exists=False)
+    url = models.URLField()
     description = models.CharField(max_length=100, blank=True)
     keywords = models.CharField(max_length=100, blank=True)
 

tests/regressiontests/model_forms_regress/models.py

@@ -57,7 +57,7 @@ class Author1(models.Model):
     full_name = models.CharField(max_length=255)
 
 class Homepage(models.Model):
-    url = models.URLField(verify_exists=False)
+    url = models.URLField()
 
 class Document(models.Model):
     myfile = models.FileField(upload_to='unused', blank=True)