Merge pull request #2221 from bmispelon/LazyObject-refactor

Fixed #21840 -- Moved dunder methods from SimpleLazyObject to LazyObject...

django/contrib/staticfiles/management/commands/collectstatic.py

@@ -7,7 +7,6 @@ from optparse import make_option
 from django.core.files.storage import FileSystemStorage
 from django.core.management.base import CommandError, NoArgsCommand
 from django.utils.encoding import smart_text
-from django.utils.functional import LazyObject
 from django.utils.six.moves import input
 
 from django.contrib.staticfiles import finders, storage
@@ -193,11 +192,7 @@ class Command(NoArgsCommand):
             self.stdout.write(msg)
 
     def is_local_storage(self):
-        if issubclass(self.storage.__class__, LazyObject):
+        return isinstance(self.storage, FileSystemStorage)
-            storage = self.storage._wrapped
-        else:
-            storage = self.storage
-        return isinstance(storage, FileSystemStorage)
 
     def clear_dir(self, path):
         """

django/utils/functional.py

@@ -265,9 +265,62 @@ class LazyObject(object):
         """
         raise NotImplementedError('subclasses of LazyObject must provide a _setup() method')
 
+    # Because we have messed with __class__ below, we confuse pickle as to what
+    # class we are pickling. It also appears to stop __reduce__ from being
+    # called. So, we define __getstate__ in a way that cooperates with the way
+    # that pickle interprets this class.  This fails when the wrapped class is
+    # a builtin, but it is better than nothing.
+    def __getstate__(self):
+        if self._wrapped is empty:
+            self._setup()
+        return self._wrapped.__dict__
+
+    # Python 3.3 will call __reduce__ when pickling; this method is needed
+    # to serialize and deserialize correctly.
+    @classmethod
+    def __newobj__(cls, *args):
+        return cls.__new__(cls, *args)
+
+    def __reduce_ex__(self, proto):
+        if proto >= 2:
+            # On Py3, since the default protocol is 3, pickle uses the
+            # ``__newobj__`` method (& more efficient opcodes) for writing.
+            return (self.__newobj__, (self.__class__,), self.__getstate__())
+        else:
+            # On Py2, the default protocol is 0 (for back-compat) & the above
+            # code fails miserably (see regression test). Instead, we return
+            # exactly what's returned if there's no ``__reduce__`` method at
+            # all.
+            return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__())
+
+    def __deepcopy__(self, memo):
+        if self._wrapped is empty:
+            # We have to use type(self), not self.__class__, because the
+            # latter is proxied.
+            result = type(self)()
+            memo[id(self)] = result
+            return result
+        return copy.deepcopy(self._wrapped, memo)
+
+    if six.PY3:
+        __bytes__ = new_method_proxy(bytes)
+        __str__ = new_method_proxy(str)
+        __bool__ = new_method_proxy(bool)
+    else:
+        __str__ = new_method_proxy(str)
+        __unicode__ = new_method_proxy(unicode)
+        __nonzero__ = new_method_proxy(bool)
+
     # Introspection support
     __dir__ = new_method_proxy(dir)
 
+    # Need to pretend to be the wrapped class, for the sake of objects that
+    # care about this (especially in equality tests)
+    __class__ = property(new_method_proxy(operator.attrgetter("__class__")))
+    __eq__ = new_method_proxy(operator.eq)
+    __ne__ = new_method_proxy(operator.ne)
+    __hash__ = new_method_proxy(hash)
+
     # Dictionary methods support
     __getitem__ = new_method_proxy(operator.getitem)
     __setitem__ = new_method_proxy(operator.setitem)
@@ -303,12 +356,14 @@ class SimpleLazyObject(LazyObject):
     def _setup(self):
         self._wrapped = self._setupfunc()
 
-    if six.PY3:
+    # Return a meaningful representation of the lazy object for debugging
-        __bytes__ = new_method_proxy(bytes)
+    # without evaluating the wrapped object.
-        __str__ = new_method_proxy(str)
+    def __repr__(self):
-    else:
+        if self._wrapped is empty:
-        __str__ = new_method_proxy(str)
+            repr_attr = self._setupfunc
-        __unicode__ = new_method_proxy(unicode)
+        else:
+            repr_attr = self._wrapped
+        return '<%s: %r>' % (type(self).__name__, repr_attr)
 
     def __deepcopy__(self, memo):
         if self._wrapped is empty:
@@ -317,54 +372,7 @@ class SimpleLazyObject(LazyObject):
             result = SimpleLazyObject(self._setupfunc)
             memo[id(self)] = result
             return result
-        else:
+        return copy.deepcopy(self._wrapped, memo)
-            return copy.deepcopy(self._wrapped, memo)
-
-    # Because we have messed with __class__ below, we confuse pickle as to what
-    # class we are pickling. It also appears to stop __reduce__ from being
-    # called. So, we define __getstate__ in a way that cooperates with the way
-    # that pickle interprets this class.  This fails when the wrapped class is
-    # a builtin, but it is better than nothing.
-    def __getstate__(self):
-        if self._wrapped is empty:
-            self._setup()
-        return self._wrapped.__dict__
-
-    # Python 3.3 will call __reduce__ when pickling; this method is needed
-    # to serialize and deserialize correctly.
-    @classmethod
-    def __newobj__(cls, *args):
-        return cls.__new__(cls, *args)
-
-    def __reduce_ex__(self, proto):
-        if proto >= 2:
-            # On Py3, since the default protocol is 3, pickle uses the
-            # ``__newobj__`` method (& more efficient opcodes) for writing.
-            return (self.__newobj__, (self.__class__,), self.__getstate__())
-        else:
-            # On Py2, the default protocol is 0 (for back-compat) & the above
-            # code fails miserably (see regression test). Instead, we return
-            # exactly what's returned if there's no ``__reduce__`` method at
-            # all.
-            return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__())
-
-    # Return a meaningful representation of the lazy object for debugging
-    # without evaluating the wrapped object.
-    def __repr__(self):
-        if self._wrapped is empty:
-            repr_attr = self._setupfunc
-        else:
-            repr_attr = self._wrapped
-        return '<SimpleLazyObject: %r>' % repr_attr
-
-    # Need to pretend to be the wrapped class, for the sake of objects that
-    # care about this (especially in equality tests)
-    __class__ = property(new_method_proxy(operator.attrgetter("__class__")))
-    __eq__ = new_method_proxy(operator.eq)
-    __ne__ = new_method_proxy(operator.ne)
-    __hash__ = new_method_proxy(hash)
-    __bool__ = new_method_proxy(bool)       # Python 3
-    __nonzero__ = __bool__                  # Python 2
 
 
 class lazy_property(property):

tests/utils_tests/test_lazyobject.py

@@ -0,0 +1,275 @@
+from __future__ import unicode_literals
+
+import copy
+import pickle
+import sys
+from unittest import TestCase
+
+from django.utils import six
+from django.utils.functional import LazyObject, SimpleLazyObject, empty
+
+
+class Foo(object):
+    """
+    A simple class with just one attribute.
+    """
+    foo = 'bar'
+
+    def __eq__(self, other):
+        return self.foo == other.foo
+
+
+class LazyObjectTestCase(TestCase):
+    def lazy_wrap(self, wrapped_object):
+        """
+        Wrap the given object into a LazyObject
+        """
+        class AdHocLazyObject(LazyObject):
+            def _setup(self):
+                self._wrapped = wrapped_object
+
+        return AdHocLazyObject()
+
+    def test_getattr(self):
+        obj = self.lazy_wrap(Foo())
+        self.assertEqual(obj.foo, 'bar')
+
+    def test_setattr(self):
+        obj = self.lazy_wrap(Foo())
+        obj.foo = 'BAR'
+        obj.bar = 'baz'
+        self.assertEqual(obj.foo, 'BAR')
+        self.assertEqual(obj.bar, 'baz')
+
+    def test_setattr2(self):
+        # Same as test_setattr but in reversed order
+        obj = self.lazy_wrap(Foo())
+        obj.bar = 'baz'
+        obj.foo = 'BAR'
+        self.assertEqual(obj.foo, 'BAR')
+        self.assertEqual(obj.bar, 'baz')
+
+    def test_delattr(self):
+        obj = self.lazy_wrap(Foo())
+        obj.bar = 'baz'
+        self.assertEqual(obj.bar, 'baz')
+        del obj.bar
+        with self.assertRaises(AttributeError):
+            obj.bar
+
+    def test_cmp(self):
+        obj1 = self.lazy_wrap('foo')
+        obj2 = self.lazy_wrap('bar')
+        obj3 = self.lazy_wrap('foo')
+        self.assertEqual(obj1, 'foo')
+        self.assertEqual(obj1, obj3)
+        self.assertNotEqual(obj1, obj2)
+        self.assertNotEqual(obj1, 'bar')
+
+    def test_bytes(self):
+        obj = self.lazy_wrap(b'foo')
+        self.assertEqual(bytes(obj), b'foo')
+
+    def test_text(self):
+        obj = self.lazy_wrap('foo')
+        self.assertEqual(six.text_type(obj), 'foo')
+
+    def test_bool(self):
+        # Refs #21840
+        for f in [False, 0, (), {}, [], None, set()]:
+            self.assertFalse(self.lazy_wrap(f))
+        for t in [True, 1, (1,), {1: 2}, [1], object(), {1}]:
+            self.assertTrue(t)
+
+    def test_dir(self):
+        obj = self.lazy_wrap('foo')
+        self.assertEqual(dir(obj), dir('foo'))
+
+    def test_len(self):
+        for seq in ['asd', [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}]:
+            obj = self.lazy_wrap(seq)
+            self.assertEqual(len(obj), 3)
+
+    def test_class(self):
+        self.assertIsInstance(self.lazy_wrap(42), int)
+
+        class Bar(Foo):
+            pass
+
+        self.assertIsInstance(self.lazy_wrap(Bar()), Foo)
+
+    def test_hash(self):
+        obj = self.lazy_wrap('foo')
+        d = {}
+        d[obj] = 'bar'
+        self.assertIn('foo', d)
+        self.assertEqual(d['foo'], 'bar')
+
+    def test_contains(self):
+        test_data = [
+            ('c', 'abcde'),
+            (2, [1, 2, 3]),
+            ('a', {'a': 1, 'b': 2, 'c': 3}),
+            (2, {1, 2, 3}),
+        ]
+        for needle, haystack in test_data:
+            self.assertIn(needle, self.lazy_wrap(haystack))
+
+        # __contains__ doesn't work when the haystack is a string and the needle a LazyObject
+        for needle_haystack in test_data[1:]:
+            self.assertIn(self.lazy_wrap(needle), haystack)
+            self.assertIn(self.lazy_wrap(needle), self.lazy_wrap(haystack))
+
+    def test_getitem(self):
+        obj_list = self.lazy_wrap([1, 2, 3])
+        obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})
+
+        self.assertEqual(obj_list[0], 1)
+        self.assertEqual(obj_list[-1], 3)
+        self.assertEqual(obj_list[1:2], [2])
+
+        self.assertEqual(obj_dict['b'], 2)
+
+        with self.assertRaises(IndexError):
+            obj_list[3]
+
+        with self.assertRaises(KeyError):
+            obj_dict['f']
+
+    def test_setitem(self):
+        obj_list = self.lazy_wrap([1, 2, 3])
+        obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})
+
+        obj_list[0] = 100
+        self.assertEqual(obj_list, [100, 2, 3])
+        obj_list[1:2] = [200, 300, 400]
+        self.assertEqual(obj_list, [100, 200, 300, 400, 3])
+
+        obj_dict['a'] = 100
+        obj_dict['d'] = 400
+        self.assertEqual(obj_dict, {'a': 100, 'b': 2, 'c': 3, 'd': 400})
+
+    def test_delitem(self):
+        obj_list = self.lazy_wrap([1, 2, 3])
+        obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})
+
+        del obj_list[-1]
+        del obj_dict['c']
+        self.assertEqual(obj_list, [1, 2])
+        self.assertEqual(obj_dict, {'a': 1, 'b': 2})
+
+        with self.assertRaises(IndexError):
+            del obj_list[3]
+
+        with self.assertRaises(KeyError):
+            del obj_dict['f']
+
+    def test_iter(self):
+        # LazyObjects don't actually implements __iter__ but you can still
+        # iterate over them because they implement __getitem__
+        obj = self.lazy_wrap([1, 2, 3])
+        for expected, actual in zip([1, 2, 3], obj):
+            self.assertEqual(expected, actual)
+
+    def test_pickle(self):
+        # See ticket #16563
+        obj = self.lazy_wrap(Foo())
+        pickled = pickle.dumps(obj)
+        unpickled = pickle.loads(pickled)
+        self.assertIsInstance(unpickled, Foo)
+        self.assertEqual(unpickled, obj)
+        self.assertEqual(unpickled.foo, obj.foo)
+
+    def test_deepcopy(self):
+        # Check that we *can* do deep copy, and that it returns the right
+        # objects.
+
+        l = [1, 2, 3]
+
+        obj = self.lazy_wrap(l)
+        len(l)  # forces evaluation
+        obj2 = copy.deepcopy(obj)
+
+        self.assertIsInstance(obj2, list)
+        self.assertEqual(obj2, [1, 2, 3])
+
+    def test_deepcopy_no_evaluation(self):
+        # copying doesn't force evaluation
+
+        l = [1, 2, 3]
+
+        obj = self.lazy_wrap(l)
+        obj2 = copy.deepcopy(obj)
+
+        # Copying shouldn't force evaluation
+        self.assertIs(obj._wrapped, empty)
+        self.assertIs(obj2._wrapped, empty)
+
+
+class SimpleLazyObjectTestCase(LazyObjectTestCase):
+    # By inheriting from LazyObjectTestCase and redefining the lazy_wrap()
+    # method which all testcases use, we get to make sure all behaviors
+    # tested in the parent testcase also apply to SimpleLazyObject.
+    def lazy_wrap(self, wrapped_object):
+        return SimpleLazyObject(lambda: wrapped_object)
+
+    def test_repr(self):
+        # First, for an unevaluated SimpleLazyObject
+        obj = self.lazy_wrap(42)
+        # __repr__ contains __repr__ of setup function and does not evaluate
+        # the SimpleLazyObject
+        self.assertRegexpMatches(repr(obj), '^<SimpleLazyObject:')
+        self.assertIs(obj._wrapped, empty)  # make sure evaluation hasn't been triggered
+
+        self.assertEqual(obj, 42)  # evaluate the lazy object
+        self.assertIsInstance(obj._wrapped, int)
+        self.assertEqual(repr(obj), '<SimpleLazyObject: 42>')
+
+    def test_trace(self):
+        # See ticket #19456
+        old_trace_func = sys.gettrace()
+        try:
+            def trace_func(frame, event, arg):
+                frame.f_locals['self'].__class__
+                if old_trace_func is not None:
+                    old_trace_func(frame, event, arg)
+            sys.settrace(trace_func)
+            self.lazy_wrap(None)
+        finally:
+            sys.settrace(old_trace_func)
+
+    def test_none(self):
+        i = [0]
+
+        def f():
+            i[0] += 1
+            return None
+
+        x = SimpleLazyObject(f)
+        self.assertEqual(str(x), "None")
+        self.assertEqual(i, [1])
+        self.assertEqual(str(x), "None")
+        self.assertEqual(i, [1])
+
+    def test_dict(self):
+        # See ticket #18447
+        lazydict = SimpleLazyObject(lambda: {'one': 1})
+        self.assertEqual(lazydict['one'], 1)
+        lazydict['one'] = -1
+        self.assertEqual(lazydict['one'], -1)
+        self.assertTrue('one' in lazydict)
+        self.assertFalse('two' in lazydict)
+        self.assertEqual(len(lazydict), 1)
+        del lazydict['one']
+        with self.assertRaises(KeyError):
+            lazydict['one']
+
+    def test_list_set(self):
+        lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5])
+        lazy_set = SimpleLazyObject(lambda: set([1, 2, 3, 4]))
+        self.assertTrue(1 in lazy_list)
+        self.assertTrue(1 in lazy_set)
+        self.assertFalse(6 in lazy_list)
+        self.assertFalse(6 in lazy_set)
+        self.assertEqual(len(lazy_list), 5)
+        self.assertEqual(len(lazy_set), 4)

tests/utils_tests/test_simplelazyobject.py

@@ -1,184 +1,14 @@
 from __future__ import unicode_literals
 
-import copy
 import pickle
-import sys
-from unittest import TestCase
 
 from django.contrib.auth.models import User
-from django.test import TestCase as DjangoTestCase
+from django.test import TestCase
 from django.utils import six
-from django.utils.functional import SimpleLazyObject, empty
+from django.utils.functional import SimpleLazyObject
 
 
-class _ComplexObject(object):
+class TestUtilsSimpleLazyObjectDjangoTestCase(TestCase):
-    def __init__(self, name):
-        self.name = name
-
-    def __eq__(self, other):
-        return self.name == other.name
-
-    def __hash__(self):
-        return hash(self.name)
-
-    if six.PY3:
-        def __bytes__(self):
-            return ("I am _ComplexObject(%r)" % self.name).encode("utf-8")
-
-        def __str__(self):
-            return self.name
-
-    else:
-        def __str__(self):
-            return b"I am _ComplexObject(%r)" % str(self.name)
-
-        def __unicode__(self):
-            return self.name
-
-    def __repr__(self):
-        return "_ComplexObject(%r)" % self.name
-
-
-complex_object = lambda: _ComplexObject("joe")
-
-
-class TestUtilsSimpleLazyObject(TestCase):
-    """
-    Tests for SimpleLazyObject
-    """
-    # Note that concrete use cases for SimpleLazyObject are also found in the
-    # auth context processor tests (unless the implementation of that function
-    # is changed).
-
-    def test_equality(self):
-        self.assertEqual(complex_object(), SimpleLazyObject(complex_object))
-        self.assertEqual(SimpleLazyObject(complex_object), complex_object())
-
-    def test_hash(self):
-        # hash() equality would not be true for many objects, but it should be
-        # for _ComplexObject
-        self.assertEqual(hash(complex_object()),
-                         hash(SimpleLazyObject(complex_object)))
-
-    def test_repr(self):
-        # First, for an unevaluated SimpleLazyObject
-        x = SimpleLazyObject(complex_object)
-        # __repr__ contains __repr__ of setup function and does not evaluate
-        # the SimpleLazyObject
-        self.assertEqual("<SimpleLazyObject: %r>" % complex_object, repr(x))
-        self.assertEqual(empty, x._wrapped)
-
-        # Second, for an evaluated SimpleLazyObject
-        x.name  # evaluate
-        self.assertIsInstance(x._wrapped, _ComplexObject)
-        # __repr__ contains __repr__ of wrapped object
-        self.assertEqual("<SimpleLazyObject: %r>" % x._wrapped, repr(x))
-
-    def test_bytes(self):
-        self.assertEqual(b"I am _ComplexObject('joe')",
-                bytes(SimpleLazyObject(complex_object)))
-
-    def test_text(self):
-        self.assertEqual("joe", six.text_type(SimpleLazyObject(complex_object)))
-
-    def test_class(self):
-        # This is important for classes that use __class__ in things like
-        # equality tests.
-        self.assertEqual(_ComplexObject, SimpleLazyObject(complex_object).__class__)
-
-    def test_deepcopy(self):
-        # Check that we *can* do deep copy, and that it returns the right
-        # objects.
-
-        # First, for an unevaluated SimpleLazyObject
-        s = SimpleLazyObject(complex_object)
-        self.assertIs(s._wrapped, empty)
-        s2 = copy.deepcopy(s)
-        # something has gone wrong is s is evaluated
-        self.assertIs(s._wrapped, empty)
-        self.assertEqual(s2, complex_object())
-
-        # Second, for an evaluated SimpleLazyObject
-        s.name  # evaluate
-        self.assertIsNot(s._wrapped, empty)
-        s3 = copy.deepcopy(s)
-        self.assertEqual(s3, complex_object())
-
-    def test_none(self):
-        i = [0]
-
-        def f():
-            i[0] += 1
-            return None
-
-        x = SimpleLazyObject(f)
-        self.assertEqual(str(x), "None")
-        self.assertEqual(i, [1])
-        self.assertEqual(str(x), "None")
-        self.assertEqual(i, [1])
-
-    def test_bool(self):
-        x = SimpleLazyObject(lambda: 3)
-        self.assertTrue(x)
-        x = SimpleLazyObject(lambda: 0)
-        self.assertFalse(x)
-
-    def test_pickle_complex(self):
-        # See ticket #16563
-        x = SimpleLazyObject(complex_object)
-        pickled = pickle.dumps(x)
-        unpickled = pickle.loads(pickled)
-        self.assertEqual(unpickled, x)
-        self.assertEqual(six.text_type(unpickled), six.text_type(x))
-        self.assertEqual(unpickled.name, x.name)
-
-    def test_dict(self):
-        # See ticket #18447
-        lazydict = SimpleLazyObject(lambda: {'one': 1})
-        self.assertEqual(lazydict['one'], 1)
-        lazydict['one'] = -1
-        self.assertEqual(lazydict['one'], -1)
-        self.assertTrue('one' in lazydict)
-        self.assertFalse('two' in lazydict)
-        self.assertEqual(len(lazydict), 1)
-        del lazydict['one']
-        with self.assertRaises(KeyError):
-            lazydict['one']
-
-    def test_trace(self):
-        # See ticket #19456
-        old_trace_func = sys.gettrace()
-        try:
-            def trace_func(frame, event, arg):
-                frame.f_locals['self'].__class__
-                if old_trace_func is not None:
-                    old_trace_func(frame, event, arg)
-            sys.settrace(trace_func)
-            SimpleLazyObject(None)
-        finally:
-            sys.settrace(old_trace_func)
-
-    def test_not_equal(self):
-        lazy1 = SimpleLazyObject(lambda: 2)
-        lazy2 = SimpleLazyObject(lambda: 2)
-        lazy3 = SimpleLazyObject(lambda: 3)
-        self.assertEqual(lazy1, lazy2)
-        self.assertNotEqual(lazy1, lazy3)
-        self.assertTrue(lazy1 != lazy3)
-        self.assertFalse(lazy1 != lazy2)
-
-    def test_list_set(self):
-        lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5])
-        lazy_set = SimpleLazyObject(lambda: set([1, 2, 3, 4]))
-        self.assertTrue(1 in lazy_list)
-        self.assertTrue(1 in lazy_set)
-        self.assertFalse(6 in lazy_list)
-        self.assertFalse(6 in lazy_set)
-        self.assertEqual(len(lazy_list), 5)
-        self.assertEqual(len(lazy_set), 4)
-
-
-class TestUtilsSimpleLazyObjectDjangoTestCase(DjangoTestCase):
 
     def test_pickle_py2_regression(self):
         # See ticket #20212