Merge pull request #2221 from bmispelon/LazyObject-refactor

Fixed #21840 -- Moved dunder methods from SimpleLazyObject to LazyObject...
This commit is contained in:
Andrew Godwin 2014-03-19 21:01:24 -07:00
commit 356f064c49
4 changed files with 341 additions and 233 deletions

View File

@ -7,7 +7,6 @@ from optparse import make_option
from django.core.files.storage import FileSystemStorage
from django.core.management.base import CommandError, NoArgsCommand
from django.utils.encoding import smart_text
from django.utils.functional import LazyObject
from django.utils.six.moves import input
from django.contrib.staticfiles import finders, storage
@ -193,11 +192,7 @@ class Command(NoArgsCommand):
self.stdout.write(msg)
def is_local_storage(self):
if issubclass(self.storage.__class__, LazyObject):
storage = self.storage._wrapped
else:
storage = self.storage
return isinstance(storage, FileSystemStorage)
return isinstance(self.storage, FileSystemStorage)
def clear_dir(self, path):
"""

View File

@ -265,9 +265,62 @@ class LazyObject(object):
"""
raise NotImplementedError('subclasses of LazyObject must provide a _setup() method')
# Because we have messed with __class__ below, we confuse pickle as to what
# class we are pickling. It also appears to stop __reduce__ from being
# called. So, we define __getstate__ in a way that cooperates with the way
# that pickle interprets this class. This fails when the wrapped class is
# a builtin, but it is better than nothing.
def __getstate__(self):
if self._wrapped is empty:
self._setup()
return self._wrapped.__dict__
# Python 3.3 will call __reduce__ when pickling; this method is needed
# to serialize and deserialize correctly.
@classmethod
def __newobj__(cls, *args):
return cls.__new__(cls, *args)
def __reduce_ex__(self, proto):
if proto >= 2:
# On Py3, since the default protocol is 3, pickle uses the
# ``__newobj__`` method (& more efficient opcodes) for writing.
return (self.__newobj__, (self.__class__,), self.__getstate__())
else:
# On Py2, the default protocol is 0 (for back-compat) & the above
# code fails miserably (see regression test). Instead, we return
# exactly what's returned if there's no ``__reduce__`` method at
# all.
return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__())
def __deepcopy__(self, memo):
if self._wrapped is empty:
# We have to use type(self), not self.__class__, because the
# latter is proxied.
result = type(self)()
memo[id(self)] = result
return result
return copy.deepcopy(self._wrapped, memo)
if six.PY3:
__bytes__ = new_method_proxy(bytes)
__str__ = new_method_proxy(str)
__bool__ = new_method_proxy(bool)
else:
__str__ = new_method_proxy(str)
__unicode__ = new_method_proxy(unicode)
__nonzero__ = new_method_proxy(bool)
# Introspection support
__dir__ = new_method_proxy(dir)
# Need to pretend to be the wrapped class, for the sake of objects that
# care about this (especially in equality tests)
__class__ = property(new_method_proxy(operator.attrgetter("__class__")))
__eq__ = new_method_proxy(operator.eq)
__ne__ = new_method_proxy(operator.ne)
__hash__ = new_method_proxy(hash)
# Dictionary methods support
__getitem__ = new_method_proxy(operator.getitem)
__setitem__ = new_method_proxy(operator.setitem)
@ -303,12 +356,14 @@ class SimpleLazyObject(LazyObject):
def _setup(self):
self._wrapped = self._setupfunc()
if six.PY3:
__bytes__ = new_method_proxy(bytes)
__str__ = new_method_proxy(str)
else:
__str__ = new_method_proxy(str)
__unicode__ = new_method_proxy(unicode)
# Return a meaningful representation of the lazy object for debugging
# without evaluating the wrapped object.
def __repr__(self):
if self._wrapped is empty:
repr_attr = self._setupfunc
else:
repr_attr = self._wrapped
return '<%s: %r>' % (type(self).__name__, repr_attr)
def __deepcopy__(self, memo):
if self._wrapped is empty:
@ -317,54 +372,7 @@ class SimpleLazyObject(LazyObject):
result = SimpleLazyObject(self._setupfunc)
memo[id(self)] = result
return result
else:
return copy.deepcopy(self._wrapped, memo)
# Because we have messed with __class__ below, we confuse pickle as to what
# class we are pickling. It also appears to stop __reduce__ from being
# called. So, we define __getstate__ in a way that cooperates with the way
# that pickle interprets this class. This fails when the wrapped class is
# a builtin, but it is better than nothing.
def __getstate__(self):
if self._wrapped is empty:
self._setup()
return self._wrapped.__dict__
# Python 3.3 will call __reduce__ when pickling; this method is needed
# to serialize and deserialize correctly.
@classmethod
def __newobj__(cls, *args):
return cls.__new__(cls, *args)
def __reduce_ex__(self, proto):
if proto >= 2:
# On Py3, since the default protocol is 3, pickle uses the
# ``__newobj__`` method (& more efficient opcodes) for writing.
return (self.__newobj__, (self.__class__,), self.__getstate__())
else:
# On Py2, the default protocol is 0 (for back-compat) & the above
# code fails miserably (see regression test). Instead, we return
# exactly what's returned if there's no ``__reduce__`` method at
# all.
return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__())
# Return a meaningful representation of the lazy object for debugging
# without evaluating the wrapped object.
def __repr__(self):
if self._wrapped is empty:
repr_attr = self._setupfunc
else:
repr_attr = self._wrapped
return '<SimpleLazyObject: %r>' % repr_attr
# Need to pretend to be the wrapped class, for the sake of objects that
# care about this (especially in equality tests)
__class__ = property(new_method_proxy(operator.attrgetter("__class__")))
__eq__ = new_method_proxy(operator.eq)
__ne__ = new_method_proxy(operator.ne)
__hash__ = new_method_proxy(hash)
__bool__ = new_method_proxy(bool) # Python 3
__nonzero__ = __bool__ # Python 2
return copy.deepcopy(self._wrapped, memo)
class lazy_property(property):

View File

@ -0,0 +1,275 @@
from __future__ import unicode_literals
import copy
import pickle
import sys
from unittest import TestCase
from django.utils import six
from django.utils.functional import LazyObject, SimpleLazyObject, empty
class Foo(object):
"""
A simple class with just one attribute.
"""
foo = 'bar'
def __eq__(self, other):
return self.foo == other.foo
class LazyObjectTestCase(TestCase):
def lazy_wrap(self, wrapped_object):
"""
Wrap the given object into a LazyObject
"""
class AdHocLazyObject(LazyObject):
def _setup(self):
self._wrapped = wrapped_object
return AdHocLazyObject()
def test_getattr(self):
obj = self.lazy_wrap(Foo())
self.assertEqual(obj.foo, 'bar')
def test_setattr(self):
obj = self.lazy_wrap(Foo())
obj.foo = 'BAR'
obj.bar = 'baz'
self.assertEqual(obj.foo, 'BAR')
self.assertEqual(obj.bar, 'baz')
def test_setattr2(self):
# Same as test_setattr but in reversed order
obj = self.lazy_wrap(Foo())
obj.bar = 'baz'
obj.foo = 'BAR'
self.assertEqual(obj.foo, 'BAR')
self.assertEqual(obj.bar, 'baz')
def test_delattr(self):
obj = self.lazy_wrap(Foo())
obj.bar = 'baz'
self.assertEqual(obj.bar, 'baz')
del obj.bar
with self.assertRaises(AttributeError):
obj.bar
def test_cmp(self):
obj1 = self.lazy_wrap('foo')
obj2 = self.lazy_wrap('bar')
obj3 = self.lazy_wrap('foo')
self.assertEqual(obj1, 'foo')
self.assertEqual(obj1, obj3)
self.assertNotEqual(obj1, obj2)
self.assertNotEqual(obj1, 'bar')
def test_bytes(self):
obj = self.lazy_wrap(b'foo')
self.assertEqual(bytes(obj), b'foo')
def test_text(self):
obj = self.lazy_wrap('foo')
self.assertEqual(six.text_type(obj), 'foo')
def test_bool(self):
# Refs #21840
for f in [False, 0, (), {}, [], None, set()]:
self.assertFalse(self.lazy_wrap(f))
for t in [True, 1, (1,), {1: 2}, [1], object(), {1}]:
self.assertTrue(t)
def test_dir(self):
obj = self.lazy_wrap('foo')
self.assertEqual(dir(obj), dir('foo'))
def test_len(self):
for seq in ['asd', [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}]:
obj = self.lazy_wrap(seq)
self.assertEqual(len(obj), 3)
def test_class(self):
self.assertIsInstance(self.lazy_wrap(42), int)
class Bar(Foo):
pass
self.assertIsInstance(self.lazy_wrap(Bar()), Foo)
def test_hash(self):
obj = self.lazy_wrap('foo')
d = {}
d[obj] = 'bar'
self.assertIn('foo', d)
self.assertEqual(d['foo'], 'bar')
def test_contains(self):
test_data = [
('c', 'abcde'),
(2, [1, 2, 3]),
('a', {'a': 1, 'b': 2, 'c': 3}),
(2, {1, 2, 3}),
]
for needle, haystack in test_data:
self.assertIn(needle, self.lazy_wrap(haystack))
# __contains__ doesn't work when the haystack is a string and the needle a LazyObject
for needle_haystack in test_data[1:]:
self.assertIn(self.lazy_wrap(needle), haystack)
self.assertIn(self.lazy_wrap(needle), self.lazy_wrap(haystack))
def test_getitem(self):
obj_list = self.lazy_wrap([1, 2, 3])
obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})
self.assertEqual(obj_list[0], 1)
self.assertEqual(obj_list[-1], 3)
self.assertEqual(obj_list[1:2], [2])
self.assertEqual(obj_dict['b'], 2)
with self.assertRaises(IndexError):
obj_list[3]
with self.assertRaises(KeyError):
obj_dict['f']
def test_setitem(self):
obj_list = self.lazy_wrap([1, 2, 3])
obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})
obj_list[0] = 100
self.assertEqual(obj_list, [100, 2, 3])
obj_list[1:2] = [200, 300, 400]
self.assertEqual(obj_list, [100, 200, 300, 400, 3])
obj_dict['a'] = 100
obj_dict['d'] = 400
self.assertEqual(obj_dict, {'a': 100, 'b': 2, 'c': 3, 'd': 400})
def test_delitem(self):
obj_list = self.lazy_wrap([1, 2, 3])
obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3})
del obj_list[-1]
del obj_dict['c']
self.assertEqual(obj_list, [1, 2])
self.assertEqual(obj_dict, {'a': 1, 'b': 2})
with self.assertRaises(IndexError):
del obj_list[3]
with self.assertRaises(KeyError):
del obj_dict['f']
def test_iter(self):
# LazyObjects don't actually implements __iter__ but you can still
# iterate over them because they implement __getitem__
obj = self.lazy_wrap([1, 2, 3])
for expected, actual in zip([1, 2, 3], obj):
self.assertEqual(expected, actual)
def test_pickle(self):
# See ticket #16563
obj = self.lazy_wrap(Foo())
pickled = pickle.dumps(obj)
unpickled = pickle.loads(pickled)
self.assertIsInstance(unpickled, Foo)
self.assertEqual(unpickled, obj)
self.assertEqual(unpickled.foo, obj.foo)
def test_deepcopy(self):
# Check that we *can* do deep copy, and that it returns the right
# objects.
l = [1, 2, 3]
obj = self.lazy_wrap(l)
len(l) # forces evaluation
obj2 = copy.deepcopy(obj)
self.assertIsInstance(obj2, list)
self.assertEqual(obj2, [1, 2, 3])
def test_deepcopy_no_evaluation(self):
# copying doesn't force evaluation
l = [1, 2, 3]
obj = self.lazy_wrap(l)
obj2 = copy.deepcopy(obj)
# Copying shouldn't force evaluation
self.assertIs(obj._wrapped, empty)
self.assertIs(obj2._wrapped, empty)
class SimpleLazyObjectTestCase(LazyObjectTestCase):
# By inheriting from LazyObjectTestCase and redefining the lazy_wrap()
# method which all testcases use, we get to make sure all behaviors
# tested in the parent testcase also apply to SimpleLazyObject.
def lazy_wrap(self, wrapped_object):
return SimpleLazyObject(lambda: wrapped_object)
def test_repr(self):
# First, for an unevaluated SimpleLazyObject
obj = self.lazy_wrap(42)
# __repr__ contains __repr__ of setup function and does not evaluate
# the SimpleLazyObject
self.assertRegexpMatches(repr(obj), '^<SimpleLazyObject:')
self.assertIs(obj._wrapped, empty) # make sure evaluation hasn't been triggered
self.assertEqual(obj, 42) # evaluate the lazy object
self.assertIsInstance(obj._wrapped, int)
self.assertEqual(repr(obj), '<SimpleLazyObject: 42>')
def test_trace(self):
# See ticket #19456
old_trace_func = sys.gettrace()
try:
def trace_func(frame, event, arg):
frame.f_locals['self'].__class__
if old_trace_func is not None:
old_trace_func(frame, event, arg)
sys.settrace(trace_func)
self.lazy_wrap(None)
finally:
sys.settrace(old_trace_func)
def test_none(self):
i = [0]
def f():
i[0] += 1
return None
x = SimpleLazyObject(f)
self.assertEqual(str(x), "None")
self.assertEqual(i, [1])
self.assertEqual(str(x), "None")
self.assertEqual(i, [1])
def test_dict(self):
# See ticket #18447
lazydict = SimpleLazyObject(lambda: {'one': 1})
self.assertEqual(lazydict['one'], 1)
lazydict['one'] = -1
self.assertEqual(lazydict['one'], -1)
self.assertTrue('one' in lazydict)
self.assertFalse('two' in lazydict)
self.assertEqual(len(lazydict), 1)
del lazydict['one']
with self.assertRaises(KeyError):
lazydict['one']
def test_list_set(self):
lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5])
lazy_set = SimpleLazyObject(lambda: set([1, 2, 3, 4]))
self.assertTrue(1 in lazy_list)
self.assertTrue(1 in lazy_set)
self.assertFalse(6 in lazy_list)
self.assertFalse(6 in lazy_set)
self.assertEqual(len(lazy_list), 5)
self.assertEqual(len(lazy_set), 4)

View File

@ -1,184 +1,14 @@
from __future__ import unicode_literals
import copy
import pickle
import sys
from unittest import TestCase
from django.contrib.auth.models import User
from django.test import TestCase as DjangoTestCase
from django.test import TestCase
from django.utils import six
from django.utils.functional import SimpleLazyObject, empty
from django.utils.functional import SimpleLazyObject
class _ComplexObject(object):
def __init__(self, name):
self.name = name
def __eq__(self, other):
return self.name == other.name
def __hash__(self):
return hash(self.name)
if six.PY3:
def __bytes__(self):
return ("I am _ComplexObject(%r)" % self.name).encode("utf-8")
def __str__(self):
return self.name
else:
def __str__(self):
return b"I am _ComplexObject(%r)" % str(self.name)
def __unicode__(self):
return self.name
def __repr__(self):
return "_ComplexObject(%r)" % self.name
complex_object = lambda: _ComplexObject("joe")
class TestUtilsSimpleLazyObject(TestCase):
"""
Tests for SimpleLazyObject
"""
# Note that concrete use cases for SimpleLazyObject are also found in the
# auth context processor tests (unless the implementation of that function
# is changed).
def test_equality(self):
self.assertEqual(complex_object(), SimpleLazyObject(complex_object))
self.assertEqual(SimpleLazyObject(complex_object), complex_object())
def test_hash(self):
# hash() equality would not be true for many objects, but it should be
# for _ComplexObject
self.assertEqual(hash(complex_object()),
hash(SimpleLazyObject(complex_object)))
def test_repr(self):
# First, for an unevaluated SimpleLazyObject
x = SimpleLazyObject(complex_object)
# __repr__ contains __repr__ of setup function and does not evaluate
# the SimpleLazyObject
self.assertEqual("<SimpleLazyObject: %r>" % complex_object, repr(x))
self.assertEqual(empty, x._wrapped)
# Second, for an evaluated SimpleLazyObject
x.name # evaluate
self.assertIsInstance(x._wrapped, _ComplexObject)
# __repr__ contains __repr__ of wrapped object
self.assertEqual("<SimpleLazyObject: %r>" % x._wrapped, repr(x))
def test_bytes(self):
self.assertEqual(b"I am _ComplexObject('joe')",
bytes(SimpleLazyObject(complex_object)))
def test_text(self):
self.assertEqual("joe", six.text_type(SimpleLazyObject(complex_object)))
def test_class(self):
# This is important for classes that use __class__ in things like
# equality tests.
self.assertEqual(_ComplexObject, SimpleLazyObject(complex_object).__class__)
def test_deepcopy(self):
# Check that we *can* do deep copy, and that it returns the right
# objects.
# First, for an unevaluated SimpleLazyObject
s = SimpleLazyObject(complex_object)
self.assertIs(s._wrapped, empty)
s2 = copy.deepcopy(s)
# something has gone wrong is s is evaluated
self.assertIs(s._wrapped, empty)
self.assertEqual(s2, complex_object())
# Second, for an evaluated SimpleLazyObject
s.name # evaluate
self.assertIsNot(s._wrapped, empty)
s3 = copy.deepcopy(s)
self.assertEqual(s3, complex_object())
def test_none(self):
i = [0]
def f():
i[0] += 1
return None
x = SimpleLazyObject(f)
self.assertEqual(str(x), "None")
self.assertEqual(i, [1])
self.assertEqual(str(x), "None")
self.assertEqual(i, [1])
def test_bool(self):
x = SimpleLazyObject(lambda: 3)
self.assertTrue(x)
x = SimpleLazyObject(lambda: 0)
self.assertFalse(x)
def test_pickle_complex(self):
# See ticket #16563
x = SimpleLazyObject(complex_object)
pickled = pickle.dumps(x)
unpickled = pickle.loads(pickled)
self.assertEqual(unpickled, x)
self.assertEqual(six.text_type(unpickled), six.text_type(x))
self.assertEqual(unpickled.name, x.name)
def test_dict(self):
# See ticket #18447
lazydict = SimpleLazyObject(lambda: {'one': 1})
self.assertEqual(lazydict['one'], 1)
lazydict['one'] = -1
self.assertEqual(lazydict['one'], -1)
self.assertTrue('one' in lazydict)
self.assertFalse('two' in lazydict)
self.assertEqual(len(lazydict), 1)
del lazydict['one']
with self.assertRaises(KeyError):
lazydict['one']
def test_trace(self):
# See ticket #19456
old_trace_func = sys.gettrace()
try:
def trace_func(frame, event, arg):
frame.f_locals['self'].__class__
if old_trace_func is not None:
old_trace_func(frame, event, arg)
sys.settrace(trace_func)
SimpleLazyObject(None)
finally:
sys.settrace(old_trace_func)
def test_not_equal(self):
lazy1 = SimpleLazyObject(lambda: 2)
lazy2 = SimpleLazyObject(lambda: 2)
lazy3 = SimpleLazyObject(lambda: 3)
self.assertEqual(lazy1, lazy2)
self.assertNotEqual(lazy1, lazy3)
self.assertTrue(lazy1 != lazy3)
self.assertFalse(lazy1 != lazy2)
def test_list_set(self):
lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5])
lazy_set = SimpleLazyObject(lambda: set([1, 2, 3, 4]))
self.assertTrue(1 in lazy_list)
self.assertTrue(1 in lazy_set)
self.assertFalse(6 in lazy_list)
self.assertFalse(6 in lazy_set)
self.assertEqual(len(lazy_list), 5)
self.assertEqual(len(lazy_set), 4)
class TestUtilsSimpleLazyObjectDjangoTestCase(DjangoTestCase):
class TestUtilsSimpleLazyObjectDjangoTestCase(TestCase):
def test_pickle_py2_regression(self):
# See ticket #20212