From 61917aa08b4ab2bc35f3ffe87b7693bd8b58e205 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Wed, 29 Jan 2014 05:53:30 +0100 Subject: [PATCH 1/2] Fixed #21840 -- Moved dunder methods from SimpleLazyObject to LazyObject. This commit also added tests for LazyObject and refactored the testsuite of SimpleLazyObject so that it can share test cases with LazyObject. --- django/utils/functional.py | 116 +++++---- tests/utils_tests/test_lazyobject.py | 275 +++++++++++++++++++++ tests/utils_tests/test_simplelazyobject.py | 176 +------------ 3 files changed, 340 insertions(+), 227 deletions(-) create mode 100644 tests/utils_tests/test_lazyobject.py diff --git a/django/utils/functional.py b/django/utils/functional.py index 9668bc682b..35f48c6251 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -265,9 +265,62 @@ class LazyObject(object): """ raise NotImplementedError('subclasses of LazyObject must provide a _setup() method') + # Because we have messed with __class__ below, we confuse pickle as to what + # class we are pickling. It also appears to stop __reduce__ from being + # called. So, we define __getstate__ in a way that cooperates with the way + # that pickle interprets this class. This fails when the wrapped class is + # a builtin, but it is better than nothing. + def __getstate__(self): + if self._wrapped is empty: + self._setup() + return self._wrapped.__dict__ + + # Python 3.3 will call __reduce__ when pickling; this method is needed + # to serialize and deserialize correctly. + @classmethod + def __newobj__(cls, *args): + return cls.__new__(cls, *args) + + def __reduce_ex__(self, proto): + if proto >= 2: + # On Py3, since the default protocol is 3, pickle uses the + # ``__newobj__`` method (& more efficient opcodes) for writing. + return (self.__newobj__, (self.__class__,), self.__getstate__()) + else: + # On Py2, the default protocol is 0 (for back-compat) & the above + # code fails miserably (see regression test). Instead, we return + # exactly what's returned if there's no ``__reduce__`` method at + # all. + return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__()) + + def __deepcopy__(self, memo): + if self._wrapped is empty: + # We have to use type(self), not self.__class__, because the + # latter is proxied. + result = type(self)() + memo[id(self)] = result + return result + return copy.deepcopy(self._wrapped, memo) + + if six.PY3: + __bytes__ = new_method_proxy(bytes) + __str__ = new_method_proxy(str) + __bool__ = new_method_proxy(bool) + else: + __str__ = new_method_proxy(str) + __unicode__ = new_method_proxy(unicode) + __nonzero__ = new_method_proxy(bool) + # Introspection support __dir__ = new_method_proxy(dir) + # Need to pretend to be the wrapped class, for the sake of objects that + # care about this (especially in equality tests) + __class__ = property(new_method_proxy(operator.attrgetter("__class__"))) + __eq__ = new_method_proxy(operator.eq) + __ne__ = new_method_proxy(operator.ne) + __hash__ = new_method_proxy(hash) + # Dictionary methods support __getitem__ = new_method_proxy(operator.getitem) __setitem__ = new_method_proxy(operator.setitem) @@ -303,51 +356,6 @@ class SimpleLazyObject(LazyObject): def _setup(self): self._wrapped = self._setupfunc() - if six.PY3: - __bytes__ = new_method_proxy(bytes) - __str__ = new_method_proxy(str) - else: - __str__ = new_method_proxy(str) - __unicode__ = new_method_proxy(unicode) - - def __deepcopy__(self, memo): - if self._wrapped is empty: - # We have to use SimpleLazyObject, not self.__class__, because the - # latter is proxied. - result = SimpleLazyObject(self._setupfunc) - memo[id(self)] = result - return result - else: - return copy.deepcopy(self._wrapped, memo) - - # Because we have messed with __class__ below, we confuse pickle as to what - # class we are pickling. It also appears to stop __reduce__ from being - # called. So, we define __getstate__ in a way that cooperates with the way - # that pickle interprets this class. This fails when the wrapped class is - # a builtin, but it is better than nothing. - def __getstate__(self): - if self._wrapped is empty: - self._setup() - return self._wrapped.__dict__ - - # Python 3.3 will call __reduce__ when pickling; this method is needed - # to serialize and deserialize correctly. - @classmethod - def __newobj__(cls, *args): - return cls.__new__(cls, *args) - - def __reduce_ex__(self, proto): - if proto >= 2: - # On Py3, since the default protocol is 3, pickle uses the - # ``__newobj__`` method (& more efficient opcodes) for writing. - return (self.__newobj__, (self.__class__,), self.__getstate__()) - else: - # On Py2, the default protocol is 0 (for back-compat) & the above - # code fails miserably (see regression test). Instead, we return - # exactly what's returned if there's no ``__reduce__`` method at - # all. - return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__()) - # Return a meaningful representation of the lazy object for debugging # without evaluating the wrapped object. def __repr__(self): @@ -355,16 +363,16 @@ class SimpleLazyObject(LazyObject): repr_attr = self._setupfunc else: repr_attr = self._wrapped - return '' % repr_attr + return '<%s: %r>' % (type(self).__name__, repr_attr) - # Need to pretend to be the wrapped class, for the sake of objects that - # care about this (especially in equality tests) - __class__ = property(new_method_proxy(operator.attrgetter("__class__"))) - __eq__ = new_method_proxy(operator.eq) - __ne__ = new_method_proxy(operator.ne) - __hash__ = new_method_proxy(hash) - __bool__ = new_method_proxy(bool) # Python 3 - __nonzero__ = __bool__ # Python 2 + def __deepcopy__(self, memo): + if self._wrapped is empty: + # We have to use type(self), not self.__class__, because the + # latter is proxied. + result = SimpleLazyObject(self._setupfunc) + memo[id(self)] = result + return result + return copy.deepcopy(self._wrapped, memo) class lazy_property(property): diff --git a/tests/utils_tests/test_lazyobject.py b/tests/utils_tests/test_lazyobject.py new file mode 100644 index 0000000000..67b857aa4d --- /dev/null +++ b/tests/utils_tests/test_lazyobject.py @@ -0,0 +1,275 @@ +from __future__ import unicode_literals + +import copy +import pickle +import sys +from unittest import TestCase + +from django.utils import six +from django.utils.functional import LazyObject, SimpleLazyObject, empty + + +class Foo(object): + """ + A simple class with just one attribute. + """ + foo = 'bar' + + def __eq__(self, other): + return self.foo == other.foo + + +class LazyObjectTestCase(TestCase): + def lazy_wrap(self, wrapped_object): + """ + Wrap the given object into a LazyObject + """ + class AdHocLazyObject(LazyObject): + def _setup(self): + self._wrapped = wrapped_object + + return AdHocLazyObject() + + def test_getattr(self): + obj = self.lazy_wrap(Foo()) + self.assertEqual(obj.foo, 'bar') + + def test_setattr(self): + obj = self.lazy_wrap(Foo()) + obj.foo = 'BAR' + obj.bar = 'baz' + self.assertEqual(obj.foo, 'BAR') + self.assertEqual(obj.bar, 'baz') + + def test_setattr2(self): + # Same as test_setattr but in reversed order + obj = self.lazy_wrap(Foo()) + obj.bar = 'baz' + obj.foo = 'BAR' + self.assertEqual(obj.foo, 'BAR') + self.assertEqual(obj.bar, 'baz') + + def test_delattr(self): + obj = self.lazy_wrap(Foo()) + obj.bar = 'baz' + self.assertEqual(obj.bar, 'baz') + del obj.bar + with self.assertRaises(AttributeError): + obj.bar + + def test_cmp(self): + obj1 = self.lazy_wrap('foo') + obj2 = self.lazy_wrap('bar') + obj3 = self.lazy_wrap('foo') + self.assertEqual(obj1, 'foo') + self.assertEqual(obj1, obj3) + self.assertNotEqual(obj1, obj2) + self.assertNotEqual(obj1, 'bar') + + def test_bytes(self): + obj = self.lazy_wrap(b'foo') + self.assertEqual(bytes(obj), b'foo') + + def test_text(self): + obj = self.lazy_wrap('foo') + self.assertEqual(six.text_type(obj), 'foo') + + def test_bool(self): + # Refs #21840 + for f in [False, 0, (), {}, [], None, set()]: + self.assertFalse(self.lazy_wrap(f)) + for t in [True, 1, (1,), {1: 2}, [1], object(), {1}]: + self.assertTrue(t) + + def test_dir(self): + obj = self.lazy_wrap('foo') + self.assertEqual(dir(obj), dir('foo')) + + def test_len(self): + for seq in ['asd', [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}]: + obj = self.lazy_wrap(seq) + self.assertEqual(len(obj), 3) + + def test_class(self): + self.assertIsInstance(self.lazy_wrap(42), int) + + class Bar(Foo): + pass + + self.assertIsInstance(self.lazy_wrap(Bar()), Foo) + + def test_hash(self): + obj = self.lazy_wrap('foo') + d = {} + d[obj] = 'bar' + self.assertIn('foo', d) + self.assertEqual(d['foo'], 'bar') + + def test_contains(self): + test_data = [ + ('c', 'abcde'), + (2, [1, 2, 3]), + ('a', {'a': 1, 'b': 2, 'c': 3}), + (2, {1, 2, 3}), + ] + for needle, haystack in test_data: + self.assertIn(needle, self.lazy_wrap(haystack)) + + # __contains__ doesn't work when the haystack is a string and the needle a LazyObject + for needle_haystack in test_data[1:]: + self.assertIn(self.lazy_wrap(needle), haystack) + self.assertIn(self.lazy_wrap(needle), self.lazy_wrap(haystack)) + + def test_getitem(self): + obj_list = self.lazy_wrap([1, 2, 3]) + obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3}) + + self.assertEqual(obj_list[0], 1) + self.assertEqual(obj_list[-1], 3) + self.assertEqual(obj_list[1:2], [2]) + + self.assertEqual(obj_dict['b'], 2) + + with self.assertRaises(IndexError): + obj_list[3] + + with self.assertRaises(KeyError): + obj_dict['f'] + + def test_setitem(self): + obj_list = self.lazy_wrap([1, 2, 3]) + obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3}) + + obj_list[0] = 100 + self.assertEqual(obj_list, [100, 2, 3]) + obj_list[1:2] = [200, 300, 400] + self.assertEqual(obj_list, [100, 200, 300, 400, 3]) + + obj_dict['a'] = 100 + obj_dict['d'] = 400 + self.assertEqual(obj_dict, {'a': 100, 'b': 2, 'c': 3, 'd': 400}) + + def test_delitem(self): + obj_list = self.lazy_wrap([1, 2, 3]) + obj_dict = self.lazy_wrap({'a': 1, 'b': 2, 'c': 3}) + + del obj_list[-1] + del obj_dict['c'] + self.assertEqual(obj_list, [1, 2]) + self.assertEqual(obj_dict, {'a': 1, 'b': 2}) + + with self.assertRaises(IndexError): + del obj_list[3] + + with self.assertRaises(KeyError): + del obj_dict['f'] + + def test_iter(self): + # LazyObjects don't actually implements __iter__ but you can still + # iterate over them because they implement __getitem__ + obj = self.lazy_wrap([1, 2, 3]) + for expected, actual in zip([1, 2, 3], obj): + self.assertEqual(expected, actual) + + def test_pickle(self): + # See ticket #16563 + obj = self.lazy_wrap(Foo()) + pickled = pickle.dumps(obj) + unpickled = pickle.loads(pickled) + self.assertIsInstance(unpickled, Foo) + self.assertEqual(unpickled, obj) + self.assertEqual(unpickled.foo, obj.foo) + + def test_deepcopy(self): + # Check that we *can* do deep copy, and that it returns the right + # objects. + + l = [1, 2, 3] + + obj = self.lazy_wrap(l) + len(l) # forces evaluation + obj2 = copy.deepcopy(obj) + + self.assertIsInstance(obj2, list) + self.assertEqual(obj2, [1, 2, 3]) + + def test_deepcopy_no_evaluation(self): + # copying doesn't force evaluation + + l = [1, 2, 3] + + obj = self.lazy_wrap(l) + obj2 = copy.deepcopy(obj) + + # Copying shouldn't force evaluation + self.assertIs(obj._wrapped, empty) + self.assertIs(obj2._wrapped, empty) + + +class SimpleLazyObjectTestCase(LazyObjectTestCase): + # By inheriting from LazyObjectTestCase and redefining the lazy_wrap() + # method which all testcases use, we get to make sure all behaviors + # tested in the parent testcase also apply to SimpleLazyObject. + def lazy_wrap(self, wrapped_object): + return SimpleLazyObject(lambda: wrapped_object) + + def test_repr(self): + # First, for an unevaluated SimpleLazyObject + obj = self.lazy_wrap(42) + # __repr__ contains __repr__ of setup function and does not evaluate + # the SimpleLazyObject + self.assertRegexpMatches(repr(obj), '^') + + def test_trace(self): + # See ticket #19456 + old_trace_func = sys.gettrace() + try: + def trace_func(frame, event, arg): + frame.f_locals['self'].__class__ + if old_trace_func is not None: + old_trace_func(frame, event, arg) + sys.settrace(trace_func) + self.lazy_wrap(None) + finally: + sys.settrace(old_trace_func) + + def test_none(self): + i = [0] + + def f(): + i[0] += 1 + return None + + x = SimpleLazyObject(f) + self.assertEqual(str(x), "None") + self.assertEqual(i, [1]) + self.assertEqual(str(x), "None") + self.assertEqual(i, [1]) + + def test_dict(self): + # See ticket #18447 + lazydict = SimpleLazyObject(lambda: {'one': 1}) + self.assertEqual(lazydict['one'], 1) + lazydict['one'] = -1 + self.assertEqual(lazydict['one'], -1) + self.assertTrue('one' in lazydict) + self.assertFalse('two' in lazydict) + self.assertEqual(len(lazydict), 1) + del lazydict['one'] + with self.assertRaises(KeyError): + lazydict['one'] + + def test_list_set(self): + lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5]) + lazy_set = SimpleLazyObject(lambda: set([1, 2, 3, 4])) + self.assertTrue(1 in lazy_list) + self.assertTrue(1 in lazy_set) + self.assertFalse(6 in lazy_list) + self.assertFalse(6 in lazy_set) + self.assertEqual(len(lazy_list), 5) + self.assertEqual(len(lazy_set), 4) diff --git a/tests/utils_tests/test_simplelazyobject.py b/tests/utils_tests/test_simplelazyobject.py index 14ad393bfa..bad47a2f89 100644 --- a/tests/utils_tests/test_simplelazyobject.py +++ b/tests/utils_tests/test_simplelazyobject.py @@ -1,184 +1,14 @@ from __future__ import unicode_literals -import copy import pickle -import sys -from unittest import TestCase from django.contrib.auth.models import User -from django.test import TestCase as DjangoTestCase +from django.test import TestCase from django.utils import six -from django.utils.functional import SimpleLazyObject, empty +from django.utils.functional import SimpleLazyObject -class _ComplexObject(object): - def __init__(self, name): - self.name = name - - def __eq__(self, other): - return self.name == other.name - - def __hash__(self): - return hash(self.name) - - if six.PY3: - def __bytes__(self): - return ("I am _ComplexObject(%r)" % self.name).encode("utf-8") - - def __str__(self): - return self.name - - else: - def __str__(self): - return b"I am _ComplexObject(%r)" % str(self.name) - - def __unicode__(self): - return self.name - - def __repr__(self): - return "_ComplexObject(%r)" % self.name - - -complex_object = lambda: _ComplexObject("joe") - - -class TestUtilsSimpleLazyObject(TestCase): - """ - Tests for SimpleLazyObject - """ - # Note that concrete use cases for SimpleLazyObject are also found in the - # auth context processor tests (unless the implementation of that function - # is changed). - - def test_equality(self): - self.assertEqual(complex_object(), SimpleLazyObject(complex_object)) - self.assertEqual(SimpleLazyObject(complex_object), complex_object()) - - def test_hash(self): - # hash() equality would not be true for many objects, but it should be - # for _ComplexObject - self.assertEqual(hash(complex_object()), - hash(SimpleLazyObject(complex_object))) - - def test_repr(self): - # First, for an unevaluated SimpleLazyObject - x = SimpleLazyObject(complex_object) - # __repr__ contains __repr__ of setup function and does not evaluate - # the SimpleLazyObject - self.assertEqual("" % complex_object, repr(x)) - self.assertEqual(empty, x._wrapped) - - # Second, for an evaluated SimpleLazyObject - x.name # evaluate - self.assertIsInstance(x._wrapped, _ComplexObject) - # __repr__ contains __repr__ of wrapped object - self.assertEqual("" % x._wrapped, repr(x)) - - def test_bytes(self): - self.assertEqual(b"I am _ComplexObject('joe')", - bytes(SimpleLazyObject(complex_object))) - - def test_text(self): - self.assertEqual("joe", six.text_type(SimpleLazyObject(complex_object))) - - def test_class(self): - # This is important for classes that use __class__ in things like - # equality tests. - self.assertEqual(_ComplexObject, SimpleLazyObject(complex_object).__class__) - - def test_deepcopy(self): - # Check that we *can* do deep copy, and that it returns the right - # objects. - - # First, for an unevaluated SimpleLazyObject - s = SimpleLazyObject(complex_object) - self.assertIs(s._wrapped, empty) - s2 = copy.deepcopy(s) - # something has gone wrong is s is evaluated - self.assertIs(s._wrapped, empty) - self.assertEqual(s2, complex_object()) - - # Second, for an evaluated SimpleLazyObject - s.name # evaluate - self.assertIsNot(s._wrapped, empty) - s3 = copy.deepcopy(s) - self.assertEqual(s3, complex_object()) - - def test_none(self): - i = [0] - - def f(): - i[0] += 1 - return None - - x = SimpleLazyObject(f) - self.assertEqual(str(x), "None") - self.assertEqual(i, [1]) - self.assertEqual(str(x), "None") - self.assertEqual(i, [1]) - - def test_bool(self): - x = SimpleLazyObject(lambda: 3) - self.assertTrue(x) - x = SimpleLazyObject(lambda: 0) - self.assertFalse(x) - - def test_pickle_complex(self): - # See ticket #16563 - x = SimpleLazyObject(complex_object) - pickled = pickle.dumps(x) - unpickled = pickle.loads(pickled) - self.assertEqual(unpickled, x) - self.assertEqual(six.text_type(unpickled), six.text_type(x)) - self.assertEqual(unpickled.name, x.name) - - def test_dict(self): - # See ticket #18447 - lazydict = SimpleLazyObject(lambda: {'one': 1}) - self.assertEqual(lazydict['one'], 1) - lazydict['one'] = -1 - self.assertEqual(lazydict['one'], -1) - self.assertTrue('one' in lazydict) - self.assertFalse('two' in lazydict) - self.assertEqual(len(lazydict), 1) - del lazydict['one'] - with self.assertRaises(KeyError): - lazydict['one'] - - def test_trace(self): - # See ticket #19456 - old_trace_func = sys.gettrace() - try: - def trace_func(frame, event, arg): - frame.f_locals['self'].__class__ - if old_trace_func is not None: - old_trace_func(frame, event, arg) - sys.settrace(trace_func) - SimpleLazyObject(None) - finally: - sys.settrace(old_trace_func) - - def test_not_equal(self): - lazy1 = SimpleLazyObject(lambda: 2) - lazy2 = SimpleLazyObject(lambda: 2) - lazy3 = SimpleLazyObject(lambda: 3) - self.assertEqual(lazy1, lazy2) - self.assertNotEqual(lazy1, lazy3) - self.assertTrue(lazy1 != lazy3) - self.assertFalse(lazy1 != lazy2) - - def test_list_set(self): - lazy_list = SimpleLazyObject(lambda: [1, 2, 3, 4, 5]) - lazy_set = SimpleLazyObject(lambda: set([1, 2, 3, 4])) - self.assertTrue(1 in lazy_list) - self.assertTrue(1 in lazy_set) - self.assertFalse(6 in lazy_list) - self.assertFalse(6 in lazy_set) - self.assertEqual(len(lazy_list), 5) - self.assertEqual(len(lazy_set), 4) - - -class TestUtilsSimpleLazyObjectDjangoTestCase(DjangoTestCase): +class TestUtilsSimpleLazyObjectDjangoTestCase(TestCase): def test_pickle_py2_regression(self): # See ticket #20212 From 5c5b266b5478aa4e8bd92f141c56f3a0997749eb Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Wed, 12 Mar 2014 23:09:18 +0100 Subject: [PATCH 2/2] Simplified implementation of collectstatic command. Since d2e242d16c6dde6f4736086fb38057424bed3edb made isinstance() calls work correctly on LazyObject, we can simplify the implementation of is_local_storage added in 7e27885c6e7588471fd94a4def16b7081577bdfc. --- .../staticfiles/management/commands/collectstatic.py | 7 +------ django/utils/functional.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py index ce2a9ebdd5..bc247c8177 100644 --- a/django/contrib/staticfiles/management/commands/collectstatic.py +++ b/django/contrib/staticfiles/management/commands/collectstatic.py @@ -7,7 +7,6 @@ from optparse import make_option from django.core.files.storage import FileSystemStorage from django.core.management.base import CommandError, NoArgsCommand from django.utils.encoding import smart_text -from django.utils.functional import LazyObject from django.utils.six.moves import input from django.contrib.staticfiles import finders, storage @@ -193,11 +192,7 @@ class Command(NoArgsCommand): self.stdout.write(msg) def is_local_storage(self): - if issubclass(self.storage.__class__, LazyObject): - storage = self.storage._wrapped - else: - storage = self.storage - return isinstance(storage, FileSystemStorage) + return isinstance(self.storage, FileSystemStorage) def clear_dir(self, path): """ diff --git a/django/utils/functional.py b/django/utils/functional.py index 35f48c6251..361b0f14fb 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -367,7 +367,7 @@ class SimpleLazyObject(LazyObject): def __deepcopy__(self, memo): if self._wrapped is empty: - # We have to use type(self), not self.__class__, because the + # We have to use SimpleLazyObject, not self.__class__, because the # latter is proxied. result = SimpleLazyObject(self._setupfunc) memo[id(self)] = result