[py3] Updated dict-like data structures for Python 3.

The keys/items/values methods return iterators in Python 3, and the
iterkeys/items/values methods don't exist in Python 3. The behavior
under Python 2 is unchanged.

django/utils/datastructures.py

@@ -1,6 +1,7 @@
 import copy
 import warnings
 from types import GeneratorType
+from django.utils import six
 
 
 class MergeDict(object):
@@ -31,30 +32,40 @@ class MergeDict(object):
         except KeyError:
             return default
 
+    # This is used by MergeDicts of MultiValueDicts.
     def getlist(self, key):
         for dict_ in self.dicts:
-            if key in dict_.keys():
+            if key in dict_:
                 return dict_.getlist(key)
         return []
 
-    def iteritems(self):
+    def _iteritems(self):
         seen = set()
         for dict_ in self.dicts:
-            for item in dict_.iteritems():
+            for item in six.iteritems(dict_):
-                k, v = item
+                k = item[0]
                 if k in seen:
                     continue
                 seen.add(k)
                 yield item
 
-    def iterkeys(self):
+    def _iterkeys(self):
-        for k, v in self.iteritems():
+        for k, v in self._iteritems():
             yield k
 
-    def itervalues(self):
+    def _itervalues(self):
-        for k, v in self.iteritems():
+        for k, v in self._iteritems():
             yield v
 
+    if six.PY3:
+        items = _iteritems
+        keys = _iterkeys
+        values = _itervalues
+    else:
+        iteritems = _iteritems
+        iterkeys = _iterkeys
+        itervalues = _itervalues
+
         def items(self):
             return list(self.iteritems())
 
@@ -71,7 +82,8 @@ class MergeDict(object):
         return False
 
     __contains__ = has_key
-    __iter__ = iterkeys
+
+    __iter__ = _iterkeys
 
     def copy(self):
         """Returns a copy of this object."""
@@ -117,7 +129,7 @@ class SortedDict(dict):
             data = list(data)
         super(SortedDict, self).__init__(data)
         if isinstance(data, dict):
-            self.keyOrder = data.keys()
+            self.keyOrder = list(six.iterkeys(data))
         else:
             self.keyOrder = []
             seen = set()
@@ -128,7 +140,7 @@ class SortedDict(dict):
 
     def __deepcopy__(self, memo):
         return self.__class__([(key, copy.deepcopy(value, memo))
-                               for key, value in self.iteritems()])
+                               for key, value in six.iteritems(self)])
 
     def __copy__(self):
         # The Python's default copy implementation will alter the state
@@ -162,28 +174,38 @@ class SortedDict(dict):
         self.keyOrder.remove(result[0])
         return result
 
-    def items(self):
+    def _iteritems(self):
-        return zip(self.keyOrder, self.values())
-
-    def iteritems(self):
         for key in self.keyOrder:
             yield key, self[key]
 
-    def keys(self):
+    def _iterkeys(self):
-        return self.keyOrder[:]
+        for key in self.keyOrder:
+            yield key
 
-    def iterkeys(self):
+    def _itervalues(self):
-        return iter(self.keyOrder)
-
-    def values(self):
-        return map(self.__getitem__, self.keyOrder)
-
-    def itervalues(self):
         for key in self.keyOrder:
             yield self[key]
 
+    if six.PY3:
+        items = _iteritems
+        keys = _iterkeys
+        values = _itervalues
+    else:
+        iteritems = _iteritems
+        iterkeys = _iterkeys
+        itervalues = _itervalues
+
+        def items(self):
+            return list(self.iteritems())
+
+        def keys(self):
+            return list(self.iterkeys())
+
+        def values(self):
+            return list(self.itervalues())
+
     def update(self, dict_):
-        for k, v in dict_.iteritems():
+        for k, v in six.iteritems(dict_):
             self[k] = v
 
     def setdefault(self, key, default):
@@ -226,7 +248,7 @@ class SortedDict(dict):
         Replaces the normal dict.__repr__ with a version that returns the keys
         in their sorted order.
         """
-        return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()])
+        return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in six.iteritems(self)])
 
     def clear(self):
         super(SortedDict, self).clear()
@@ -356,37 +378,40 @@ class MultiValueDict(dict):
         """Appends an item to the internal list associated with key."""
         self.setlistdefault(key).append(value)
 
-    def items(self):
+    def _iteritems(self):
-        """
-        Returns a list of (key, value) pairs, where value is the last item in
-        the list associated with the key.
-        """
-        return [(key, self[key]) for key in self.keys()]
-
-    def iteritems(self):
         """
         Yields (key, value) pairs, where value is the last item in the list
         associated with the key.
         """
-        for key in self.keys():
+        for key in self:
-            yield (key, self[key])
+            yield key, self[key]
+
+    def _iterlists(self):
+        """Yields (key, list) pairs."""
+        return six.iteritems(super(MultiValueDict, self))
+
+    def _itervalues(self):
+        """Yield the last value on every key list."""
+        for key in self:
+            yield self[key]
+
+    if six.PY3:
+        items = _iteritems
+        lists = _iterlists
+        values = _itervalues
+    else:
+        iteritems = _iteritems
+        iterlists = _iterlists
+        itervalues = _itervalues
+
+        def items(self):
+            return list(self.iteritems())
 
         def lists(self):
-        """Returns a list of (key, list) pairs."""
+            return list(self.iterlists())
-        return super(MultiValueDict, self).items()
-
-    def iterlists(self):
-        """Yields (key, list) pairs."""
-        return super(MultiValueDict, self).iteritems()
 
         def values(self):
-        """Returns a list of the last value on every key list."""
+            return list(self.itervalues())
-        return [self[key] for key in self.keys()]
-
-    def itervalues(self):
-        """Yield the last value on every key list."""
-        for key in self.iterkeys():
-            yield self[key]
 
     def copy(self):
         """Returns a shallow copy of this object."""
@@ -410,7 +435,7 @@ class MultiValueDict(dict):
                         self.setlistdefault(key).append(value)
                 except TypeError:
                     raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in kwargs.iteritems():
+        for key, value in six.iteritems(kwargs):
             self.setlistdefault(key).append(value)
 
     def dict(self):

django/utils/six.py

@@ -355,4 +355,13 @@ def with_metaclass(meta, base=object):
 
 ### Additional customizations for Django ###
 
+if PY3:
+    _iterlists = "lists"
+else:
+    _iterlists = "iterlists"
+
+def iterlists(d):
+    """Return an iterator over the values of a MultiValueDict."""
+    return getattr(d, _iterlists)()
+
 add_move(MovedModule("_dummy_thread", "dummy_thread"))

docs/topics/python3.txt

@@ -120,3 +120,18 @@ If you need different code in Python 2 and Python 3, check :data:`six.PY3`::
 
 This is a last resort solution when :mod:`six` doesn't provide an appropriate
 function.
+
+.. module:: django.utils.six
+
+Customizations of six
+=====================
+
+The version of six bundled with Django includes a few additional tools:
+
+.. function:: iterlists(MultiValueDict)
+
+    Returns an iterator over the lists of values of a
+    :class:`~django.utils.datastructures.MultiValueDict`. This replaces
+    :meth:`~django.utils.datastructures.MultiValueDict.iterlists()` on Python
+    2 and :meth:`~django.utils.datastructures.MultiValueDict.lists()` on
+    Python 3.

tests/regressiontests/utils/datastructures.py

@@ -9,6 +9,7 @@ import warnings
 from django.test import SimpleTestCase
 from django.utils.datastructures import (DictWrapper, ImmutableList,
     MultiValueDict, MultiValueDictKeyError, MergeDict, SortedDict)
+from django.utils import six
 
 
 class SortedDictTests(SimpleTestCase):
@@ -25,19 +26,19 @@ class SortedDictTests(SimpleTestCase):
         self.d2[7] = 'seven'
 
     def test_basic_methods(self):
-        self.assertEqual(self.d1.keys(), [7, 1, 9])
+        self.assertEqual(list(six.iterkeys(self.d1)), [7, 1, 9])
-        self.assertEqual(self.d1.values(), ['seven', 'one', 'nine'])
+        self.assertEqual(list(six.itervalues(self.d1)), ['seven', 'one', 'nine'])
-        self.assertEqual(self.d1.items(), [(7, 'seven'), (1, 'one'), (9, 'nine')])
+        self.assertEqual(list(six.iteritems(self.d1)), [(7, 'seven'), (1, 'one'), (9, 'nine')])
 
     def test_overwrite_ordering(self):
-        """ Overwriting an item keeps it's place. """
+        """ Overwriting an item keeps its place. """
         self.d1[1] = 'ONE'
-        self.assertEqual(self.d1.values(), ['seven', 'ONE', 'nine'])
+        self.assertEqual(list(six.itervalues(self.d1)), ['seven', 'ONE', 'nine'])
 
     def test_append_items(self):
         """ New items go to the end. """
         self.d1[0] = 'nil'
-        self.assertEqual(self.d1.keys(), [7, 1, 9, 0])
+        self.assertEqual(list(six.iterkeys(self.d1)), [7, 1, 9, 0])
 
     def test_delete_and_insert(self):
         """
@@ -45,14 +46,18 @@ class SortedDictTests(SimpleTestCase):
         at the end.
         """
         del self.d2[7]
-        self.assertEqual(self.d2.keys(), [1, 9, 0])
+        self.assertEqual(list(six.iterkeys(self.d2)), [1, 9, 0])
         self.d2[7] = 'lucky number 7'
-        self.assertEqual(self.d2.keys(), [1, 9, 0, 7])
+        self.assertEqual(list(six.iterkeys(self.d2)), [1, 9, 0, 7])
 
+    if not six.PY3:
         def test_change_keys(self):
             """
             Changing the keys won't do anything, it's only a copy of the
             keys dict.
+
+            This test doesn't make sense under Python 3 because keys is
+            an iterator.
             """
             k = self.d2.keys()
             k.remove(9)
@@ -68,18 +73,18 @@ class SortedDictTests(SimpleTestCase):
         tuples = ((2, 'two'), (1, 'one'), (2, 'second-two'))
         d = SortedDict(tuples)
 
-        self.assertEqual(d.keys(), [2, 1])
+        self.assertEqual(list(six.iterkeys(d)), [2, 1])
 
         real_dict = dict(tuples)
-        self.assertEqual(sorted(real_dict.values()), ['one', 'second-two'])
+        self.assertEqual(sorted(six.itervalues(real_dict)), ['one', 'second-two'])
 
         # Here the order of SortedDict values *is* what we are testing
-        self.assertEqual(d.values(), ['second-two', 'one'])
+        self.assertEqual(list(six.itervalues(d)), ['second-two', 'one'])
 
     def test_overwrite(self):
         self.d1[1] = 'not one'
         self.assertEqual(self.d1[1], 'not one')
-        self.assertEqual(self.d1.keys(), self.d1.copy().keys())
+        self.assertEqual(list(six.iterkeys(self.d1)), list(six.iterkeys(self.d1.copy())))
 
     def test_append(self):
         self.d1[13] = 'thirteen'
@@ -115,8 +120,8 @@ class SortedDictTests(SimpleTestCase):
     def test_copy(self):
         orig = SortedDict(((1, "one"), (0, "zero"), (2, "two")))
         copied = copy.copy(orig)
-        self.assertEqual(orig.keys(), [1, 0, 2])
+        self.assertEqual(list(six.iterkeys(orig)), [1, 0, 2])
-        self.assertEqual(copied.keys(), [1, 0, 2])
+        self.assertEqual(list(six.iterkeys(copied)), [1, 0, 2])
 
     def test_clear(self):
         self.d1.clear()
@@ -178,12 +183,12 @@ class MergeDictTests(SimpleTestCase):
         self.assertEqual(mm.getlist('key4'), ['value5', 'value6'])
         self.assertEqual(mm.getlist('undefined'), [])
 
-        self.assertEqual(sorted(mm.keys()), ['key1', 'key2', 'key4'])
+        self.assertEqual(sorted(six.iterkeys(mm)), ['key1', 'key2', 'key4'])
-        self.assertEqual(len(mm.values()), 3)
+        self.assertEqual(len(list(six.itervalues(mm))), 3)
 
-        self.assertTrue('value1' in mm.values())
+        self.assertTrue('value1' in six.itervalues(mm))
 
-        self.assertEqual(sorted(mm.items(), key=lambda k: k[0]),
+        self.assertEqual(sorted(six.iteritems(mm), key=lambda k: k[0]),
                           [('key1', 'value1'), ('key2', 'value3'),
                            ('key4', 'value6')])
 
@@ -201,10 +206,10 @@ class MultiValueDictTests(SimpleTestCase):
         self.assertEqual(d['name'], 'Simon')
         self.assertEqual(d.get('name'), 'Simon')
         self.assertEqual(d.getlist('name'), ['Adrian', 'Simon'])
-        self.assertEqual(list(d.iteritems()),
+        self.assertEqual(list(six.iteritems(d)),
                           [('position', 'Developer'), ('name', 'Simon')])
 
-        self.assertEqual(list(d.iterlists()),
+        self.assertEqual(list(six.iterlists(d)),
                           [('position', ['Developer']),
                            ('name', ['Adrian', 'Simon'])])
 
@@ -224,8 +229,7 @@ class MultiValueDictTests(SimpleTestCase):
 
         d.setlist('lastname', ['Holovaty', 'Willison'])
         self.assertEqual(d.getlist('lastname'), ['Holovaty', 'Willison'])
-        self.assertEqual(d.values(), ['Developer', 'Simon', 'Willison'])
+        self.assertEqual(list(six.itervalues(d)),
-        self.assertEqual(list(d.itervalues()),
                           ['Developer', 'Simon', 'Willison'])
 
     def test_appendlist(self):
@@ -260,8 +264,8 @@ class MultiValueDictTests(SimpleTestCase):
             'pm': ['Rory'],
         })
         d = mvd.dict()
-        self.assertEqual(d.keys(), mvd.keys())
+        self.assertEqual(list(six.iterkeys(d)), list(six.iterkeys(mvd)))
-        for key in mvd.keys():
+        for key in six.iterkeys(mvd):
             self.assertEqual(d[key], mvd[key])
 
         self.assertEqual({}, MultiValueDict().dict())