From 3cc5f01d9bd52930cec3c08d6ec7e19d0c2a6489 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Wed, 18 Jan 2017 21:30:21 +0100 Subject: [PATCH] Refs #23919 -- Stopped using django.utils.lru_cache(). --- django/apps/registry.py | 6 +- django/conf/urls/i18n.py | 5 +- django/contrib/auth/hashers.py | 6 +- django/contrib/auth/password_validation.py | 4 +- django/contrib/staticfiles/finders.py | 4 +- django/core/management/__init__.py | 5 +- django/core/management/color.py | 5 +- django/core/management/commands/loaddata.py | 4 +- django/db/models/fields/related.py | 4 +- django/db/models/query_utils.py | 4 +- django/forms/renderers.py | 4 +- django/template/engine.py | 5 +- django/template/utils.py | 4 +- django/urls/resolvers.py | 5 +- django/urls/utils.py | 4 +- django/utils/lru_cache.py | 175 +------------------- django/utils/timezone.py | 4 +- django/utils/translation/trans_real.py | 8 +- django/utils/version.py | 5 +- django/views/debug.py | 5 +- setup.cfg | 2 +- 21 files changed, 52 insertions(+), 216 deletions(-) diff --git a/django/apps/registry.py b/django/apps/registry.py index 453e4d42f62..c67c5ee0baa 100644 --- a/django/apps/registry.py +++ b/django/apps/registry.py @@ -1,3 +1,4 @@ +import functools import sys import threading import warnings @@ -5,7 +6,6 @@ from collections import Counter, OrderedDict, defaultdict from functools import partial from django.core.exceptions import AppRegistryNotReady, ImproperlyConfigured -from django.utils import lru_cache from .config import AppConfig @@ -156,7 +156,7 @@ class Apps(object): raise LookupError(message) # This method is performance-critical at least for Django's test suite. - @lru_cache.lru_cache(maxsize=None) + @functools.lru_cache(maxsize=None) def get_models(self, include_auto_created=False, include_swapped=False): """ Returns a list of all installed models. @@ -268,7 +268,7 @@ class Apps(object): "Model '%s.%s' not registered." % (app_label, model_name)) return model - @lru_cache.lru_cache(maxsize=None) + @functools.lru_cache(maxsize=None) def get_swappable_settings_name(self, to_string): """ For a given model string (e.g. "auth.User"), return the name of the diff --git a/django/conf/urls/i18n.py b/django/conf/urls/i18n.py index 14f4c6971b0..23e4bbcd74b 100644 --- a/django/conf/urls/i18n.py +++ b/django/conf/urls/i18n.py @@ -1,7 +1,8 @@ +import functools + from django.conf import settings from django.conf.urls import url from django.urls import LocaleRegexURLResolver, get_resolver -from django.utils import lru_cache from django.views.i18n import set_language @@ -18,7 +19,7 @@ def i18n_patterns(*urls, **kwargs): return [LocaleRegexURLResolver(list(urls), prefix_default_language=prefix_default_language)] -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def is_language_prefix_patterns_used(urlconf): """ Return a tuple of two booleans: ( diff --git a/django/contrib/auth/hashers.py b/django/contrib/auth/hashers.py index 4b8b12782e8..871519257af 100644 --- a/django/contrib/auth/hashers.py +++ b/django/contrib/auth/hashers.py @@ -1,5 +1,6 @@ import base64 import binascii +import functools import hashlib import importlib import warnings @@ -9,7 +10,6 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.core.signals import setting_changed from django.dispatch import receiver -from django.utils import lru_cache from django.utils.crypto import ( constant_time_compare, get_random_string, pbkdf2, ) @@ -82,7 +82,7 @@ def make_password(password, salt=None, hasher='default'): return hasher.encode(password, salt) -@lru_cache.lru_cache() +@functools.lru_cache() def get_hashers(): hashers = [] for hasher_path in settings.PASSWORD_HASHERS: @@ -95,7 +95,7 @@ def get_hashers(): return hashers -@lru_cache.lru_cache() +@functools.lru_cache() def get_hashers_by_algorithm(): return {hasher.algorithm: hasher for hasher in get_hashers()} diff --git a/django/contrib/auth/password_validation.py b/django/contrib/auth/password_validation.py index 1cf32e02198..dee1ebf6747 100644 --- a/django/contrib/auth/password_validation.py +++ b/django/contrib/auth/password_validation.py @@ -1,3 +1,4 @@ +import functools import gzip import os import re @@ -7,7 +8,6 @@ from django.conf import settings from django.core.exceptions import ( FieldDoesNotExist, ImproperlyConfigured, ValidationError, ) -from django.utils import lru_cache from django.utils._os import upath from django.utils.encoding import force_text from django.utils.functional import lazy @@ -16,7 +16,7 @@ from django.utils.module_loading import import_string from django.utils.translation import ugettext as _, ungettext -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def get_default_password_validators(): return get_password_validators(settings.AUTH_PASSWORD_VALIDATORS) diff --git a/django/contrib/staticfiles/finders.py b/django/contrib/staticfiles/finders.py index c3da95ddd3b..96dd705050a 100644 --- a/django/contrib/staticfiles/finders.py +++ b/django/contrib/staticfiles/finders.py @@ -1,3 +1,4 @@ +import functools import os from collections import OrderedDict @@ -8,7 +9,6 @@ from django.core.exceptions import ImproperlyConfigured from django.core.files.storage import ( FileSystemStorage, Storage, default_storage, ) -from django.utils import lru_cache from django.utils._os import safe_join from django.utils.functional import LazyObject, empty from django.utils.module_loading import import_string @@ -264,7 +264,7 @@ def get_finders(): yield get_finder(finder_path) -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def get_finder(import_path): """ Imports the staticfiles finder class described by import_path, where diff --git a/django/core/management/__init__.py b/django/core/management/__init__.py index 00ee58a34be..0200f77e7d0 100644 --- a/django/core/management/__init__.py +++ b/django/core/management/__init__.py @@ -1,3 +1,4 @@ +import functools import os import pkgutil import sys @@ -12,7 +13,7 @@ from django.core.management.base import ( BaseCommand, CommandError, CommandParser, handle_default_options, ) from django.core.management.color import color_style -from django.utils import autoreload, lru_cache +from django.utils import autoreload from django.utils._os import npath, upath from django.utils.encoding import force_text @@ -39,7 +40,7 @@ def load_command_class(app_name, name): return module.Command() -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def get_commands(): """ Returns a dictionary mapping command names to their callback applications. diff --git a/django/core/management/color.py b/django/core/management/color.py index 7a323708608..76985420cba 100644 --- a/django/core/management/color.py +++ b/django/core/management/color.py @@ -2,10 +2,11 @@ Sets up the terminal color scheme. """ +import functools import os import sys -from django.utils import lru_cache, termcolors +from django.utils import termcolors def supports_color(): @@ -57,7 +58,7 @@ def make_style(config_string=''): return style -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def no_style(): """ Returns a Style object with no color scheme. diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 9f71400862a..9edd642b433 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -1,3 +1,4 @@ +import functools import glob import gzip import os @@ -16,7 +17,6 @@ from django.db import ( DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connections, router, transaction, ) -from django.utils import lru_cache from django.utils._os import upath from django.utils.encoding import force_text from django.utils.functional import cached_property @@ -202,7 +202,7 @@ class Command(BaseCommand): RuntimeWarning ) - @lru_cache.lru_cache(maxsize=None) + @functools.lru_cache(maxsize=None) def find_fixtures(self, fixture_label): """ Finds fixture files for a given label. diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 65d0da41e17..b5dd0a2d14b 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -1,3 +1,4 @@ +import functools import inspect from functools import partial @@ -13,7 +14,6 @@ from django.db.models.query_utils import PathInfo from django.db.models.utils import make_model_tuple from django.utils.encoding import force_text from django.utils.functional import cached_property, curry -from django.utils.lru_cache import lru_cache from django.utils.translation import ugettext_lazy as _ from . import Field @@ -710,7 +710,7 @@ class ForeignObject(RelatedField): return pathinfos @classmethod - @lru_cache(maxsize=None) + @functools.lru_cache(maxsize=None) def get_lookups(cls): bases = inspect.getmro(cls) bases = bases[:bases.index(ForeignObject) + 1] diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py index 112c9586d07..03d133e7240 100644 --- a/django/db/models/query_utils.py +++ b/django/db/models/query_utils.py @@ -5,12 +5,12 @@ Factored out from django.db.models.query to avoid making the main module very large and/or so that they can be used by other modules without getting into circular import difficulties. """ +import functools import inspect from collections import namedtuple from django.db.models.constants import LOOKUP_SEP from django.utils import tree -from django.utils.lru_cache import lru_cache # PathInfo is used when converting lookups (fk__somecol). The contents # describe the relation in Model terms (model Options and Fields for both @@ -137,7 +137,7 @@ class RegisterLookupMixin(object): return cls.get_lookups().get(lookup_name, None) @classmethod - @lru_cache(maxsize=None) + @functools.lru_cache(maxsize=None) def get_lookups(cls): class_lookups = [parent.__dict__.get('class_lookups', {}) for parent in inspect.getmro(cls)] return cls.merge_dicts(class_lookups) diff --git a/django/forms/renderers.py b/django/forms/renderers.py index d0b3c3e2db0..2c31d7adc5a 100644 --- a/django/forms/renderers.py +++ b/django/forms/renderers.py @@ -1,9 +1,9 @@ +import functools import os from django.conf import settings from django.template.backends.django import DjangoTemplates from django.template.loader import get_template -from django.utils import lru_cache from django.utils._os import upath from django.utils.functional import cached_property from django.utils.module_loading import import_string @@ -17,7 +17,7 @@ except ImportError: ROOT = upath(os.path.dirname(__file__)) -@lru_cache.lru_cache() +@functools.lru_cache() def get_default_renderer(): renderer_class = import_string(settings.FORM_RENDERER) return renderer_class() diff --git a/django/template/engine.py b/django/template/engine.py index 6d73569e022..d334926c45e 100644 --- a/django/template/engine.py +++ b/django/template/engine.py @@ -1,5 +1,6 @@ +import functools + from django.core.exceptions import ImproperlyConfigured -from django.utils import lru_cache from django.utils.functional import cached_property from django.utils.module_loading import import_string @@ -52,7 +53,7 @@ class Engine(object): self.template_builtins = self.get_template_builtins(self.builtins) @staticmethod - @lru_cache.lru_cache() + @functools.lru_cache() def get_default(): """ When only one DjangoTemplates backend is configured, returns it. diff --git a/django/template/utils.py b/django/template/utils.py index 3c7b3c98ace..6ff2499dcbf 100644 --- a/django/template/utils.py +++ b/django/template/utils.py @@ -1,10 +1,10 @@ +import functools import os from collections import Counter, OrderedDict from django.apps import apps from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.utils import lru_cache from django.utils._os import upath from django.utils.functional import cached_property from django.utils.module_loading import import_string @@ -89,7 +89,7 @@ class EngineHandler(object): return [self[alias] for alias in self] -@lru_cache.lru_cache() +@functools.lru_cache() def get_app_template_dirs(dirname): """ Return an iterable of paths of directories to load app templates from. diff --git a/django/urls/resolvers.py b/django/urls/resolvers.py index cf2fe0ecec6..7e344896af0 100644 --- a/django/urls/resolvers.py +++ b/django/urls/resolvers.py @@ -14,7 +14,6 @@ from django.conf import settings from django.core.checks import Warning from django.core.checks.urls import check_resolver from django.core.exceptions import ImproperlyConfigured -from django.utils import lru_cache from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_str, force_text from django.utils.functional import cached_property @@ -60,7 +59,7 @@ class ResolverMatch(object): ) -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def get_resolver(urlconf=None): if urlconf is None: from django.conf import settings @@ -68,7 +67,7 @@ def get_resolver(urlconf=None): return RegexURLResolver(r'^/', urlconf) -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def get_ns_resolver(ns_pattern, resolver): # Build a namespaced resolver for the given parent URLconf pattern. # This makes it possible to have captured parameters in the parent diff --git a/django/urls/utils.py b/django/urls/utils.py index 3cf5439e77f..e59ab9fdbd0 100644 --- a/django/urls/utils.py +++ b/django/urls/utils.py @@ -1,11 +1,11 @@ +import functools from importlib import import_module from django.core.exceptions import ViewDoesNotExist -from django.utils import lru_cache from django.utils.module_loading import module_has_submodule -@lru_cache.lru_cache(maxsize=None) +@functools.lru_cache(maxsize=None) def get_callable(lookup_view): """ Return a callable corresponding to lookup_view. diff --git a/django/utils/lru_cache.py b/django/utils/lru_cache.py index 543296e6481..f5187ded565 100644 --- a/django/utils/lru_cache.py +++ b/django/utils/lru_cache.py @@ -1,172 +1,5 @@ -try: - from functools import lru_cache +from functools import lru_cache # noqa -except ImportError: - # backport of Python's 3.3 lru_cache, written by Raymond Hettinger and - # licensed under MIT license, from: - # - # Should be removed when Django only supports Python 3.2 and above. - - from collections import namedtuple - from functools import update_wrapper - from threading import RLock - - _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) - - class _HashedSeq(list): - __slots__ = 'hashvalue' - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - - def _make_key(args, kwds, typed, - kwd_mark = (object(),), - fasttypes = {int, str, frozenset, type(None)}, - sorted=sorted, tuple=tuple, type=type, len=len): - 'Make a cache key from optionally typed positional and keyword arguments' - key = args - if kwds: - sorted_items = sorted(kwds.items()) - key += kwd_mark - for item in sorted_items: - key += item - if typed: - key += tuple(type(v) for v in args) - if kwds: - key += tuple(type(v) for k, v in sorted_items) - elif len(key) == 1 and type(key[0]) in fasttypes: - return key[0] - return _HashedSeq(key) - - def lru_cache(maxsize=100, typed=False): - """Least-recently-used cache decorator. - - If *maxsize* is set to None, the LRU features are disabled and the cache - can grow without bound. - - If *typed* is True, arguments of different types will be cached separately. - For example, f(3.0) and f(3) will be treated as distinct calls with - distinct results. - - Arguments to the cached function must be hashable. - - View the cache statistics named tuple (hits, misses, maxsize, currsize) with - f.cache_info(). Clear the cache and statistics with f.cache_clear(). - Access the underlying function with f.__wrapped__. - - See: https://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used - """ - - # Users should only access the lru_cache through its public API: - # cache_info, cache_clear, and f.__wrapped__ - # The internals of the lru_cache are encapsulated for thread safety and - # to allow the implementation to change (including a possible C version). - - def decorating_function(user_function): - - cache = dict() - stats = [0, 0] # make statistics updateable non-locally - HITS, MISSES = 0, 1 # names for the stats fields - make_key = _make_key - cache_get = cache.get # bound method to lookup key or return None - _len = len # localize the global len() function - lock = RLock() # because linkedlist updates aren't threadsafe - root = [] # root of the circular doubly linked list - root[:] = [root, root, None, None] # initialize by pointing to self - nonlocal_root = [root] # make updateable non-locally - PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields - - if maxsize == 0: - - def wrapper(*args, **kwds): - # no caching, just do a statistics update after a successful call - result = user_function(*args, **kwds) - stats[MISSES] += 1 - return result - - elif maxsize is None: - - def wrapper(*args, **kwds): - # simple caching without ordering or size limit - key = make_key(args, kwds, typed) - result = cache_get(key, root) # root used here as a unique not-found sentinel - if result is not root: - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - cache[key] = result - stats[MISSES] += 1 - return result - - else: - - def wrapper(*args, **kwds): - # size limited caching that tracks accesses by recency - key = make_key(args, kwds, typed) if kwds or typed else args - with lock: - link = cache_get(key) - if link is not None: - # record recent use of the key by moving it to the front of the list - root, = nonlocal_root - link_prev, link_next, key, result = link - link_prev[NEXT] = link_next - link_next[PREV] = link_prev - last = root[PREV] - last[NEXT] = root[PREV] = link - link[PREV] = last - link[NEXT] = root - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - with lock: - root, = nonlocal_root - if key in cache: - # getting here means that this same key was added to the - # cache while the lock was released. since the link - # update is already done, we need only return the - # computed result and update the count of misses. - pass - elif _len(cache) >= maxsize: - # use the old root to store the new key and result - oldroot = root - oldroot[KEY] = key - oldroot[RESULT] = result - # empty the oldest link and make it the new root - root = nonlocal_root[0] = oldroot[NEXT] - oldkey = root[KEY] - oldvalue = root[RESULT] - root[KEY] = root[RESULT] = None - # now update the cache dictionary for the new links - del cache[oldkey] - cache[key] = oldroot - else: - # put result in a new link at the front of the list - last = root[PREV] - link = [last, root, key, result] - last[NEXT] = root[PREV] = cache[key] = link - stats[MISSES] += 1 - return result - - def cache_info(): - """Report cache statistics""" - with lock: - return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) - - def cache_clear(): - """Clear the cache and cache statistics""" - with lock: - cache.clear() - root = nonlocal_root[0] - root[:] = [root, root, None, None] - stats[:] = [0, 0] - - wrapper.__wrapped__ = user_function - wrapper.cache_info = cache_info - wrapper.cache_clear = cache_clear - return update_wrapper(wrapper, user_function) - - return decorating_function +# Deprecate or remove this module when no supported version of Django still +# supports Python 2. Until then, keep it to allow pluggable apps to support +# Python 2 and Python 3 without raising a deprecation warning. diff --git a/django/utils/timezone.py b/django/utils/timezone.py index 992e80086ab..66bfa00030a 100644 --- a/django/utils/timezone.py +++ b/django/utils/timezone.py @@ -2,13 +2,13 @@ Timezone-related classes and functions. """ +import functools from datetime import datetime, timedelta, tzinfo from threading import local import pytz from django.conf import settings -from django.utils import lru_cache from django.utils.decorators import ContextDecorator __all__ = [ @@ -69,7 +69,7 @@ def get_fixed_timezone(offset): # In order to avoid accessing settings at compile time, # wrap the logic in a function and cache the result. -@lru_cache.lru_cache() +@functools.lru_cache() def get_default_timezone(): """ Returns the default time zone as a tzinfo instance. diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index aca57ded0eb..99acda9aabc 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -1,4 +1,5 @@ """Translation helper functions.""" +import functools import gettext as gettext_module import os import re @@ -13,7 +14,6 @@ from django.conf.locale import LANG_INFO from django.core.exceptions import AppRegistryNotReady from django.core.signals import setting_changed from django.dispatch import receiver -from django.utils import lru_cache from django.utils._os import upath from django.utils.encoding import force_text from django.utils.safestring import SafeData, mark_safe @@ -403,7 +403,7 @@ def all_locale_paths(): return [globalpath] + list(settings.LOCALE_PATHS) -@lru_cache.lru_cache(maxsize=1000) +@functools.lru_cache(maxsize=1000) def check_for_language(lang_code): """ Checks whether there is a global language file for the given language @@ -423,7 +423,7 @@ def check_for_language(lang_code): return False -@lru_cache.lru_cache() +@functools.lru_cache() def get_languages(): """ Cache of settings.LANGUAGES in an OrderedDict for easy lookups by key. @@ -431,7 +431,7 @@ def get_languages(): return OrderedDict(settings.LANGUAGES) -@lru_cache.lru_cache(maxsize=1000) +@functools.lru_cache(maxsize=1000) def get_supported_language_variant(lang_code, strict=False): """ Returns the language-code that's listed in supported languages, possibly diff --git a/django/utils/version.py b/django/utils/version.py index dcf0c218664..e9666f906e1 100644 --- a/django/utils/version.py +++ b/django/utils/version.py @@ -1,9 +1,8 @@ import datetime +import functools import os import subprocess -from django.utils.lru_cache import lru_cache - def get_version(version=None): "Returns a PEP 440-compliant version number from VERSION." @@ -57,7 +56,7 @@ def get_docs_version(version=None): return '%d.%d' % version[:2] -@lru_cache() +@functools.lru_cache() def get_git_changeset(): """Returns a numeric identifier of the latest git changeset. diff --git a/django/views/debug.py b/django/views/debug.py index de8c4319bfc..b457969427f 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -1,3 +1,4 @@ +import functools import re import sys import types @@ -7,7 +8,7 @@ from django.http import HttpResponse, HttpResponseNotFound from django.template import Context, Engine, TemplateDoesNotExist from django.template.defaultfilters import force_escape, pprint from django.urls import Resolver404, resolve -from django.utils import lru_cache, timezone +from django.utils import timezone from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_bytes, force_text from django.utils.module_loading import import_string @@ -83,7 +84,7 @@ def technical_500_response(request, exc_type, exc_value, tb, status_code=500): return HttpResponse(html, status=status_code, content_type='text/html') -@lru_cache.lru_cache() +@functools.lru_cache() def get_default_exception_reporter_filter(): # Instantiate the default filter for the first time and cache it. return import_string(settings.DEFAULT_EXCEPTION_REPORTER_FILTER)() diff --git a/setup.cfg b/setup.cfg index c1ff45c56a9..726bddb3af4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ doc_files = docs extras AUTHORS INSTALL LICENSE README.rst install-script = scripts/rpm-install.sh [flake8] -exclude = build,.git,.tox,./django/utils/lru_cache.py,./django/utils/six.py,./django/conf/app_template/*,./tests/.env,./xmlrunner +exclude = build,.git,.tox,./django/utils/six.py,./django/conf/app_template/*,./tests/.env,./xmlrunner ignore = W601 max-line-length = 119