Fixed pyinotify performance regression in 15f82c7011

Refs #9722. Thanks Tim Graham for the review.
This commit is contained in:
Claude Paroz 2014-07-05 14:43:12 +02:00
parent b144bfb5ce
commit 6d302f6396
3 changed files with 54 additions and 10 deletions

View File

@ -12,6 +12,10 @@ class StaticFilesHandler(WSGIHandler):
WSGI middleware that intercepts calls to the static files directory, as WSGI middleware that intercepts calls to the static files directory, as
defined by the STATIC_URL setting, and serves those files. defined by the STATIC_URL setting, and serves those files.
""" """
# May be used to differentiate between handler types (e.g. in a
# request_finished signal)
handles_files = True
def __init__(self, application): def __init__(self, application):
self.application = application self.application = application
self.base_url = urlparse(self.get_base_url()) self.base_url = urlparse(self.get_base_url())

View File

@ -77,20 +77,31 @@ _mtimes = {}
_win = (sys.platform == "win32") _win = (sys.platform == "win32")
_error_files = [] _error_files = []
_cached_modules = set()
_cached_filenames = []
def gen_filenames(only_new=False):
def gen_filenames():
""" """
Yields a generator over filenames referenced in sys.modules and translation Returns a list of filenames referenced in sys.modules and translation
files. files.
""" """
# N.B. ``list(...)`` is needed, because this runs in parallel with # N.B. ``list(...)`` is needed, because this runs in parallel with
# application code which might be mutating ``sys.modules``, and this will # application code which might be mutating ``sys.modules``, and this will
# fail with RuntimeError: cannot mutate dictionary while iterating # fail with RuntimeError: cannot mutate dictionary while iterating
filenames = [filename.__file__ for filename in list(sys.modules.values()) global _cached_modules, _cached_filenames
if hasattr(filename, '__file__')] module_values = set(sys.modules.values())
if _cached_modules == module_values:
# No changes in module list, short-circuit the function
if only_new:
return []
else:
return _cached_filenames
if settings.USE_I18N: new_modules = module_values - _cached_modules
new_filenames = [filename.__file__ for filename in new_modules
if hasattr(filename, '__file__')]
if not _cached_filenames and settings.USE_I18N:
# Add the names of the .mo files that can be generated # Add the names of the .mo files that can be generated
# by compilemessages management command to the list of files watched. # by compilemessages management command to the list of files watched.
basedirs = [os.path.join(os.path.dirname(os.path.dirname(__file__)), basedirs = [os.path.join(os.path.dirname(os.path.dirname(__file__)),
@ -105,9 +116,14 @@ def gen_filenames():
for dirpath, dirnames, locale_filenames in os.walk(basedir): for dirpath, dirnames, locale_filenames in os.walk(basedir):
for filename in locale_filenames: for filename in locale_filenames:
if filename.endswith('.mo'): if filename.endswith('.mo'):
filenames.append(os.path.join(dirpath, filename)) new_filenames.append(os.path.join(dirpath, filename))
for filename in filenames + _error_files: if only_new:
filelist = new_filenames
else:
filelist = _cached_filenames + new_filenames + _error_files
filenames = []
for filename in filelist:
if not filename: if not filename:
continue continue
if filename.endswith(".pyc") or filename.endswith(".pyo"): if filename.endswith(".pyc") or filename.endswith(".pyo"):
@ -115,7 +131,10 @@ def gen_filenames():
if filename.endswith("$py.class"): if filename.endswith("$py.class"):
filename = filename[:-9] + ".py" filename = filename[:-9] + ".py"
if os.path.exists(filename): if os.path.exists(filename):
yield filename filenames.append(filename)
_cached_modules = _cached_modules.union(new_modules)
_cached_filenames += new_filenames
return filenames
def reset_translations(): def reset_translations():
@ -145,6 +164,10 @@ def inotify_code_changed():
notifier = pyinotify.Notifier(wm, EventHandler()) notifier = pyinotify.Notifier(wm, EventHandler())
def update_watch(sender=None, **kwargs): def update_watch(sender=None, **kwargs):
if sender and getattr(sender, 'handles_files', False):
# No need to update watches when request serves files.
# (sender is supposed to be a django.core.handlers.BaseHandler subclass)
return
mask = ( mask = (
pyinotify.IN_MODIFY | pyinotify.IN_MODIFY |
pyinotify.IN_DELETE | pyinotify.IN_DELETE |
@ -153,7 +176,7 @@ def inotify_code_changed():
pyinotify.IN_MOVED_TO | pyinotify.IN_MOVED_TO |
pyinotify.IN_CREATE pyinotify.IN_CREATE
) )
for path in gen_filenames(): for path in gen_filenames(only_new=True):
wm.add_watch(path, mask) wm.add_watch(path, mask)
# New modules may get imported when a request is processed. # New modules may get imported when a request is processed.

View File

@ -9,6 +9,12 @@ LOCALE_PATH = os.path.join(os.path.dirname(__file__), 'locale')
class TestFilenameGenerator(TestCase): class TestFilenameGenerator(TestCase):
def setUp(self):
# Empty cached variables
from django.utils import autoreload
autoreload._cached_modules = set()
autoreload._cached_filenames = []
def test_django_locales(self): def test_django_locales(self):
""" """
Test that gen_filenames() also yields the built-in django locale files. Test that gen_filenames() also yields the built-in django locale files.
@ -64,3 +70,14 @@ class TestFilenameGenerator(TestCase):
os.path.join(os.path.dirname(conf.__file__), 'locale', 'nl', os.path.join(os.path.dirname(conf.__file__), 'locale', 'nl',
'LC_MESSAGES', 'django.mo'), 'LC_MESSAGES', 'django.mo'),
filenames) filenames)
def test_only_new_files(self):
"""
When calling a second time gen_filenames with only_new = True, only
files from newly loaded modules should be given.
"""
filenames1 = list(gen_filenames())
from fractions import Fraction
filenames2 = list(gen_filenames(only_new=True))
self.assertEqual(len(filenames2), 1)
self.assertTrue(filenames2[0].endswith('fractions.py'))