Fixed #27685 -- Added watchman support to the autoreloader.

Removed support for pyinotify (refs #9722).

django/apps/registry.py

@@ -42,6 +42,8 @@ class Apps:
 
         # Whether the registry is populated.
         self.apps_ready = self.models_ready = self.ready = False
+        # For the autoreloader.
+        self.ready_event = threading.Event()
 
         # Lock for thread-safe population.
         self._lock = threading.RLock()
@@ -120,6 +122,7 @@ class Apps:
                 app_config.ready()
 
             self.ready = True
+            self.ready_event.set()
 
     def check_apps_ready(self):
         """Raise an exception if all apps haven't been imported yet."""

django/core/management/commands/runserver.py

@@ -99,7 +99,7 @@ class Command(BaseCommand):
         use_reloader = options['use_reloader']
 
         if use_reloader:
-            autoreload.main(self.inner_run, None, options)
+            autoreload.run_with_reloader(self.inner_run, **options)
         else:
             self.inner_run(None, **options)
 

django/db/migrations/state.py

@@ -264,9 +264,11 @@ class StateApps(Apps):
         app_configs = [AppConfigStub(label) for label in sorted([*real_apps, *app_labels])]
         super().__init__(app_configs)
 
-        # The lock gets in the way of copying as implemented in clone(), which
+        # These locks get in the way of copying as implemented in clone(),
-        # is called whenever Django duplicates a StateApps before updating it.
+        # which is called whenever Django duplicates a StateApps before
+        # updating it.
         self._lock = None
+        self.ready_event = None
 
         self.render_multiple([*models.values(), *self.real_models])
 

django/utils/autoreload.py

@@ -1,224 +1,52 @@
-# Autoreloading launcher.
+import functools
-# Borrowed from Peter Hunt and the CherryPy project (https://cherrypy.org/).
+import itertools
-# Some taken from Ian Bicking's Paste (http://pythonpaste.org/).
+import logging
-#
-# Portions copyright (c) 2004, CherryPy Team (team@cherrypy.org)
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimer in the documentation
-#       and/or other materials provided with the distribution.
-#     * Neither the name of the CherryPy Team nor the names of its contributors
-#       may be used to endorse or promote products derived from this software
-#       without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 import os
+import pathlib
 import signal
 import subprocess
 import sys
+import threading
 import time
 import traceback
-
+from collections import defaultdict
-import _thread
+from pathlib import Path
+from types import ModuleType
+from zipimport import zipimporter
 
 from django.apps import apps
-from django.conf import settings
 from django.core.signals import request_finished
+from django.dispatch import Signal
+from django.utils.functional import cached_property
+from django.utils.version import get_version_tuple
 
-# This import does nothing, but it's necessary to avoid some race conditions
+autoreload_started = Signal()
-# in the threading module. See https://code.djangoproject.com/ticket/2330 .
+file_changed = Signal(providing_args=['file_path', 'kind'])
-try:
+
-    import threading  # NOQA
+DJANGO_AUTORELOAD_ENV = 'RUN_MAIN'
-except ImportError:
+
-    pass
+logger = logging.getLogger('django.utils.autoreload')
+
+# If an error is raised while importing a file, it's not placed in sys.modules.
+# This means that any future modifications aren't caught. Keep a list of these
+# file paths to allow watching them in the future.
+_error_files = []
+_exception = None
 
 try:
     import termios
 except ImportError:
     termios = None
 
-USE_INOTIFY = False
+
 try:
-    # Test whether inotify is enabled and likely to work
+    import pywatchman
-    import pyinotify
-
-    fd = pyinotify.INotifyWrapper.create().inotify_init()
-    if fd >= 0:
-        USE_INOTIFY = True
-        os.close(fd)
 except ImportError:
-    pass
+    pywatchman = None
-
-RUN_RELOADER = True
-
-FILE_MODIFIED = 1
-I18N_MODIFIED = 2
-
-_mtimes = {}
-_win = (sys.platform == "win32")
-
-_exception = None
-_error_files = []
-_cached_modules = set()
-_cached_filenames = []
-
-
-def gen_filenames(only_new=False):
-    """
-    Return a list of filenames referenced in sys.modules and translation files.
-    """
-    # N.B. ``list(...)`` is needed, because this runs in parallel with
-    # application code which might be mutating ``sys.modules``, and this will
-    # fail with RuntimeError: cannot mutate dictionary while iterating
-    global _cached_modules, _cached_filenames
-    module_values = set(sys.modules.values())
-    _cached_filenames = clean_files(_cached_filenames)
-    if _cached_modules == module_values:
-        # No changes in module list, short-circuit the function
-        if only_new:
-            return []
-        else:
-            return _cached_filenames + clean_files(_error_files)
-
-    new_modules = module_values - _cached_modules
-    new_filenames = clean_files(
-        [filename.__file__ for filename in new_modules
-         if hasattr(filename, '__file__')])
-
-    if not _cached_filenames and settings.USE_I18N:
-        # Add the names of the .mo files that can be generated
-        # by compilemessages management command to the list of files watched.
-        basedirs = [os.path.join(os.path.dirname(os.path.dirname(__file__)),
-                                 'conf', 'locale'),
-                    'locale']
-        for app_config in reversed(list(apps.get_app_configs())):
-            basedirs.append(os.path.join(app_config.path, 'locale'))
-        basedirs.extend(settings.LOCALE_PATHS)
-        basedirs = [os.path.abspath(basedir) for basedir in basedirs
-                    if os.path.isdir(basedir)]
-        for basedir in basedirs:
-            for dirpath, dirnames, locale_filenames in os.walk(basedir):
-                for filename in locale_filenames:
-                    if filename.endswith('.mo'):
-                        new_filenames.append(os.path.join(dirpath, filename))
-
-    _cached_modules = _cached_modules.union(new_modules)
-    _cached_filenames += new_filenames
-    if only_new:
-        return new_filenames + clean_files(_error_files)
-    else:
-        return _cached_filenames + clean_files(_error_files)
-
-
-def clean_files(filelist):
-    filenames = []
-    for filename in filelist:
-        if not filename:
-            continue
-        if filename.endswith(".pyc") or filename.endswith(".pyo"):
-            filename = filename[:-1]
-        if filename.endswith("$py.class"):
-            filename = filename[:-9] + ".py"
-        if os.path.exists(filename):
-            filenames.append(filename)
-    return filenames
-
-
-def reset_translations():
-    import gettext
-    from django.utils.translation import trans_real
-    gettext._translations = {}
-    trans_real._translations = {}
-    trans_real._default = None
-    trans_real._active = threading.local()
-
-
-def inotify_code_changed():
-    """
-    Check for changed code using inotify. After being called
-    it blocks until a change event has been fired.
-    """
-    class EventHandler(pyinotify.ProcessEvent):
-        modified_code = None
-
-        def process_default(self, event):
-            if event.path.endswith('.mo'):
-                EventHandler.modified_code = I18N_MODIFIED
-            else:
-                EventHandler.modified_code = FILE_MODIFIED
-
-    wm = pyinotify.WatchManager()
-    notifier = pyinotify.Notifier(wm, EventHandler())
-
-    def update_watch(sender=None, **kwargs):
-        if sender and getattr(sender, 'handles_files', False):
-            # No need to update watches when request serves files.
-            # (sender is supposed to be a django.core.handlers.BaseHandler subclass)
-            return
-        mask = (
-            pyinotify.IN_MODIFY |
-            pyinotify.IN_DELETE |
-            pyinotify.IN_ATTRIB |
-            pyinotify.IN_MOVED_FROM |
-            pyinotify.IN_MOVED_TO |
-            pyinotify.IN_CREATE |
-            pyinotify.IN_DELETE_SELF |
-            pyinotify.IN_MOVE_SELF
-        )
-        for path in gen_filenames(only_new=True):
-            wm.add_watch(path, mask)
-
-    # New modules may get imported when a request is processed.
-    request_finished.connect(update_watch)
-
-    # Block until an event happens.
-    update_watch()
-    notifier.check_events(timeout=None)
-    notifier.read_events()
-    notifier.process_events()
-    notifier.stop()
-
-    # If we are here the code must have changed.
-    return EventHandler.modified_code
-
-
-def code_changed():
-    global _mtimes, _win
-    for filename in gen_filenames():
-        stat = os.stat(filename)
-        mtime = stat.st_mtime
-        if _win:
-            mtime -= stat.st_ctime
-        if filename not in _mtimes:
-            _mtimes[filename] = mtime
-            continue
-        if mtime != _mtimes[filename]:
-            _mtimes = {}
-            try:
-                del _error_files[_error_files.index(filename)]
-            except ValueError:
-                pass
-            return I18N_MODIFIED if filename.endswith('.mo') else FILE_MODIFIED
-    return False
 
 
 def check_errors(fn):
+    @functools.wraps(fn)
     def wrapper(*args, **kwargs):
         global _exception
         try:
@@ -245,7 +73,7 @@ def check_errors(fn):
 def raise_last_exception():
     global _exception
     if _exception is not None:
-        raise _exception[1]
+        raise _exception[0](_exception[1]).with_traceback(_exception[2])
 
 
 def ensure_echo_on():
@@ -264,60 +92,496 @@ def ensure_echo_on():
                     signal.signal(signal.SIGTTOU, old_handler)
 
 
-def reloader_thread():
+def iter_all_python_module_files():
-    ensure_echo_on()
+    # This is a hot path during reloading. Create a stable sorted list of
-    if USE_INOTIFY:
+    # modules based on the module name and pass it to iter_modules_and_files().
-        fn = inotify_code_changed
+    # This ensures cached results are returned in the usual case that modules
+    # aren't loaded on the fly.
+    modules_view = sorted(list(sys.modules.items()), key=lambda i: i[0])
+    modules = tuple(m[1] for m in modules_view)
+    return iter_modules_and_files(modules, frozenset(_error_files))
+
+
+@functools.lru_cache(maxsize=1)
+def iter_modules_and_files(modules, extra_files):
+    """Iterate through all modules needed to be watched."""
+    sys_file_paths = []
+    for module in modules:
+        # During debugging (with PyDev) the 'typing.io' and 'typing.re' objects
+        # are added to sys.modules, however they are types not modules and so
+        # cause issues here.
+        if not isinstance(module, ModuleType) or module.__spec__ is None:
+            continue
+        spec = module.__spec__
+        # Modules could be loaded from places without a concrete location. If
+        # this is the case, skip them.
+        if spec.has_location:
+            origin = spec.loader.archive if isinstance(spec.loader, zipimporter) else spec.origin
+            sys_file_paths.append(origin)
+
+    results = set()
+    for filename in itertools.chain(sys_file_paths, extra_files):
+        if not filename:
+            continue
+        path = pathlib.Path(filename)
+        if not path.exists():
+            # The module could have been removed, don't fail loudly if this
+            # is the case.
+            continue
+        results.add(path.resolve().absolute())
+    return frozenset(results)
+
+
+@functools.lru_cache(maxsize=1)
+def common_roots(paths):
+    """
+    Return a tuple of common roots that are shared between the given paths.
+    File system watchers operate on directories and aren't cheap to create.
+    Try to find the minimum set of directories to watch that encompass all of
+    the files that need to be watched.
+    """
+    # Inspired from Werkzeug:
+    # https://github.com/pallets/werkzeug/blob/7477be2853df70a022d9613e765581b9411c3c39/werkzeug/_reloader.py
+    # Create a sorted list of the path components, longest first.
+    path_parts = sorted([x.parts for x in paths], key=len, reverse=True)
+    tree = {}
+    for chunks in path_parts:
+        node = tree
+        # Add each part of the path to the tree.
+        for chunk in chunks:
+            node = node.setdefault(chunk, {})
+        # Clear the last leaf in the tree.
+        node.clear()
+
+    # Turn the tree into a list of Path instances.
+    def _walk(node, path):
+        for prefix, child in node.items():
+            yield from _walk(child, path + (prefix,))
+        if not node:
+            yield Path(*path)
+
+    return tuple(_walk(tree, ()))
+
+
+def sys_path_directories():
+    """
+    Yield absolute directories from sys.path, ignoring entries that don't
+    exist.
+    """
+    for path in sys.path:
+        path = Path(path)
+        if not path.exists():
+            continue
+        path = path.resolve().absolute()
+        # If the path is a file (like a zip file), watch the parent directory.
+        if path.is_file():
+            yield path.parent
+        else:
+            yield path
+
+
+def get_child_arguments():
+    """
+    Return the executable. This contains a workaround for Windows if the
+    executable is reported to not have the .exe extension which can cause bugs
+    on reloading.
+    """
+    import django.__main__
+
+    args = [sys.executable] + ['-W%s' % o for o in sys.warnoptions]
+    if sys.argv[0] == django.__main__.__file__:
+        # The server was started with `python -m django runserver`.
+        args += ['-m', 'django']
+        args += sys.argv[1:]
     else:
-        fn = code_changed
+        args += sys.argv
-    while RUN_RELOADER:
+    return args
-        change = fn()
+
-        if change == FILE_MODIFIED:
+
-            sys.exit(3)  # force reload
+def trigger_reload(filename):
-        elif change == I18N_MODIFIED:
+    logger.info('%s changed, reloading.', filename)
-            reset_translations()
+    sys.exit(3)
-        time.sleep(1)
 
 
 def restart_with_reloader():
-    import django.__main__
+    new_environ = {**os.environ, DJANGO_AUTORELOAD_ENV: 'true'}
+    args = get_child_arguments()
     while True:
-        args = [sys.executable] + ['-W%s' % o for o in sys.warnoptions]
+        exit_code = subprocess.call(args, env=new_environ, close_fds=False)
-        if sys.argv[0] == django.__main__.__file__:
-            # The server was started with `python -m django runserver`.
-            args += ['-m', 'django']
-            args += sys.argv[1:]
-        else:
-            args += sys.argv
-        new_environ = {**os.environ, 'RUN_MAIN': 'true'}
-        exit_code = subprocess.call(args, env=new_environ)
         if exit_code != 3:
             return exit_code
 
 
-def python_reloader(main_func, args, kwargs):
+class BaseReloader:
-    if os.environ.get("RUN_MAIN") == "true":
+    def __init__(self):
-        _thread.start_new_thread(main_func, args, kwargs)
+        self.extra_files = set()
+        self.directory_globs = defaultdict(set)
+        self._stop_condition = threading.Event()
+
+    def watch_dir(self, path, glob):
+        path = Path(path)
+        if not path.is_absolute():
+            raise ValueError('%s must be absolute.' % path)
+        logger.debug('Watching dir %s with glob %s.', path, glob)
+        self.directory_globs[path].add(glob)
+
+    def watch_file(self, path):
+        path = Path(path)
+        if not path.is_absolute():
+            raise ValueError('%s must be absolute.' % path)
+        logger.debug('Watching file %s.', path)
+        self.extra_files.add(path)
+
+    def watched_files(self, include_globs=True):
+        """
+        Yield all files that need to be watched, including module files and
+        files within globs.
+        """
+        yield from iter_all_python_module_files()
+        yield from self.extra_files
+        if include_globs:
+            for directory, patterns in self.directory_globs.items():
+                for pattern in patterns:
+                    yield from directory.glob(pattern)
+
+    def wait_for_apps_ready(self, app_reg, django_main_thread):
+        """
+        Wait until Django reports that the apps have been loaded. If the given
+        thread has terminated before the apps are ready, then a SyntaxError or
+        other non-recoverable error has been raised. In that case, stop waiting
+        for the apps_ready event and continue processing.
+
+        Return True if the thread is alive and the ready event has been
+        triggered, or False if the thread is terminated while waiting for the
+        event.
+        """
+        while django_main_thread.is_alive():
+            if app_reg.ready_event.wait(timeout=0.1):
+                return True
+        else:
+            logger.debug('Main Django thread has terminated before apps are ready.')
+            return False
+
+    def run(self, django_main_thread):
+        logger.debug('Waiting for apps ready_event.')
+        self.wait_for_apps_ready(apps, django_main_thread)
+        from django.urls import get_resolver
+        # Prevent a race condition where URL modules aren't loaded when the
+        # reloader starts by accessing the urlconf_module property.
+        get_resolver().urlconf_module
+        logger.debug('Apps ready_event triggered. Sending autoreload_started signal.')
+        autoreload_started.send(sender=self)
+        self.run_loop()
+
+    def run_loop(self):
+        ticker = self.tick()
+        while not self.should_stop:
+            try:
+                next(ticker)
+            except StopIteration:
+                break
+        self.stop()
+
+    def tick(self):
+        """
+        This generator is called in a loop from run_loop. It's important that
+        the method takes care of pausing or otherwise waiting for a period of
+        time. This split between run_loop() and tick() is to improve the
+        testability of the reloader implementations by decoupling the work they
+        do from the loop.
+        """
+        raise NotImplementedError('subclasses must implement tick().')
+
+    @classmethod
+    def check_availability(cls):
+        raise NotImplementedError('subclasses must implement check_availability().')
+
+    def notify_file_changed(self, path):
+        results = file_changed.send(sender=self, file_path=path)
+        logger.debug('%s notified as changed. Signal results: %s.', path, results)
+        if not any(res[1] for res in results):
+            trigger_reload(path)
+
+    # These are primarily used for testing.
+    @property
+    def should_stop(self):
+        return self._stop_condition.is_set()
+
+    def stop(self):
+        self._stop_condition.set()
+
+
+class StatReloader(BaseReloader):
+    SLEEP_TIME = 1  # Check for changes once per second.
+
+    def tick(self):
+        state, previous_timestamp = {}, time.time()
+        while True:
+            state.update(self.loop_files(state, previous_timestamp))
+            previous_timestamp = time.time()
+            time.sleep(self.SLEEP_TIME)
+            yield
+
+    def loop_files(self, previous_times, previous_timestamp):
+        updated_times = {}
+        for path, mtime in self.snapshot_files():
+            previous_time = previous_times.get(path)
+            # If there are overlapping globs, a file may be iterated twice.
+            if path in updated_times:
+                continue
+            # A new file has been detected. This could happen due to it being
+            # imported at runtime and only being polled now, or because the
+            # file was just created. Compare the file's mtime to the
+            # previous_timestamp and send a notification if it was created
+            # since the last poll.
+            is_newly_created = previous_time is None and mtime > previous_timestamp
+            is_changed = previous_time is not None and previous_time != mtime
+            if is_newly_created or is_changed:
+                logger.debug('File %s. is_changed: %s, is_new: %s', path, is_changed, is_newly_created)
+                logger.debug('File %s previous mtime: %s, current mtime: %s', path, previous_time, mtime)
+                self.notify_file_changed(path)
+                updated_times[path] = mtime
+        return updated_times
+
+    def snapshot_files(self):
+        for file in self.watched_files():
+            try:
+                mtime = file.stat().st_mtime
+            except OSError:
+                # This is thrown when the file does not exist.
+                continue
+            yield file, mtime
+
+    @classmethod
+    def check_availability(cls):
+        return True
+
+
+class WatchmanUnavailable(RuntimeError):
+    pass
+
+
+class WatchmanReloader(BaseReloader):
+    def __init__(self):
+        self.roots = defaultdict(set)
+        self.processed_request = threading.Event()
+        super().__init__()
+
+    @cached_property
+    def client(self):
+        return pywatchman.client()
+
+    def _watch_root(self, root):
+        # In practice this shouldn't occur, however, it's possible that a
+        # directory that doesn't exist yet is being watched. If it's outside of
+        # sys.path then this will end up a new root. How to handle this isn't
+        # clear: Not adding the root will likely break when subscribing to the
+        # changes, however, as this is currently an internal API,  no files
+        # will be being watched outside of sys.path. Fixing this by checking
+        # inside watch_glob() and watch_dir() is expensive, instead this could
+        # could fall back to the StatReloader if this case is detected? For
+        # now, watching its parent, if possible, is sufficient.
+        if not root.exists():
+            if not root.parent.exists():
+                logger.warning('Unable to watch root dir %s as neither it or its parent exist.', root)
+                return
+            root = root.parent
+        result = self.client.query('watch-project', str(root.absolute()))
+        if 'warning' in result:
+            logger.warning('Watchman warning: %s', result['warning'])
+        logger.debug('Watchman watch-project result: %s', result)
+        return result['watch'], result.get('relative_path')
+
+    @functools.lru_cache()
+    def _get_clock(self, root):
+        return self.client.query('clock', root)['clock']
+
+    def _subscribe(self, directory, name, expression):
+        root, rel_path = self._watch_root(directory)
+        query = {
+            'expression': expression,
+            'fields': ['name'],
+            'since': self._get_clock(root),
+            'dedup_results': True,
+        }
+        if rel_path:
+            query['relative_root'] = rel_path
+        logger.debug('Issuing watchman subscription %s, for root %s. Query: %s', name, root, query)
+        self.client.query('subscribe', root, name, query)
+
+    def _subscribe_dir(self, directory, filenames):
+        if not directory.exists():
+            if not directory.parent.exists():
+                logger.warning('Unable to watch directory %s as neither it or its parent exist.', directory)
+                return
+            prefix = 'files-parent-%s' % directory.name
+            filenames = ['%s/%s' % (directory.name, filename) for filename in filenames]
+            directory = directory.parent
+            expression = ['name', filenames, 'wholename']
+        else:
+            prefix = 'files'
+            expression = ['name', filenames]
+        self._subscribe(directory, '%s:%s' % (prefix, directory), expression)
+
+    def _watch_glob(self, directory, patterns):
+        """
+        Watch a directory with a specific glob. If the directory doesn't yet
+        exist, attempt to watch the parent directory and amend the patterns to
+        include this. It's important this method isn't called more than one per
+        directory when updating all subscriptions. Subsequent calls will
+        overwrite the named subscription, so it must include all possible glob
+        expressions.
+        """
+        prefix = 'glob'
+        if not directory.exists():
+            if not directory.parent.exists():
+                logger.warning('Unable to watch directory %s as neither it or its parent exist.', directory)
+                return
+            prefix = 'glob-parent-%s' % directory.name
+            patterns = ['%s/%s' % (directory.name, pattern) for pattern in patterns]
+            directory = directory.parent
+
+        expression = ['anyof']
+        for pattern in patterns:
+            expression.append(['match', pattern, 'wholename'])
+        self._subscribe(directory, '%s:%s' % (prefix, directory), expression)
+
+    def watched_roots(self, watched_files):
+        extra_directories = self.directory_globs.keys()
+        watched_file_dirs = [f.parent for f in watched_files]
+        sys_paths = list(sys_path_directories())
+        return frozenset((*extra_directories, *watched_file_dirs, *sys_paths))
+
+    def _update_watches(self):
+        watched_files = list(self.watched_files(include_globs=False))
+        found_roots = common_roots(self.watched_roots(watched_files))
+        logger.debug('Watching %s files', len(watched_files))
+        logger.debug('Found common roots: %s', found_roots)
+        # Setup initial roots for performance, shortest roots first.
+        for root in sorted(found_roots):
+            self._watch_root(root)
+        for directory, patterns in self.directory_globs.items():
+            self._watch_glob(directory, patterns)
+        # Group sorted watched_files by their parent directory.
+        sorted_files = sorted(watched_files, key=lambda p: p.parent)
+        for directory, group in itertools.groupby(sorted_files, key=lambda p: p.parent):
+            # These paths need to be relative to the parent directory.
+            self._subscribe_dir(directory, [str(p.relative_to(directory)) for p in group])
+
+    def update_watches(self):
         try:
-            reloader_thread()
+            self._update_watches()
-        except KeyboardInterrupt:
+        except Exception as ex:
-            pass
+            # If the service is still available, raise the original exception.
-    else:
+            if self.check_server_status(ex):
-        try:
+                raise
-            exit_code = restart_with_reloader()
+
-            if exit_code < 0:
+    def _check_subscription(self, sub):
-                os.kill(os.getpid(), -exit_code)
+        subscription = self.client.getSubscription(sub)
+        if not subscription:
+            return
+        logger.debug('Watchman subscription %s has results.', sub)
+        for result in subscription:
+            # When using watch-project, it's not simple to get the relative
+            # directory without storing some specific state. Store the full
+            # path to the directory in the subscription name, prefixed by its
+            # type (glob, files).
+            root_directory = Path(result['subscription'].split(':', 1)[1])
+            logger.debug('Found root directory %s', root_directory)
+            for file in result.get('files', []):
+                self.notify_file_changed(root_directory / file)
+
+    def request_processed(self, **kwargs):
+        logger.debug('Request processed. Setting update_watches event.')
+        self.processed_request.set()
+
+    def tick(self):
+        request_finished.connect(self.request_processed)
+        self.update_watches()
+        while True:
+            if self.processed_request.is_set():
+                self.update_watches()
+                self.processed_request.clear()
+            try:
+                self.client.receive()
+            except pywatchman.WatchmanError as ex:
+                self.check_server_status(ex)
             else:
-                sys.exit(exit_code)
+                for sub in list(self.client.subs.keys()):
-        except KeyboardInterrupt:
+                    self._check_subscription(sub)
-            pass
+            yield
+
+    def stop(self):
+        self.client.close()
+        super().stop()
+
+    def check_server_status(self, inner_ex=None):
+        """Return True if the server is available."""
+        try:
+            self.client.query('version')
+        except Exception:
+            raise WatchmanUnavailable(str(inner_ex)) from inner_ex
+        return True
+
+    @classmethod
+    def check_availability(cls):
+        if not pywatchman:
+            raise WatchmanUnavailable('pywatchman not installed.')
+        client = pywatchman.client(timeout=0.01)
+        try:
+            result = client.capabilityCheck()
+        except Exception:
+            # The service is down?
+            raise WatchmanUnavailable('Cannot connect to the watchman service.')
+        version = get_version_tuple(result['version'])
+        # Watchman 4.9 includes multiple improvements to watching project
+        # directories as well as case insensitive filesystems.
+        logger.debug('Watchman version %s', version)
+        if version < (4, 9):
+            raise WatchmanUnavailable('Watchman 4.9 or later is required.')
 
 
-def main(main_func, args=None, kwargs=None):
+def get_reloader():
-    if args is None:
+    """Return the most suitable reloader for this environment."""
-        args = ()
+    try:
-    if kwargs is None:
+        WatchmanReloader.check_availability()
-        kwargs = {}
+    except WatchmanUnavailable:
+        return StatReloader()
+    return WatchmanReloader()
 
-    wrapped_main_func = check_errors(main_func)
+
-    python_reloader(wrapped_main_func, args, kwargs)
+def start_django(reloader, main_func, *args, **kwargs):
+    ensure_echo_on()
+
+    main_func = check_errors(main_func)
+    django_main_thread = threading.Thread(target=main_func, args=args, kwargs=kwargs)
+    django_main_thread.setDaemon(True)
+    django_main_thread.start()
+
+    while not reloader.should_stop:
+        try:
+            reloader.run(django_main_thread)
+        except WatchmanUnavailable as ex:
+            # It's possible that the watchman service shuts down or otherwise
+            # becomes unavailable. In that case, use the StatReloader.
+            reloader = StatReloader()
+            logger.error('Error connecting to Watchman: %s', ex)
+            logger.info('Watching for file changes with %s', reloader.__class__.__name__)
+
+
+def run_with_reloader(main_func, *args, **kwargs):
+    signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
+    try:
+        if os.environ.get(DJANGO_AUTORELOAD_ENV) == 'true':
+            reloader = get_reloader()
+            logger.info('Watching for file changes with %s', reloader.__class__.__name__)
+            start_django(reloader, main_func, *args, **kwargs)
+        else:
+            try:
+                WatchmanReloader.check_availability()
+            except WatchmanUnavailable as e:
+                logger.info('Watchman unavailable: %s.', e)
+            exit_code = restart_with_reloader()
+            sys.exit(exit_code)
+    except KeyboardInterrupt:
+        pass

django/utils/translation/__init__.py

@@ -4,6 +4,7 @@ Internationalization support.
 import re
 from contextlib import ContextDecorator
 
+from django.utils.autoreload import autoreload_started, file_changed
 from django.utils.functional import lazy
 
 __all__ = [
@@ -52,6 +53,9 @@ class Trans:
         from django.conf import settings
         if settings.USE_I18N:
             from django.utils.translation import trans_real as trans
+            from django.utils.translation.reloader import watch_for_translation_changes, translation_file_changed
+            autoreload_started.connect(watch_for_translation_changes, dispatch_uid='translation_file_changed')
+            file_changed.connect(translation_file_changed, dispatch_uid='translation_file_changed')
         else:
             from django.utils.translation import trans_null as trans
         setattr(self, real_name, getattr(trans, real_name))

django/utils/translation/reloader.py

@@ -0,0 +1,29 @@
+import threading
+from pathlib import Path
+
+from django.apps import apps
+
+
+def watch_for_translation_changes(sender, **kwargs):
+    """Register file watchers for .mo files in potential locale paths."""
+    from django.conf import settings
+
+    if settings.USE_I18N:
+        directories = [Path('locale')]
+        directories.extend(Path(config.path) / 'locale' for config in apps.get_app_configs())
+        directories.extend(Path(p) for p in settings.LOCALE_PATHS)
+        for path in directories:
+            absolute_path = path.absolute()
+            sender.watch_dir(absolute_path, '**/*.mo')
+
+
+def translation_file_changed(sender, file_path, **kwargs):
+    """Clear the internal translations cache if a .mo file is modified."""
+    if file_path.suffix == '.mo':
+        import gettext
+        from django.utils.translation import trans_real
+        gettext._translations = {}
+        trans_real._translations = {}
+        trans_real._default = None
+        trans_real._active = threading.local()
+        return True

docs/internals/contributing/writing-code/unit-tests.txt

@@ -229,6 +229,7 @@ dependencies:
 *  Pillow_
 *  PyYAML_
 *  pytz_ (required)
+*  pywatchman_
 *  setuptools_
 *  memcached_, plus a :ref:`supported Python binding <memcached>`
 *  gettext_ (:ref:`gettext_on_windows`)
@@ -258,6 +259,9 @@ and install the Geospatial libraries</ref/contrib/gis/install/index>`.
 Each of these dependencies is optional. If you're missing any of them, the
 associated tests will be skipped.
 
+To run some of the autoreload tests, you'll need to install the Watchman_
+service.
+
 .. _argon2-cffi: https://pypi.org/project/argon2_cffi/
 .. _bcrypt: https://pypi.org/project/bcrypt/
 .. _docutils: https://pypi.org/project/docutils/
@@ -267,12 +271,14 @@ associated tests will be skipped.
 .. _Pillow: https://pypi.org/project/Pillow/
 .. _PyYAML: https://pyyaml.org/wiki/PyYAML
 .. _pytz: https://pypi.org/project/pytz/
+.. _pywatchman: https://pypi.org/project/pywatchman/
 .. _setuptools: https://pypi.org/project/setuptools/
 .. _memcached: https://memcached.org/
 .. _gettext: https://www.gnu.org/software/gettext/manual/gettext.html
 .. _selenium: https://pypi.org/project/selenium/
 .. _sqlparse: https://pypi.org/project/sqlparse/
 .. _pip requirements files: https://pip.pypa.io/en/latest/user_guide/#requirements-files
+.. _Watchman: https://facebook.github.io/watchman/
 
 Code coverage
 -------------

docs/ref/django-admin.txt

@@ -879,13 +879,26 @@ needed. You don't need to restart the server for code changes to take effect.
 However, some actions like adding files don't trigger a restart, so you'll
 have to restart the server in these cases.
 
-If you are using Linux and install `pyinotify`_, kernel signals will be used to
+If you're using Linux or MacOS and install both `pywatchman`_ and the
-autoreload the server (rather than polling file modification timestamps each
+`Watchman`_ service, kernel signals will be used to autoreload the server
-second). This offers better scaling to large projects, reduction in response
+(rather than polling file modification timestamps each second). This offers
-time to code modification, more robust change detection, and battery usage
+better performance on large projects, reduced response time after code changes,
-reduction.
+more robust change detection, and a reduction in power usage.
 
-.. _pyinotify: https://pypi.org/project/pyinotify/
+.. admonition:: Large directories with many files may cause performance issues
+
+    When using Watchman with a project that includes large non-Python
+    directories like ``node_modules``, it's advisable to ignore this directory
+    for optimal performance. See the `watchman documentation`_ for information
+    on how to do this.
+
+.. _Watchman: https://facebook.github.io/watchman/
+.. _pywatchman: https://pypi.org/project/pywatchman/
+.. _watchman documentation: https://facebook.github.io/watchman/docs/config.html#ignore_dirs
+
+.. versionchanged:: 2.2
+
+   Watchman support replaced support for `pyinotify`.
 
 When you start the server, and each time you change Python code while the
 server is running, the system check framework will check your entire Django

docs/releases/2.2.txt

@@ -203,6 +203,10 @@ Management Commands
   comments in generated migration file(s). This option is also available for
   :djadmin:`squashmigrations`.
 
+* :djadmin:`runserver` can now use `Watchman
+  <https://facebook.github.io/watchman/>`_ to improve the performance of
+  watching a large number of files for changes.
+
 Migrations
 ~~~~~~~~~~
 
@@ -487,6 +491,8 @@ Miscellaneous
   :func:`~django.contrib.sitemaps.ping_google` function, set the new
   ``sitemap_uses_https`` argument to ``False``.
 
+* :djadmin:`runserver` no longer supports `pyinotify` (replaced by Watchman).
+
 .. _deprecated-features-2.2:
 
 Features deprecated in 2.2

tests/apps/tests.py

@@ -48,6 +48,9 @@ class AppsTests(SimpleTestCase):
         self.assertIs(apps.ready, True)
         # Non-master app registries are populated in __init__.
         self.assertIs(Apps().ready, True)
+        # The condition is set when apps are ready
+        self.assertIs(apps.ready_event.is_set(), True)
+        self.assertIs(Apps().ready_event.is_set(), True)
 
     def test_bad_app_config(self):
         """

tests/i18n/tests.py

@@ -7,9 +7,12 @@ import re
 import tempfile
 from contextlib import contextmanager
 from importlib import import_module
+from pathlib import Path
 from threading import local
 from unittest import mock
 
+import _thread
+
 from django import forms
 from django.apps import AppConfig
 from django.conf import settings
@@ -33,6 +36,9 @@ from django.utils.translation import (
     npgettext, npgettext_lazy, pgettext, to_language, to_locale, trans_null,
     trans_real, ugettext, ugettext_lazy, ungettext, ungettext_lazy,
 )
+from django.utils.translation.reloader import (
+    translation_file_changed, watch_for_translation_changes,
+)
 
 from .forms import CompanyForm, I18nForm, SelectDateForm
 from .models import Company, TestModel
@@ -1790,3 +1796,65 @@ class NonDjangoLanguageTests(SimpleTestCase):
     def test_plural_non_django_language(self):
         self.assertEqual(get_language(), 'xyz')
         self.assertEqual(ngettext('year', 'years', 2), 'years')
+
+
+@override_settings(USE_I18N=True)
+class WatchForTranslationChangesTests(SimpleTestCase):
+    @override_settings(USE_I18N=False)
+    def test_i18n_disabled(self):
+        mocked_sender = mock.MagicMock()
+        watch_for_translation_changes(mocked_sender)
+        mocked_sender.watch_dir.assert_not_called()
+
+    def test_i18n_enabled(self):
+        mocked_sender = mock.MagicMock()
+        watch_for_translation_changes(mocked_sender)
+        self.assertGreater(mocked_sender.watch_dir.call_count, 1)
+
+    def test_i18n_locale_paths(self):
+        mocked_sender = mock.MagicMock()
+        with tempfile.TemporaryDirectory() as app_dir:
+            with self.settings(LOCALE_PATHS=[app_dir]):
+                watch_for_translation_changes(mocked_sender)
+            mocked_sender.watch_dir.assert_any_call(Path(app_dir), '**/*.mo')
+
+    def test_i18n_app_dirs(self):
+        mocked_sender = mock.MagicMock()
+        with self.settings(INSTALLED_APPS=['tests.i18n.sampleproject']):
+            watch_for_translation_changes(mocked_sender)
+        project_dir = Path(__file__).parent / 'sampleproject' / 'locale'
+        mocked_sender.watch_dir.assert_any_call(project_dir, '**/*.mo')
+
+    def test_i18n_local_locale(self):
+        mocked_sender = mock.MagicMock()
+        watch_for_translation_changes(mocked_sender)
+        locale_dir = Path(__file__).parent / 'locale'
+        mocked_sender.watch_dir.assert_any_call(locale_dir, '**/*.mo')
+
+
+class TranslationFileChangedTests(SimpleTestCase):
+    def setUp(self):
+        self.gettext_translations = gettext_module._translations.copy()
+        self.trans_real_translations = trans_real._translations.copy()
+
+    def tearDown(self):
+        gettext._translations = self.gettext_translations
+        trans_real._translations = self.trans_real_translations
+
+    def test_ignores_non_mo_files(self):
+        gettext_module._translations = {'foo': 'bar'}
+        path = Path('test.py')
+        self.assertIsNone(translation_file_changed(None, path))
+        self.assertEqual(gettext_module._translations, {'foo': 'bar'})
+
+    def test_resets_cache_with_mo_files(self):
+        gettext_module._translations = {'foo': 'bar'}
+        trans_real._translations = {'foo': 'bar'}
+        trans_real._default = 1
+        trans_real._active = False
+        path = Path('test.mo')
+        self.assertIs(translation_file_changed(None, path), True)
+        self.assertEqual(gettext_module._translations, {})
+        self.assertEqual(trans_real._translations, {})
+        self.assertIsNone(trans_real._default)
+        self.assertIsInstance(trans_real._active, _thread._local)

tests/requirements/py3.txt

@@ -9,6 +9,7 @@ Pillow != 5.4.0
 pylibmc; sys.platform != 'win32'
 python-memcached >= 1.59
 pytz
+pywatchman; sys.platform != 'win32'
 PyYAML
 selenium
 sqlparse

tests/utils_tests/locale/nl/LC_MESSAGES/django.po

@@ -1,17 +0,0 @@
-# SOME DESCRIPTIVE TITLE.
-# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
-# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
-#
-#, fuzzy
-msgid ""
-msgstr ""
-"Project-Id-Version: PACKAGE VERSION\n"
-"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2007-09-15 19:15+0200\n"
-"PO-Revision-Date: 2010-05-12 12:41-0300\n"
-"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
-"Language-Team: LANGUAGE <LL@li.org>\n"
-"MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=UTF-8\n"
-"Content-Transfer-Encoding: 8bit\n"

tests/utils_tests/test_autoreload.py

@@ -1,257 +1,279 @@
-import gettext
+import contextlib
 import os
+import py_compile
 import shutil
+import sys
 import tempfile
+import threading
+import time
+import zipfile
 from importlib import import_module
-from unittest import mock
+from pathlib import Path
+from unittest import mock, skip
 
-import _thread
+from django.apps.registry import Apps
-
+from django.test import SimpleTestCase
-from django import conf
-from django.contrib import admin
-from django.test import SimpleTestCase, override_settings
 from django.test.utils import extend_sys_path
 from django.utils import autoreload
-from django.utils.translation import trans_real
+from django.utils.autoreload import WatchmanUnavailable
-
-LOCALE_PATH = os.path.join(os.path.dirname(__file__), 'locale')
 
 
-class TestFilenameGenerator(SimpleTestCase):
+class TestIterModulesAndFiles(SimpleTestCase):
+    def import_and_cleanup(self, name):
+        import_module(name)
+        self.addCleanup(lambda: sys.path_importer_cache.clear())
+        self.addCleanup(lambda: sys.modules.pop(name, None))
 
     def clear_autoreload_caches(self):
-        autoreload._cached_modules = set()
+        autoreload.iter_modules_and_files.cache_clear()
-        autoreload._cached_filenames = []
 
     def assertFileFound(self, filename):
+        # Some temp directories are symlinks. Python resolves these fully while
+        # importing.
+        resolved_filename = filename.resolve()
         self.clear_autoreload_caches()
         # Test uncached access
-        self.assertIn(filename, autoreload.gen_filenames())
+        self.assertIn(resolved_filename, list(autoreload.iter_all_python_module_files()))
         # Test cached access
-        self.assertIn(filename, autoreload.gen_filenames())
+        self.assertIn(resolved_filename, list(autoreload.iter_all_python_module_files()))
+        self.assertEqual(autoreload.iter_modules_and_files.cache_info().hits, 1)
 
     def assertFileNotFound(self, filename):
+        resolved_filename = filename.resolve()
         self.clear_autoreload_caches()
         # Test uncached access
-        self.assertNotIn(filename, autoreload.gen_filenames())
+        self.assertNotIn(resolved_filename, list(autoreload.iter_all_python_module_files()))
         # Test cached access
-        self.assertNotIn(filename, autoreload.gen_filenames())
+        self.assertNotIn(resolved_filename, list(autoreload.iter_all_python_module_files()))
+        self.assertEqual(autoreload.iter_modules_and_files.cache_info().hits, 1)
 
-    def assertFileFoundOnlyNew(self, filename):
+    def temporary_file(self, filename):
-        self.clear_autoreload_caches()
-        # Test uncached access
-        self.assertIn(filename, autoreload.gen_filenames(only_new=True))
-        # Test cached access
-        self.assertNotIn(filename, autoreload.gen_filenames(only_new=True))
-
-    def test_django_locales(self):
-        """
-        gen_filenames() yields the built-in Django locale files.
-        """
-        django_dir = os.path.join(os.path.dirname(conf.__file__), 'locale')
-        django_mo = os.path.join(django_dir, 'nl', 'LC_MESSAGES', 'django.mo')
-        self.assertFileFound(django_mo)
-
-    @override_settings(LOCALE_PATHS=[LOCALE_PATH])
-    def test_locale_paths_setting(self):
-        """
-        gen_filenames also yields from LOCALE_PATHS locales.
-        """
-        locale_paths_mo = os.path.join(LOCALE_PATH, 'nl', 'LC_MESSAGES', 'django.mo')
-        self.assertFileFound(locale_paths_mo)
-
-    @override_settings(INSTALLED_APPS=[])
-    def test_project_root_locale(self):
-        """
-        gen_filenames() also yields from the current directory (project root).
-        """
-        old_cwd = os.getcwd()
-        os.chdir(os.path.dirname(__file__))
-        current_dir = os.path.join(os.path.dirname(__file__), 'locale')
-        current_dir_mo = os.path.join(current_dir, 'nl', 'LC_MESSAGES', 'django.mo')
-        try:
-            self.assertFileFound(current_dir_mo)
-        finally:
-            os.chdir(old_cwd)
-
-    @override_settings(INSTALLED_APPS=['django.contrib.admin'])
-    def test_app_locales(self):
-        """
-        gen_filenames() also yields from locale dirs in installed apps.
-        """
-        admin_dir = os.path.join(os.path.dirname(admin.__file__), 'locale')
-        admin_mo = os.path.join(admin_dir, 'nl', 'LC_MESSAGES', 'django.mo')
-        self.assertFileFound(admin_mo)
-
-    @override_settings(USE_I18N=False)
-    def test_no_i18n(self):
-        """
-        If i18n machinery is disabled, there is no need for watching the
-        locale files.
-        """
-        django_dir = os.path.join(os.path.dirname(conf.__file__), 'locale')
-        django_mo = os.path.join(django_dir, 'nl', 'LC_MESSAGES', 'django.mo')
-        self.assertFileNotFound(django_mo)
-
-    def test_paths_are_native_strings(self):
-        for filename in autoreload.gen_filenames():
-            self.assertIsInstance(filename, str)
-
-    def test_only_new_files(self):
-        """
-        When calling a second time gen_filenames with only_new = True, only
-        files from newly loaded modules should be given.
-        """
         dirname = tempfile.mkdtemp()
-        filename = os.path.join(dirname, 'test_only_new_module.py')
         self.addCleanup(shutil.rmtree, dirname)
-        with open(filename, 'w'):
+        return Path(dirname) / filename
-            pass
 
-        # Test uncached access
+    def test_paths_are_pathlib_instances(self):
-        self.clear_autoreload_caches()
+        for filename in autoreload.iter_all_python_module_files():
-        filenames = set(autoreload.gen_filenames(only_new=True))
+            self.assertIsInstance(filename, Path)
-        filenames_reference = set(autoreload.gen_filenames())
-        self.assertEqual(filenames, filenames_reference)
 
-        # Test cached access: no changes
+    def test_file_added(self):
-        filenames = set(autoreload.gen_filenames(only_new=True))
-        self.assertEqual(filenames, set())
-
-        # Test cached access: add a module
-        with extend_sys_path(dirname):
-            import_module('test_only_new_module')
-        filenames = set(autoreload.gen_filenames(only_new=True))
-        self.assertEqual(filenames, {filename})
-
-    def test_deleted_removed(self):
         """
-        When a file is deleted, gen_filenames() no longer returns it.
+        When a file is added, it's returned by iter_all_python_module_files().
         """
-        dirname = tempfile.mkdtemp()
+        filename = self.temporary_file('test_deleted_removed_module.py')
-        filename = os.path.join(dirname, 'test_deleted_removed_module.py')
+        filename.touch()
-        self.addCleanup(shutil.rmtree, dirname)
-        with open(filename, 'w'):
-            pass
 
-        with extend_sys_path(dirname):
+        with extend_sys_path(str(filename.parent)):
-            import_module('test_deleted_removed_module')
+            self.import_and_cleanup('test_deleted_removed_module')
-        self.assertFileFound(filename)
 
-        os.unlink(filename)
+        self.assertFileFound(filename.absolute())
-        self.assertFileNotFound(filename)
 
     def test_check_errors(self):
         """
         When a file containing an error is imported in a function wrapped by
         check_errors(), gen_filenames() returns it.
         """
-        dirname = tempfile.mkdtemp()
+        filename = self.temporary_file('test_syntax_error.py')
-        filename = os.path.join(dirname, 'test_syntax_error.py')
+        filename.write_text("Ceci n'est pas du Python.")
-        self.addCleanup(shutil.rmtree, dirname)
-        with open(filename, 'w') as f:
-            f.write("Ceci n'est pas du Python.")
 
-        with extend_sys_path(dirname):
+        with extend_sys_path(str(filename.parent)):
             with self.assertRaises(SyntaxError):
                 autoreload.check_errors(import_module)('test_syntax_error')
         self.assertFileFound(filename)
 
-    def test_check_errors_only_new(self):
-        """
-        When a file containing an error is imported in a function wrapped by
-        check_errors(), gen_filenames(only_new=True) returns it.
-        """
-        dirname = tempfile.mkdtemp()
-        filename = os.path.join(dirname, 'test_syntax_error.py')
-        self.addCleanup(shutil.rmtree, dirname)
-        with open(filename, 'w') as f:
-            f.write("Ceci n'est pas du Python.")
-
-        with extend_sys_path(dirname):
-            with self.assertRaises(SyntaxError):
-                autoreload.check_errors(import_module)('test_syntax_error')
-        self.assertFileFoundOnlyNew(filename)
-
     def test_check_errors_catches_all_exceptions(self):
         """
         Since Python may raise arbitrary exceptions when importing code,
         check_errors() must catch Exception, not just some subclasses.
         """
-        dirname = tempfile.mkdtemp()
+        filename = self.temporary_file('test_exception.py')
-        filename = os.path.join(dirname, 'test_exception.py')
+        filename.write_text('raise Exception')
-        self.addCleanup(shutil.rmtree, dirname)
+        with extend_sys_path(str(filename.parent)):
-        with open(filename, 'w') as f:
-            f.write("raise Exception")
-
-        with extend_sys_path(dirname):
             with self.assertRaises(Exception):
                 autoreload.check_errors(import_module)('test_exception')
         self.assertFileFound(filename)
 
+    def test_zip_reload(self):
+        """
+        Modules imported from zipped files have their archive location included
+        in the result.
+        """
+        zip_file = self.temporary_file('zip_import.zip')
+        with zipfile.ZipFile(str(zip_file), 'w', zipfile.ZIP_DEFLATED) as zipf:
+            zipf.writestr('test_zipped_file.py', '')
 
-class CleanFilesTests(SimpleTestCase):
+        with extend_sys_path(str(zip_file)):
-    TEST_MAP = {
+            self.import_and_cleanup('test_zipped_file')
-        # description: (input_file_list, expected_returned_file_list)
+        self.assertFileFound(zip_file)
-        'falsies': ([None, False], []),
+
-        'pycs': (['myfile.pyc'], ['myfile.py']),
+    def test_bytecode_conversion_to_source(self):
-        'pyos': (['myfile.pyo'], ['myfile.py']),
+        """.pyc and .pyo files are included in the files list."""
-        '$py.class': (['myclass$py.class'], ['myclass.py']),
+        filename = self.temporary_file('test_compiled.py')
-        'combined': (
+        filename.touch()
-            [None, 'file1.pyo', 'file2.pyc', 'myclass$py.class'],
+        compiled_file = Path(py_compile.compile(str(filename), str(filename.with_suffix('.pyc'))))
-            ['file1.py', 'file2.py', 'myclass.py'],
+        filename.unlink()
+        with extend_sys_path(str(compiled_file.parent)):
+            self.import_and_cleanup('test_compiled')
+        self.assertFileFound(compiled_file)
+
+
+class TestCommonRoots(SimpleTestCase):
+    def test_common_roots(self):
+        paths = (
+            Path('/first/second'),
+            Path('/first/second/third'),
+            Path('/first/'),
+            Path('/root/first/'),
         )
-    }
+        results = autoreload.common_roots(paths)
-
+        self.assertCountEqual(results, [Path('/first/'), Path('/root/first/')])
-    def _run_tests(self, mock_files_exist=True):
-        with mock.patch('django.utils.autoreload.os.path.exists', return_value=mock_files_exist):
-            for description, values in self.TEST_MAP.items():
-                filenames, expected_returned_filenames = values
-                self.assertEqual(
-                    autoreload.clean_files(filenames),
-                    expected_returned_filenames if mock_files_exist else [],
-                    msg='{} failed for input file list: {}; returned file list: {}'.format(
-                        description, filenames, expected_returned_filenames
-                    ),
-                )
-
-    def test_files_exist(self):
-        """
-        If the file exists, any compiled files (pyc, pyo, $py.class) are
-        transformed as their source files.
-        """
-        self._run_tests()
-
-    def test_files_do_not_exist(self):
-        """
-        If the files don't exist, they aren't in the returned file list.
-        """
-        self._run_tests(mock_files_exist=False)
 
 
-class ResetTranslationsTests(SimpleTestCase):
+class TestSysPathDirectories(SimpleTestCase):
-
     def setUp(self):
-        self.gettext_translations = gettext._translations.copy()
+        self._directory = tempfile.TemporaryDirectory()
-        self.trans_real_translations = trans_real._translations.copy()
+        self.directory = Path(self._directory.name).resolve().absolute()
+        self.file = self.directory / 'test'
+        self.file.touch()
 
     def tearDown(self):
-        gettext._translations = self.gettext_translations
+        self._directory.cleanup()
-        trans_real._translations = self.trans_real_translations
 
-    def test_resets_gettext(self):
+    def test_sys_paths_with_directories(self):
-        gettext._translations = {'foo': 'bar'}
+        with extend_sys_path(str(self.file)):
-        autoreload.reset_translations()
+            paths = list(autoreload.sys_path_directories())
-        self.assertEqual(gettext._translations, {})
+        self.assertIn(self.file.parent, paths)
 
-    def test_resets_trans_real(self):
+    def test_sys_paths_non_existing(self):
-        trans_real._translations = {'foo': 'bar'}
+        nonexistant_file = Path(self.directory.name) / 'does_not_exist'
-        trans_real._default = 1
+        with extend_sys_path(str(nonexistant_file)):
-        trans_real._active = False
+            paths = list(autoreload.sys_path_directories())
-        autoreload.reset_translations()
+        self.assertNotIn(nonexistant_file, paths)
-        self.assertEqual(trans_real._translations, {})
+        self.assertNotIn(nonexistant_file.parent, paths)
-        self.assertIsNone(trans_real._default)
+
-        self.assertIsInstance(trans_real._active, _thread._local)
+    def test_sys_paths_absolute(self):
+        paths = list(autoreload.sys_path_directories())
+        self.assertTrue(all(p.is_absolute() for p in paths))
+
+    def test_sys_paths_directories(self):
+        with extend_sys_path(str(self.directory)):
+            paths = list(autoreload.sys_path_directories())
+        self.assertIn(self.directory, paths)
+
+
+class GetReloaderTests(SimpleTestCase):
+    @mock.patch('django.utils.autoreload.WatchmanReloader')
+    def test_watchman_unavailable(self, mocked_watchman):
+        mocked_watchman.check_availability.side_effect = WatchmanUnavailable
+        self.assertIsInstance(autoreload.get_reloader(), autoreload.StatReloader)
+
+    @mock.patch.object(autoreload.WatchmanReloader, 'check_availability')
+    def test_watchman_available(self, mocked_available):
+        # If WatchmanUnavailable isn't raised, Watchman will be chosen.
+        mocked_available.return_value = None
+        result = autoreload.get_reloader()
+        self.assertIsInstance(result, autoreload.WatchmanReloader)
+
+
+class RunWithReloaderTests(SimpleTestCase):
+    @mock.patch.dict(os.environ, {autoreload.DJANGO_AUTORELOAD_ENV: 'true'})
+    @mock.patch('django.utils.autoreload.get_reloader')
+    def test_swallows_keyboard_interrupt(self, mocked_get_reloader):
+        mocked_get_reloader.side_effect = KeyboardInterrupt()
+        autoreload.run_with_reloader(lambda: None)  # No exception
+
+    @mock.patch.dict(os.environ, {autoreload.DJANGO_AUTORELOAD_ENV: 'false'})
+    @mock.patch('django.utils.autoreload.restart_with_reloader')
+    def test_calls_sys_exit(self, mocked_restart_reloader):
+        mocked_restart_reloader.return_value = 1
+        with self.assertRaises(SystemExit) as exc:
+            autoreload.run_with_reloader(lambda: None)
+        self.assertEqual(exc.exception.code, 1)
+
+    @mock.patch.dict(os.environ, {autoreload.DJANGO_AUTORELOAD_ENV: 'true'})
+    @mock.patch('django.utils.autoreload.start_django')
+    @mock.patch('django.utils.autoreload.get_reloader')
+    def test_calls_start_django(self, mocked_reloader, mocked_start_django):
+        mocked_reloader.return_value = mock.sentinel.RELOADER
+        autoreload.run_with_reloader(mock.sentinel.METHOD)
+        self.assertEqual(mocked_start_django.call_count, 1)
+        self.assertSequenceEqual(
+            mocked_start_django.call_args[0],
+            [mock.sentinel.RELOADER, mock.sentinel.METHOD]
+        )
+
+
+class StartDjangoTests(SimpleTestCase):
+    @mock.patch('django.utils.autoreload.StatReloader')
+    def test_watchman_becomes_unavailable(self, mocked_stat):
+        mocked_stat.should_stop.return_value = True
+        fake_reloader = mock.MagicMock()
+        fake_reloader.should_stop = False
+        fake_reloader.run.side_effect = autoreload.WatchmanUnavailable()
+
+        autoreload.start_django(fake_reloader, lambda: None)
+        self.assertEqual(mocked_stat.call_count, 1)
+
+    @mock.patch('django.utils.autoreload.ensure_echo_on')
+    def test_echo_on_called(self, mocked_echo):
+        fake_reloader = mock.MagicMock()
+        autoreload.start_django(fake_reloader, lambda: None)
+        self.assertEqual(mocked_echo.call_count, 1)
+
+    @mock.patch('django.utils.autoreload.check_errors')
+    def test_check_errors_called(self, mocked_check_errors):
+        fake_method = mock.MagicMock(return_value=None)
+        fake_reloader = mock.MagicMock()
+        autoreload.start_django(fake_reloader, fake_method)
+        self.assertCountEqual(mocked_check_errors.call_args[0], [fake_method])
+
+    @mock.patch('threading.Thread')
+    @mock.patch('django.utils.autoreload.check_errors')
+    def test_starts_thread_with_args(self, mocked_check_errors, mocked_thread):
+        fake_reloader = mock.MagicMock()
+        fake_main_func = mock.MagicMock()
+        fake_thread = mock.MagicMock()
+        mocked_check_errors.return_value = fake_main_func
+        mocked_thread.return_value = fake_thread
+        autoreload.start_django(fake_reloader, fake_main_func, 123, abc=123)
+        self.assertEqual(mocked_thread.call_count, 1)
+        self.assertEqual(
+            mocked_thread.call_args[1],
+            {'target': fake_main_func, 'args': (123,), 'kwargs': {'abc': 123}}
+        )
+        self.assertSequenceEqual(fake_thread.setDaemon.call_args[0], [True])
+        self.assertTrue(fake_thread.start.called)
+
+
+class TestCheckErrors(SimpleTestCase):
+    def test_mutates_error_files(self):
+        fake_method = mock.MagicMock(side_effect=RuntimeError())
+        wrapped = autoreload.check_errors(fake_method)
+        with mock.patch.object(autoreload, '_error_files') as mocked_error_files:
+            with self.assertRaises(RuntimeError):
+                wrapped()
+        self.assertEqual(mocked_error_files.append.call_count, 1)
+
+
+class TestRaiseLastException(SimpleTestCase):
+    @mock.patch('django.utils.autoreload._exception', None)
+    def test_no_exception(self):
+        # Should raise no exception if _exception is None
+        autoreload.raise_last_exception()
+
+    def test_raises_exception(self):
+        class MyException(Exception):
+            pass
+
+        # Create an exception
+        try:
+            raise MyException('Test Message')
+        except MyException:
+            exc_info = sys.exc_info()
+
+        with mock.patch('django.utils.autoreload._exception', exc_info):
+            with self.assertRaises(MyException, msg='Test Message'):
+                autoreload.raise_last_exception()
 
 
 class RestartWithReloaderTests(SimpleTestCase):
@@ -286,3 +308,363 @@ class RestartWithReloaderTests(SimpleTestCase):
             autoreload.restart_with_reloader()
             self.assertEqual(mock_call.call_count, 1)
             self.assertEqual(mock_call.call_args[0][0], [self.executable, '-Wall', '-m', 'django'] + argv[1:])
+
+
+class ReloaderTests(SimpleTestCase):
+    RELOADER_CLS = None
+
+    def setUp(self):
+        self._tempdir = tempfile.TemporaryDirectory()
+        self.tempdir = Path(self._tempdir.name).resolve().absolute()
+        self.existing_file = self.ensure_file(self.tempdir / 'test.py')
+        self.nonexistant_file = (self.tempdir / 'does_not_exist.py').absolute()
+        self.reloader = self.RELOADER_CLS()
+
+    def tearDown(self):
+        self._tempdir.cleanup()
+        self.reloader.stop()
+
+    def ensure_file(self, path):
+        path.parent.mkdir(exist_ok=True, parents=True)
+        path.touch()
+        # On Linux and Windows updating the mtime of a file using touch() will set a timestamp
+        # value that is in the past, as the time value for the last kernel tick is used rather
+        # than getting the correct absolute time.
+        # To make testing simpler set the mtime to be the observed time when this function is
+        # called.
+        self.set_mtime(path, time.time())
+        return path.absolute()
+
+    def set_mtime(self, fp, value):
+        os.utime(str(fp), (value, value))
+
+    def increment_mtime(self, fp, by=1):
+        current_time = time.time()
+        self.set_mtime(fp, current_time + by)
+
+    @contextlib.contextmanager
+    def tick_twice(self):
+        ticker = self.reloader.tick()
+        next(ticker)
+        yield
+        next(ticker)
+
+
+class IntegrationTests:
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_file(self, mocked_modules, notify_mock):
+        self.reloader.watch_file(self.existing_file)
+        with self.tick_twice():
+            self.increment_mtime(self.existing_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [self.existing_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_nonexistant_file(self, mocked_modules, notify_mock):
+        self.reloader.watch_file(self.nonexistant_file)
+        with self.tick_twice():
+            self.ensure_file(self.nonexistant_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [self.nonexistant_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_nonexistant_file_in_non_existing_directory(self, mocked_modules, notify_mock):
+        non_existing_directory = self.tempdir / 'non_existing_dir'
+        nonexistant_file = non_existing_directory / 'test'
+        self.reloader.watch_file(nonexistant_file)
+        with self.tick_twice():
+            self.ensure_file(nonexistant_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [nonexistant_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_glob(self, mocked_modules, notify_mock):
+        non_py_file = self.ensure_file(self.tempdir / 'non_py_file')
+        self.reloader.watch_dir(self.tempdir, '*.py')
+        with self.tick_twice():
+            self.increment_mtime(non_py_file)
+            self.increment_mtime(self.existing_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [self.existing_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_glob_non_existing_directory(self, mocked_modules, notify_mock):
+        non_existing_directory = self.tempdir / 'does_not_exist'
+        nonexistant_file = non_existing_directory / 'test.py'
+        self.reloader.watch_dir(non_existing_directory, '*.py')
+        with self.tick_twice():
+            self.ensure_file(nonexistant_file)
+            self.set_mtime(nonexistant_file, time.time())
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [nonexistant_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_multiple_globs(self, mocked_modules, notify_mock):
+        self.ensure_file(self.tempdir / 'x.test')
+        self.reloader.watch_dir(self.tempdir, '*.py')
+        self.reloader.watch_dir(self.tempdir, '*.test')
+        with self.tick_twice():
+            self.increment_mtime(self.existing_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [self.existing_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_overlapping_globs(self, mocked_modules, notify_mock):
+        self.reloader.watch_dir(self.tempdir, '*.py')
+        self.reloader.watch_dir(self.tempdir, '*.p*')
+        with self.tick_twice():
+            self.increment_mtime(self.existing_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [self.existing_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_glob_recursive(self, mocked_modules, notify_mock):
+        non_py_file = self.ensure_file(self.tempdir / 'dir' / 'non_py_file')
+        py_file = self.ensure_file(self.tempdir / 'dir' / 'file.py')
+        self.reloader.watch_dir(self.tempdir, '**/*.py')
+        with self.tick_twice():
+            self.increment_mtime(non_py_file)
+            self.increment_mtime(py_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [py_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_multiple_recursive_globs(self, mocked_modules, notify_mock):
+        non_py_file = self.ensure_file(self.tempdir / 'dir' / 'test.txt')
+        py_file = self.ensure_file(self.tempdir / 'dir' / 'file.py')
+        self.reloader.watch_dir(self.tempdir, '**/*.txt')
+        self.reloader.watch_dir(self.tempdir, '**/*.py')
+        with self.tick_twice():
+            self.increment_mtime(non_py_file)
+            self.increment_mtime(py_file)
+        self.assertEqual(notify_mock.call_count, 2)
+        self.assertCountEqual(notify_mock.call_args_list, [mock.call(py_file), mock.call(non_py_file)])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_nested_glob_recursive(self, mocked_modules, notify_mock):
+        inner_py_file = self.ensure_file(self.tempdir / 'dir' / 'file.py')
+        self.reloader.watch_dir(self.tempdir, '**/*.py')
+        self.reloader.watch_dir(inner_py_file.parent, '**/*.py')
+        with self.tick_twice():
+            self.increment_mtime(inner_py_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [inner_py_file])
+
+    @mock.patch('django.utils.autoreload.BaseReloader.notify_file_changed')
+    @mock.patch('django.utils.autoreload.iter_all_python_module_files', return_value=frozenset())
+    def test_overlapping_glob_recursive(self, mocked_modules, notify_mock):
+        py_file = self.ensure_file(self.tempdir / 'dir' / 'file.py')
+        self.reloader.watch_dir(self.tempdir, '**/*.p*')
+        self.reloader.watch_dir(self.tempdir, '**/*.py*')
+        with self.tick_twice():
+            self.increment_mtime(py_file)
+        self.assertEqual(notify_mock.call_count, 1)
+        self.assertCountEqual(notify_mock.call_args[0], [py_file])
+
+
+class BaseReloaderTests(ReloaderTests):
+    RELOADER_CLS = autoreload.BaseReloader
+
+    def test_watch_without_absolute(self):
+        with self.assertRaisesMessage(ValueError, 'test.py must be absolute.'):
+            self.reloader.watch_file('test.py')
+
+    def test_watch_with_single_file(self):
+        self.reloader.watch_file(self.existing_file)
+        watched_files = list(self.reloader.watched_files())
+        self.assertIn(self.existing_file, watched_files)
+
+    def test_watch_with_glob(self):
+        self.reloader.watch_dir(self.tempdir, '*.py')
+        watched_files = list(self.reloader.watched_files())
+        self.assertIn(self.existing_file, watched_files)
+
+    def test_watch_files_with_recursive_glob(self):
+        inner_file = self.ensure_file(self.tempdir / 'test' / 'test.py')
+        self.reloader.watch_dir(self.tempdir, '**/*.py')
+        watched_files = list(self.reloader.watched_files())
+        self.assertIn(self.existing_file, watched_files)
+        self.assertIn(inner_file, watched_files)
+
+    def test_run_loop_catches_stopiteration(self):
+        def mocked_tick():
+            yield
+
+        with mock.patch.object(self.reloader, 'tick', side_effect=mocked_tick) as tick:
+            self.reloader.run_loop()
+        self.assertEqual(tick.call_count, 1)
+
+    def test_run_loop_stop_and_return(self):
+        def mocked_tick(*args):
+            yield
+            self.reloader.stop()
+            return  # Raises StopIteration
+
+        with mock.patch.object(self.reloader, 'tick', side_effect=mocked_tick) as tick:
+            self.reloader.run_loop()
+
+        self.assertEqual(tick.call_count, 1)
+
+    def test_wait_for_apps_ready_checks_for_exception(self):
+        app_reg = Apps()
+        app_reg.ready_event.set()
+        # thread.is_alive() is False if it's not started.
+        dead_thread = threading.Thread()
+        self.assertFalse(self.reloader.wait_for_apps_ready(app_reg, dead_thread))
+
+    def test_wait_for_apps_ready_without_exception(self):
+        app_reg = Apps()
+        app_reg.ready_event.set()
+        thread = mock.MagicMock()
+        thread.is_alive.return_value = True
+        self.assertTrue(self.reloader.wait_for_apps_ready(app_reg, thread))
+
+
+def skip_unless_watchman_available():
+    try:
+        autoreload.WatchmanReloader.check_availability()
+    except WatchmanUnavailable as e:
+        return skip('Watchman unavailable: %s' % e)
+    return lambda func: func
+
+
+@skip_unless_watchman_available()
+class WatchmanReloaderTests(ReloaderTests, IntegrationTests):
+    RELOADER_CLS = autoreload.WatchmanReloader
+
+    def test_watch_glob_ignores_non_existing_directories_two_levels(self):
+        with mock.patch.object(self.reloader, '_subscribe') as mocked_subscribe:
+            self.reloader._watch_glob(self.tempdir / 'does_not_exist' / 'more', ['*'])
+        self.assertFalse(mocked_subscribe.called)
+
+    def test_watch_glob_uses_existing_parent_directories(self):
+        with mock.patch.object(self.reloader, '_subscribe') as mocked_subscribe:
+            self.reloader._watch_glob(self.tempdir / 'does_not_exist', ['*'])
+        self.assertSequenceEqual(
+            mocked_subscribe.call_args[0],
+            [
+                self.tempdir, 'glob-parent-does_not_exist:%s' % self.tempdir,
+                ['anyof', ['match', 'does_not_exist/*', 'wholename']]
+            ]
+        )
+
+    def test_watch_glob_multiple_patterns(self):
+        with mock.patch.object(self.reloader, '_subscribe') as mocked_subscribe:
+            self.reloader._watch_glob(self.tempdir, ['*', '*.py'])
+        self.assertSequenceEqual(
+            mocked_subscribe.call_args[0],
+            [
+                self.tempdir, 'glob:%s' % self.tempdir,
+                ['anyof', ['match', '*', 'wholename'], ['match', '*.py', 'wholename']]
+            ]
+        )
+
+    def test_watched_roots_contains_files(self):
+        paths = self.reloader.watched_roots([self.existing_file])
+        self.assertIn(self.existing_file.parent, paths)
+
+    def test_watched_roots_contains_directory_globs(self):
+        self.reloader.watch_dir(self.tempdir, '*.py')
+        paths = self.reloader.watched_roots([])
+        self.assertIn(self.tempdir, paths)
+
+    def test_watched_roots_contains_sys_path(self):
+        with extend_sys_path(str(self.tempdir)):
+            paths = self.reloader.watched_roots([])
+        self.assertIn(self.tempdir, paths)
+
+    def test_check_server_status(self):
+        self.assertTrue(self.reloader.check_server_status())
+
+    def test_check_server_status_raises_error(self):
+        with mock.patch.object(self.reloader.client, 'query') as mocked_query:
+            mocked_query.side_effect = Exception()
+            with self.assertRaises(autoreload.WatchmanUnavailable):
+                self.reloader.check_server_status()
+
+    @mock.patch('pywatchman.client')
+    def test_check_availability(self, mocked_client):
+        mocked_client().capabilityCheck.side_effect = Exception()
+        with self.assertRaisesMessage(WatchmanUnavailable, 'Cannot connect to the watchman service'):
+            self.RELOADER_CLS.check_availability()
+
+    @mock.patch('pywatchman.client')
+    def test_check_availability_lower_version(self, mocked_client):
+        mocked_client().capabilityCheck.return_value = {'version': '4.8.10'}
+        with self.assertRaisesMessage(WatchmanUnavailable, 'Watchman 4.9 or later is required.'):
+            self.RELOADER_CLS.check_availability()
+
+    def test_pywatchman_not_available(self):
+        with mock.patch.object(autoreload, 'pywatchman') as mocked:
+            mocked.__bool__.return_value = False
+            with self.assertRaisesMessage(WatchmanUnavailable, 'pywatchman not installed.'):
+                self.RELOADER_CLS.check_availability()
+
+    def test_update_watches_raises_exceptions(self):
+        class TestException(Exception):
+            pass
+
+        with mock.patch.object(self.reloader, '_update_watches') as mocked_watches:
+            with mock.patch.object(self.reloader, 'check_server_status') as mocked_server_status:
+                mocked_watches.side_effect = TestException()
+                mocked_server_status.return_value = True
+                with self.assertRaises(TestException):
+                    self.reloader.update_watches()
+                self.assertIsInstance(mocked_server_status.call_args[0][0], TestException)
+
+
+class StatReloaderTests(ReloaderTests, IntegrationTests):
+    RELOADER_CLS = autoreload.StatReloader
+
+    def setUp(self):
+        super().setUp()
+        # Shorten the sleep time to speed up tests.
+        self.reloader.SLEEP_TIME = 0.01
+
+    def test_snapshot_files_ignores_missing_files(self):
+        with mock.patch.object(self.reloader, 'watched_files', return_value=[self.nonexistant_file]):
+            self.assertEqual(dict(self.reloader.snapshot_files()), {})
+
+    def test_snapshot_files_updates(self):
+        with mock.patch.object(self.reloader, 'watched_files', return_value=[self.existing_file]):
+            snapshot1 = dict(self.reloader.snapshot_files())
+            self.assertIn(self.existing_file, snapshot1)
+            self.increment_mtime(self.existing_file)
+            snapshot2 = dict(self.reloader.snapshot_files())
+            self.assertNotEqual(snapshot1[self.existing_file], snapshot2[self.existing_file])
+
+    def test_does_not_fire_without_changes(self):
+        with mock.patch.object(self.reloader, 'watched_files', return_value=[self.existing_file]), \
+                mock.patch.object(self.reloader, 'notify_file_changed') as notifier:
+            mtime = self.existing_file.stat().st_mtime
+            initial_snapshot = {self.existing_file: mtime}
+            second_snapshot = self.reloader.loop_files(initial_snapshot, time.time())
+            self.assertEqual(second_snapshot, {})
+            notifier.assert_not_called()
+
+    def test_fires_when_created(self):
+        with mock.patch.object(self.reloader, 'watched_files', return_value=[self.nonexistant_file]), \
+                mock.patch.object(self.reloader, 'notify_file_changed') as notifier:
+            self.nonexistant_file.touch()
+            mtime = self.nonexistant_file.stat().st_mtime
+            second_snapshot = self.reloader.loop_files({}, mtime - 1)
+            self.assertCountEqual(second_snapshot.keys(), [self.nonexistant_file])
+            notifier.assert_called_once_with(self.nonexistant_file)
+
+    def test_fires_with_changes(self):
+        with mock.patch.object(self.reloader, 'watched_files', return_value=[self.existing_file]), \
+                mock.patch.object(self.reloader, 'notify_file_changed') as notifier:
+            initial_snapshot = {self.existing_file: 1}
+            second_snapshot = self.reloader.loop_files(initial_snapshot, time.time())
+            notifier.assert_called_once_with(self.existing_file)
+            self.assertCountEqual(second_snapshot.keys(), [self.existing_file])