diff --git a/django/utils/module_loading.py b/django/utils/module_loading.py
index 826aed8c000..cd0c8bc5cc8 100644
--- a/django/utils/module_loading.py
+++ b/django/utils/module_loading.py
@@ -64,26 +64,26 @@ def autodiscover_modules(*args, **kwargs):
 
     register_to = kwargs.get('register_to')
     for app_config in apps.get_app_configs():
-        # Attempt to import the app's module.
-        try:
-            if register_to:
-                before_import_registry = copy.copy(register_to._registry)
+        for module_to_search in args:
+            # Attempt to import the app's module.
+            try:
+                if register_to:
+                    before_import_registry = copy.copy(register_to._registry)
 
-            for module_to_search in args:
                 import_module('%s.%s' % (app_config.name, module_to_search))
-        except:
-            # Reset the model registry to the state before the last import as
-            # this import will have to reoccur on the next request and this
-            # could raise NotRegistered and AlreadyRegistered exceptions
-            # (see #8245).
-            if register_to:
-                register_to._registry = before_import_registry
+            except:
+                # Reset the registry to the state before the last import
+                # as this import will have to reoccur on the next request and
+                # this could raise NotRegistered and AlreadyRegistered
+                # exceptions (see #8245).
+                if register_to:
+                    register_to._registry = before_import_registry
 
-            # Decide whether to bubble up this error. If the app just
-            # doesn't have an admin module, we can ignore the error
-            # attempting to import it, otherwise we want it to bubble up.
-            if module_has_submodule(app_config.module, module_to_search):
-                raise
+                # Decide whether to bubble up this error. If the app just
+                # doesn't have the module in question, we can ignore the error
+                # attempting to import it, otherwise we want it to bubble up.
+                if module_has_submodule(app_config.module, module_to_search):
+                    raise
 
 
 if sys.version_info[:2] >= (3, 3):
diff --git a/tests/utils_tests/test_module/another_good_module.py b/tests/utils_tests/test_module/another_good_module.py
index ed4b2d34a28..fd69a3b30cb 100644
--- a/tests/utils_tests/test_module/another_good_module.py
+++ b/tests/utils_tests/test_module/another_good_module.py
@@ -1 +1,6 @@
+from . import site
 content = 'Another Good Module'
+
+site._registry.update({
+    'lorem': 'ipsum',
+})
diff --git a/tests/utils_tests/test_module_loading.py b/tests/utils_tests/test_module_loading.py
index 9cdaf8df09e..31608bfcc1e 100644
--- a/tests/utils_tests/test_module_loading.py
+++ b/tests/utils_tests/test_module_loading.py
@@ -155,6 +155,15 @@ class ModuleImportTestCase(IgnoreDeprecationWarningsMixin, unittest.TestCase):
 @modify_settings(INSTALLED_APPS={'append': 'utils_tests.test_module'})
 class AutodiscoverModulesTestCase(SimpleTestCase):
 
+    def tearDown(self):
+        sys.path_importer_cache.clear()
+
+        sys.modules.pop('utils_tests.test_module.another_bad_module', None)
+        sys.modules.pop('utils_tests.test_module.another_good_module', None)
+        sys.modules.pop('utils_tests.test_module.bad_module', None)
+        sys.modules.pop('utils_tests.test_module.good_module', None)
+        sys.modules.pop('utils_tests.test_module', None)
+
     def test_autodiscover_modules_found(self):
         autodiscover_modules('good_module')
 
@@ -172,12 +181,28 @@ class AutodiscoverModulesTestCase(SimpleTestCase):
     def test_autodiscover_modules_several_found(self):
         autodiscover_modules('good_module', 'another_good_module')
 
+    def test_autodiscover_modules_several_found_with_registry(self):
+        from .test_module import site
+        autodiscover_modules('good_module', 'another_good_module', register_to=site)
+        self.assertEqual(site._registry, {'lorem': 'ipsum'})
+
     def test_validate_registry_keeps_intact(self):
         from .test_module import site
         with six.assertRaisesRegex(self, Exception, "Some random exception."):
             autodiscover_modules('another_bad_module', register_to=site)
         self.assertEqual(site._registry, {})
 
+    def test_validate_registry_resets_after_erroneous_module(self):
+        from .test_module import site
+        with six.assertRaisesRegex(self, Exception, "Some random exception."):
+            autodiscover_modules('another_good_module', 'another_bad_module', register_to=site)
+        self.assertEqual(site._registry, {'lorem': 'ipsum'})
+
+    def test_validate_registry_resets_after_missing_module(self):
+        from .test_module import site
+        autodiscover_modules('does_not_exist', 'another_good_module', 'does_not_exist2', register_to=site)
+        self.assertEqual(site._registry, {'lorem': 'ipsum'})
+
 
 class ProxyFinder(object):
     def __init__(self):