diff --git a/django/contrib/comments/__init__.py b/django/contrib/comments/__init__.py
index 42384e786b6..1798c1adb5d 100644
--- a/django/contrib/comments/__init__.py
+++ b/django/contrib/comments/__init__.py
@@ -20,9 +20,9 @@ def get_comment_app():
     # Try to import the package
     try:
         package = import_module(comments_app)
-    except ImportError:
+    except ImportError as e:
         raise ImproperlyConfigured("The COMMENTS_APP setting refers to "\
-                                   "a non-existing package.")
+                                   "a non-existing package. (%s)" % e)
 
     return package
 
diff --git a/tests/regressiontests/comment_tests/tests/app_api_tests.py b/tests/regressiontests/comment_tests/tests/app_api_tests.py
index 8a63e8c188e..a756068790b 100644
--- a/tests/regressiontests/comment_tests/tests/app_api_tests.py
+++ b/tests/regressiontests/comment_tests/tests/app_api_tests.py
@@ -4,6 +4,9 @@ from django.conf import settings
 from django.contrib import comments
 from django.contrib.comments.models import Comment
 from django.contrib.comments.forms import CommentForm
+from django.core.exceptions import ImproperlyConfigured
+from django.test.utils import override_settings
+from django.utils import six
 
 from . import CommentTestCase
 
@@ -14,6 +17,14 @@ class CommentAppAPITests(CommentTestCase):
     def testGetCommentApp(self):
         self.assertEqual(comments.get_comment_app(), comments)
 
+    @override_settings(
+        COMMENTS_APP='missing_app',
+        INSTALLED_APPS=list(settings.INSTALLED_APPS) + ['missing_app'],
+    )
+    def testGetMissingCommentApp(self):
+        with six.assertRaisesRegex(self, ImproperlyConfigured, 'missing_app'):
+            _ = comments.get_comment_app()
+
     def testGetForm(self):
         self.assertEqual(comments.get_form(), CommentForm)
 
@@ -33,20 +44,14 @@ class CommentAppAPITests(CommentTestCase):
         self.assertEqual(comments.get_approve_url(c), "/approve/12345/")
 
 
+@override_settings(
+    COMMENTS_APP='regressiontests.comment_tests.custom_comments',
+    INSTALLED_APPS=list(settings.INSTALLED_APPS) + [
+        'regressiontests.comment_tests.custom_comments'],
+)
 class CustomCommentTest(CommentTestCase):
     urls = 'regressiontests.comment_tests.urls'
 
-    def setUp(self):
-        self.old_comments_app = getattr(settings, 'COMMENTS_APP', None)
-        settings.COMMENTS_APP = 'regressiontests.comment_tests.custom_comments'
-        settings.INSTALLED_APPS = list(settings.INSTALLED_APPS) + [settings.COMMENTS_APP,]
-
-    def tearDown(self):
-        del settings.INSTALLED_APPS[-1]
-        settings.COMMENTS_APP = self.old_comments_app
-        if settings.COMMENTS_APP is None:
-            del settings._wrapped.COMMENTS_APP
-
     def testGetCommentApp(self):
         from regressiontests.comment_tests import custom_comments
         self.assertEqual(comments.get_comment_app(), custom_comments)