diff --git a/django/db/utils.py b/django/db/utils.py
index b114577a7f..b884cc57b1 100644
--- a/django/db/utils.py
+++ b/django/db/utils.py
@@ -209,6 +209,9 @@ class ConnectionHandler(object):
                               "to a %s entry in the TEST setting" % (key, new_key),
                               RemovedInDjango19Warning, stacklevel=2)
                 test_settings[new_key] = value
+        for key in conn.keys():
+            if key.startswith('TEST_'):
+                del conn[key]
         # Check that they didn't just use the old name with 'TEST_' removed
         for key, new_key in six.iteritems(self.TEST_SETTING_RENAMES):
             if key in test_settings: