diff --git a/django/conf/__init__.py b/django/conf/__init__.py
index 2f8fba14dd..efc9115b65 100644
--- a/django/conf/__init__.py
+++ b/django/conf/__init__.py
@@ -168,7 +168,10 @@ class UserSettingsHolder(BaseSettings):
             super(UserSettingsHolder, self).__delattr__(name)
 
     def __dir__(self):
-        return list(self.__dict__) + dir(self.default_settings)
+        return sorted(
+            s for s in list(self.__dict__) + dir(self.default_settings)
+            if s not in self._deleted
+        )
 
     def is_overridden(self, setting):
         deleted = (setting in self._deleted)
diff --git a/tests/settings_tests/tests.py b/tests/settings_tests/tests.py
index 8cee706d5b..e5e034115e 100644
--- a/tests/settings_tests/tests.py
+++ b/tests/settings_tests/tests.py
@@ -256,6 +256,8 @@ class SettingsTests(SimpleTestCase):
             del settings.USE_L10N
             with self.assertRaises(AttributeError):
                 getattr(settings, 'USE_L10N')
+            self.assertNotIn('USE_I18N', dir(settings))
+            self.assertNotIn('USE_L10N', dir(settings))
         self.assertEqual(settings.USE_I18N, previous_i18n)
         self.assertEqual(settings.USE_L10N, previous_l10n)