diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py
index 70117f03274..dc82c24ecfb 100644
--- a/tests/test_runner/test_discover_runner.py
+++ b/tests/test_runner/test_discover_runner.py
@@ -121,9 +121,11 @@ class DiscoverRunnerTest(TestCase):
         """
         Tests shouldn't be discovered twice when discovering on overlapping paths.
         """
-        single = DiscoverRunner().build_suite(["gis_tests"]).countTestCases()
-        dups = DiscoverRunner().build_suite(
-            ["gis_tests", "gis_tests.geo3d"]).countTestCases()
+        base_app = 'gis_tests'
+        sub_app = 'gis_tests.geo3d'
+        with self.modify_settings(INSTALLED_APPS={'append': sub_app}):
+            single = DiscoverRunner().build_suite([base_app]).countTestCases()
+            dups = DiscoverRunner().build_suite([base_app, sub_app]).countTestCases()
         self.assertEqual(single, dups)
 
     def test_reverse(self):