django/tests/test_runner/tests.py

"""
Tests for django test runner
"""
import collections.abc
import unittest
from unittest import mock

from admin_scripts.tests import AdminScriptTestCase

from django import db
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.management import call_command
from django.core.management.base import SystemCheckError
from django.test import (
    SimpleTestCase, TransactionTestCase, skipUnlessDBFeature,
)
from django.test.runner import (
    DiscoverRunner, Shuffler, reorder_test_bin, reorder_tests, shuffle_tests,
)
from django.test.testcases import connections_support_transactions
from django.test.utils import (
    captured_stderr, dependency_ordered, get_unique_databases_and_mirrors,
    iter_test_cases,
)
from django.utils.deprecation import RemovedInDjango50Warning

from .models import B, Person, Through


class MySuite:
    def __init__(self):
        self.tests = []

    def addTest(self, test):
        self.tests.append(test)

    def __iter__(self):
        yield from self.tests


class TestSuiteTests(SimpleTestCase):
    def build_test_suite(self, test_classes, suite=None, suite_class=None):
        if suite_class is None:
            suite_class = unittest.TestSuite
        if suite is None:
            suite = suite_class()

        loader = unittest.defaultTestLoader
        for test_class in test_classes:
            tests = loader.loadTestsFromTestCase(test_class)
            subsuite = suite_class()
            # Only use addTest() to simplify testing a custom TestSuite.
            for test in tests:
                subsuite.addTest(test)
            suite.addTest(subsuite)

        return suite

    def make_test_suite(self, suite=None, suite_class=None):
        class Tests1(unittest.TestCase):
            def test1(self):
                pass

            def test2(self):
                pass

        class Tests2(unittest.TestCase):
            def test1(self):
                pass

            def test2(self):
                pass

        return self.build_test_suite(
            (Tests1, Tests2),
            suite=suite,
            suite_class=suite_class,
        )

    def assertTestNames(self, tests, expected):
        # Each test.id() has a form like the following:
        # "test_runner.tests.IterTestCasesTests.test_iter_test_cases.<locals>.Tests1.test1".
        # It suffices to check only the last two parts.
        names = ['.'.join(test.id().split('.')[-2:]) for test in tests]
        self.assertEqual(names, expected)

    def test_iter_test_cases_basic(self):
        suite = self.make_test_suite()
        tests = iter_test_cases(suite)
        self.assertTestNames(tests, expected=[
            'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
        ])

    def test_iter_test_cases_string_input(self):
        msg = (
            "Test 'a' must be a test case or test suite not string (was found "
            "in 'abc')."
        )
        with self.assertRaisesMessage(TypeError, msg):
            list(iter_test_cases('abc'))

    def test_iter_test_cases_iterable_of_tests(self):
        class Tests(unittest.TestCase):
            def test1(self):
                pass

            def test2(self):
                pass

        tests = list(unittest.defaultTestLoader.loadTestsFromTestCase(Tests))
        actual_tests = iter_test_cases(tests)
        self.assertTestNames(actual_tests, expected=[
            'Tests.test1', 'Tests.test2',
        ])

    def test_iter_test_cases_custom_test_suite_class(self):
        suite = self.make_test_suite(suite_class=MySuite)
        tests = iter_test_cases(suite)
        self.assertTestNames(tests, expected=[
            'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
        ])

    def test_iter_test_cases_mixed_test_suite_classes(self):
        suite = self.make_test_suite(suite=MySuite())
        child_suite = list(suite)[0]
        self.assertNotIsInstance(child_suite, MySuite)
        tests = list(iter_test_cases(suite))
        self.assertEqual(len(tests), 4)
        self.assertNotIsInstance(tests[0], unittest.TestSuite)

    def make_tests(self):
        """Return an iterable of tests."""
        suite = self.make_test_suite()
        tests = list(iter_test_cases(suite))
        return tests

    def test_shuffle_tests(self):
        tests = self.make_tests()
        # Choose a seed that shuffles both the classes and methods.
        shuffler = Shuffler(seed=9)
        shuffled_tests = shuffle_tests(tests, shuffler)
        self.assertIsInstance(shuffled_tests, collections.abc.Iterator)
        self.assertTestNames(shuffled_tests, expected=[
            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
        ])

    def test_reorder_test_bin_no_arguments(self):
        tests = self.make_tests()
        reordered_tests = reorder_test_bin(tests)
        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
        self.assertTestNames(reordered_tests, expected=[
            'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
        ])

    def test_reorder_test_bin_reverse(self):
        tests = self.make_tests()
        reordered_tests = reorder_test_bin(tests, reverse=True)
        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
        self.assertTestNames(reordered_tests, expected=[
            'Tests2.test2', 'Tests2.test1', 'Tests1.test2', 'Tests1.test1',
        ])

    def test_reorder_test_bin_random(self):
        tests = self.make_tests()
        # Choose a seed that shuffles both the classes and methods.
        shuffler = Shuffler(seed=9)
        reordered_tests = reorder_test_bin(tests, shuffler=shuffler)
        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
        self.assertTestNames(reordered_tests, expected=[
            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
        ])

    def test_reorder_test_bin_random_and_reverse(self):
        tests = self.make_tests()
        # Choose a seed that shuffles both the classes and methods.
        shuffler = Shuffler(seed=9)
        reordered_tests = reorder_test_bin(tests, shuffler=shuffler, reverse=True)
        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
        self.assertTestNames(reordered_tests, expected=[
            'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1',
        ])

    def test_reorder_tests_same_type_consecutive(self):
        """Tests of the same type are made consecutive."""
        tests = self.make_tests()
        # Move the last item to the front.
        tests.insert(0, tests.pop())
        self.assertTestNames(tests, expected=[
            'Tests2.test2', 'Tests1.test1', 'Tests1.test2', 'Tests2.test1',
        ])
        reordered_tests = reorder_tests(tests, classes=[])
        self.assertTestNames(reordered_tests, expected=[
            'Tests2.test2', 'Tests2.test1', 'Tests1.test1', 'Tests1.test2',
        ])

    def test_reorder_tests_random(self):
        tests = self.make_tests()
        # Choose a seed that shuffles both the classes and methods.
        shuffler = Shuffler(seed=9)
        reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
        self.assertTestNames(reordered_tests, expected=[
            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
        ])

    def test_reorder_tests_random_mixed_classes(self):
        tests = self.make_tests()
        # Move the last item to the front.
        tests.insert(0, tests.pop())
        shuffler = Shuffler(seed=9)
        self.assertTestNames(tests, expected=[
            'Tests2.test2', 'Tests1.test1', 'Tests1.test2', 'Tests2.test1',
        ])
        reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
        self.assertTestNames(reordered_tests, expected=[
            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
        ])

    def test_reorder_tests_reverse_with_duplicates(self):
        class Tests1(unittest.TestCase):
            def test1(self):
                pass

        class Tests2(unittest.TestCase):
            def test2(self):
                pass

            def test3(self):
                pass

        suite = self.build_test_suite((Tests1, Tests2))
        subsuite = list(suite)[0]
        suite.addTest(subsuite)
        tests = list(iter_test_cases(suite))
        self.assertTestNames(tests, expected=[
            'Tests1.test1', 'Tests2.test2', 'Tests2.test3', 'Tests1.test1',
        ])
        reordered_tests = reorder_tests(tests, classes=[])
        self.assertTestNames(reordered_tests, expected=[
            'Tests1.test1', 'Tests2.test2', 'Tests2.test3',
        ])
        reordered_tests = reorder_tests(tests, classes=[], reverse=True)
        self.assertTestNames(reordered_tests, expected=[
            'Tests2.test3', 'Tests2.test2', 'Tests1.test1',
        ])


class DependencyOrderingTests(unittest.TestCase):

    def test_simple_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
            ('s3', ('s3_db', ['charlie'])),
        ]
        dependencies = {
            'alpha': ['charlie'],
            'bravo': ['charlie'],
        }

        ordered = dependency_ordered(raw, dependencies=dependencies)
        ordered_sigs = [sig for sig, value in ordered]

        self.assertIn('s1', ordered_sigs)
        self.assertIn('s2', ordered_sigs)
        self.assertIn('s3', ordered_sigs)
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))

    def test_chained_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
            ('s3', ('s3_db', ['charlie'])),
        ]
        dependencies = {
            'alpha': ['bravo'],
            'bravo': ['charlie'],
        }

        ordered = dependency_ordered(raw, dependencies=dependencies)
        ordered_sigs = [sig for sig, value in ordered]

        self.assertIn('s1', ordered_sigs)
        self.assertIn('s2', ordered_sigs)
        self.assertIn('s3', ordered_sigs)

        # Explicit dependencies
        self.assertLess(ordered_sigs.index('s2'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))

        # Implied dependencies
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))

    def test_multiple_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
            ('s3', ('s3_db', ['charlie'])),
            ('s4', ('s4_db', ['delta'])),
        ]
        dependencies = {
            'alpha': ['bravo', 'delta'],
            'bravo': ['charlie'],
            'delta': ['charlie'],
        }

        ordered = dependency_ordered(raw, dependencies=dependencies)
        ordered_sigs = [sig for sig, aliases in ordered]

        self.assertIn('s1', ordered_sigs)
        self.assertIn('s2', ordered_sigs)
        self.assertIn('s3', ordered_sigs)
        self.assertIn('s4', ordered_sigs)

        # Explicit dependencies
        self.assertLess(ordered_sigs.index('s2'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s4'), ordered_sigs.index('s1'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s4'))

        # Implicit dependencies
        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))

    def test_circular_dependencies(self):
        raw = [
            ('s1', ('s1_db', ['alpha'])),
            ('s2', ('s2_db', ['bravo'])),
        ]
        dependencies = {
            'bravo': ['alpha'],
            'alpha': ['bravo'],
        }

        with self.assertRaises(ImproperlyConfigured):
            dependency_ordered(raw, dependencies=dependencies)

    def test_own_alias_dependency(self):
        raw = [
            ('s1', ('s1_db', ['alpha', 'bravo']))
        ]
        dependencies = {
            'alpha': ['bravo']
        }

        with self.assertRaises(ImproperlyConfigured):
            dependency_ordered(raw, dependencies=dependencies)

        # reordering aliases shouldn't matter
        raw = [
            ('s1', ('s1_db', ['bravo', 'alpha']))
        ]

        with self.assertRaises(ImproperlyConfigured):
            dependency_ordered(raw, dependencies=dependencies)


class MockTestRunner:
    def __init__(self, *args, **kwargs):
        pass


MockTestRunner.run_tests = mock.Mock(return_value=[])


class ManageCommandTests(unittest.TestCase):

    def test_custom_test_runner(self):
        call_command('test', 'sites',
                     testrunner='test_runner.tests.MockTestRunner')
        MockTestRunner.run_tests.assert_called_with(('sites',))

    def test_bad_test_runner(self):
        with self.assertRaises(AttributeError):
            call_command('test', 'sites', testrunner='test_runner.NonexistentRunner')

    def test_time_recorded(self):
        with captured_stderr() as stderr:
            call_command('test', '--timing', 'sites', testrunner='test_runner.tests.MockTestRunner')
        self.assertIn('Total run took', stderr.getvalue())


class CustomTestRunnerOptionsSettingsTests(AdminScriptTestCase):
    """
    Custom runners can add command line arguments. The runner is specified
    through a settings file.
    """
    def setUp(self):
        super().setUp()
        settings = {
            'TEST_RUNNER': '\'test_runner.runner.CustomOptionsTestRunner\'',
        }
        self.write_settings('settings.py', sdict=settings)

    def test_default_options(self):
        args = ['test', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '1:2:3')

    def test_default_and_given_options(self):
        args = ['test', '--settings=test_project.settings', '--option_b=foo']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '1:foo:3')

    def test_option_name_and_value_separated(self):
        args = ['test', '--settings=test_project.settings', '--option_b', 'foo']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, '1:foo:3')

    def test_all_options_given(self):
        args = ['test', '--settings=test_project.settings', '--option_a=bar',
                '--option_b=foo', '--option_c=31337']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, 'bar:foo:31337')


class CustomTestRunnerOptionsCmdlineTests(AdminScriptTestCase):
    """
    Custom runners can add command line arguments when the runner is specified
    using --testrunner.
    """
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_testrunner_option(self):
        args = [
            'test', '--testrunner', 'test_runner.runner.CustomOptionsTestRunner',
            '--option_a=bar', '--option_b=foo', '--option_c=31337'
        ]
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'bar:foo:31337')

    def test_testrunner_equals(self):
        args = [
            'test', '--testrunner=test_runner.runner.CustomOptionsTestRunner',
            '--option_a=bar', '--option_b=foo', '--option_c=31337'
        ]
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, 'bar:foo:31337')

    def test_no_testrunner(self):
        args = ['test', '--testrunner']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertIn('usage', err)
        self.assertNotIn('Traceback', err)
        self.assertNoOutput(out)


class Ticket17477RegressionTests(AdminScriptTestCase):
    def setUp(self):
        super().setUp()
        self.write_settings('settings.py')

    def test_ticket_17477(self):
        """'manage.py help test' works after r16352."""
        args = ['help', 'test']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)


class SQLiteInMemoryTestDbs(TransactionTestCase):
    available_apps = ['test_runner']
    databases = {'default', 'other'}

    @unittest.skipUnless(all(db.connections[conn].vendor == 'sqlite' for conn in db.connections),
                         "This is an sqlite-specific issue")
    def test_transaction_support(self):
        # Assert connections mocking is appropriately applied by preventing
        # any attempts at calling create_test_db on the global connection
        # objects.
        for connection in db.connections.all():
            create_test_db = mock.patch.object(
                connection.creation,
                'create_test_db',
                side_effect=AssertionError("Global connection object shouldn't be manipulated.")
            )
            create_test_db.start()
            self.addCleanup(create_test_db.stop)
        for option_key, option_value in (
                ('NAME', ':memory:'), ('TEST', {'NAME': ':memory:'})):
            tested_connections = db.ConnectionHandler({
                'default': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    option_key: option_value,
                },
                'other': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    option_key: option_value,
                },
            })
            with mock.patch('django.test.utils.connections', new=tested_connections):
                other = tested_connections['other']
                DiscoverRunner(verbosity=0).setup_databases()
                msg = (
                    "DATABASES setting '%s' option set to sqlite3's ':memory:' value "
                    "shouldn't interfere with transaction support detection." % option_key
                )
                # Transaction support is properly initialized for the 'other' DB.
                self.assertTrue(other.features.supports_transactions, msg)
                # And all the DBs report that they support transactions.
                self.assertTrue(connections_support_transactions(), msg)


class DummyBackendTest(unittest.TestCase):
    def test_setup_databases(self):
        """
        setup_databases() doesn't fail with dummy database backend.
        """
        tested_connections = db.ConnectionHandler({})
        with mock.patch('django.test.utils.connections', new=tested_connections):
            runner_instance = DiscoverRunner(verbosity=0)
            old_config = runner_instance.setup_databases()
            runner_instance.teardown_databases(old_config)


class AliasedDefaultTestSetupTest(unittest.TestCase):
    def test_setup_aliased_default_database(self):
        """
        setup_databases() doesn't fail when 'default' is aliased
        """
        tested_connections = db.ConnectionHandler({
            'default': {
                'NAME': 'dummy'
            },
            'aliased': {
                'NAME': 'dummy'
            }
        })
        with mock.patch('django.test.utils.connections', new=tested_connections):
            runner_instance = DiscoverRunner(verbosity=0)
            old_config = runner_instance.setup_databases()
            runner_instance.teardown_databases(old_config)


class SetupDatabasesTests(SimpleTestCase):

    def setUp(self):
        self.runner_instance = DiscoverRunner(verbosity=0)

    def test_setup_aliased_databases(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': 'django.db.backends.dummy',
                'NAME': 'dbname',
            },
            'other': {
                'ENGINE': 'django.db.backends.dummy',
                'NAME': 'dbname',
            }
        })

        with mock.patch('django.db.backends.dummy.base.DatabaseWrapper.creation_class') as mocked_db_creation:
            with mock.patch('django.test.utils.connections', new=tested_connections):
                old_config = self.runner_instance.setup_databases()
                self.runner_instance.teardown_databases(old_config)
        mocked_db_creation.return_value.destroy_test_db.assert_called_once_with('dbname', 0, False)

    def test_setup_test_database_aliases(self):
        """
        The default database must be the first because data migrations
        use the default alias by default.
        """
        tested_connections = db.ConnectionHandler({
            'other': {
                'ENGINE': 'django.db.backends.dummy',
                'NAME': 'dbname',
            },
            'default': {
                'ENGINE': 'django.db.backends.dummy',
                'NAME': 'dbname',
            }
        })
        with mock.patch('django.test.utils.connections', new=tested_connections):
            test_databases, _ = get_unique_databases_and_mirrors()
            self.assertEqual(
                test_databases,
                {
                    ('', '', 'django.db.backends.dummy', 'test_dbname'): (
                        'dbname',
                        ['default', 'other'],
                    ),
                },
            )

    def test_destroy_test_db_restores_db_name(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': settings.DATABASES[db.DEFAULT_DB_ALIAS]["ENGINE"],
                'NAME': 'xxx_test_database',
            },
        })
        # Using the real current name as old_name to not mess with the test suite.
        old_name = settings.DATABASES[db.DEFAULT_DB_ALIAS]["NAME"]
        with mock.patch('django.db.connections', new=tested_connections):
            tested_connections['default'].creation.destroy_test_db(old_name, verbosity=0, keepdb=True)
            self.assertEqual(tested_connections['default'].settings_dict["NAME"], old_name)

    def test_serialization(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': 'django.db.backends.dummy',
            },
        })
        with mock.patch('django.db.backends.dummy.base.DatabaseWrapper.creation_class') as mocked_db_creation:
            with mock.patch('django.test.utils.connections', new=tested_connections):
                self.runner_instance.setup_databases()
        mocked_db_creation.return_value.create_test_db.assert_called_once_with(
            verbosity=0, autoclobber=False, serialize=True, keepdb=False
        )

    def test_serialized_off(self):
        tested_connections = db.ConnectionHandler({
            'default': {
                'ENGINE': 'django.db.backends.dummy',
                'TEST': {'SERIALIZE': False},
            },
        })
        msg = (
            'The SERIALIZE test database setting is deprecated as it can be '
            'inferred from the TestCase/TransactionTestCase.databases that '
            'enable the serialized_rollback feature.'
        )
        with mock.patch('django.db.backends.dummy.base.DatabaseWrapper.creation_class') as mocked_db_creation:
            with mock.patch('django.test.utils.connections', new=tested_connections):
                with self.assertWarnsMessage(RemovedInDjango50Warning, msg):
                    self.runner_instance.setup_databases()
        mocked_db_creation.return_value.create_test_db.assert_called_once_with(
            verbosity=0, autoclobber=False, serialize=False, keepdb=False
        )


@skipUnlessDBFeature('supports_sequence_reset')
class AutoIncrementResetTest(TransactionTestCase):
    """
    Creating the same models in different test methods receive the same PK
    values since the sequences are reset before each test method.
    """

    available_apps = ['test_runner']

    reset_sequences = True

    def _test(self):
        # Regular model
        p = Person.objects.create(first_name='Jack', last_name='Smith')
        self.assertEqual(p.pk, 1)
        # Auto-created many-to-many through model
        p.friends.add(Person.objects.create(first_name='Jacky', last_name='Smith'))
        self.assertEqual(p.friends.through.objects.first().pk, 1)
        # Many-to-many through model
        b = B.objects.create()
        t = Through.objects.create(person=p, b=b)
        self.assertEqual(t.pk, 1)

    def test_autoincrement_reset1(self):
        self._test()

    def test_autoincrement_reset2(self):
        self._test()


class EmptyDefaultDatabaseTest(unittest.TestCase):
    def test_empty_default_database(self):
        """
        An empty default database in settings does not raise an ImproperlyConfigured
        error when running a unit test that does not use a database.
        """
        tested_connections = db.ConnectionHandler({'default': {}})
        with mock.patch('django.db.connections', new=tested_connections):
            connection = tested_connections[db.utils.DEFAULT_DB_ALIAS]
            self.assertEqual(connection.settings_dict['ENGINE'], 'django.db.backends.dummy')
            connections_support_transactions()


class RunTestsExceptionHandlingTests(unittest.TestCase):
    def test_run_checks_raises(self):
        """
        Teardown functions are run when run_checks() raises SystemCheckError.
        """
        with mock.patch('django.test.runner.DiscoverRunner.setup_test_environment'), \
                mock.patch('django.test.runner.DiscoverRunner.setup_databases'), \
                mock.patch('django.test.runner.DiscoverRunner.build_suite'), \
                mock.patch('django.test.runner.DiscoverRunner.run_checks', side_effect=SystemCheckError), \
                mock.patch('django.test.runner.DiscoverRunner.teardown_databases') as teardown_databases, \
                mock.patch('django.test.runner.DiscoverRunner.teardown_test_environment') as teardown_test_environment:
            runner = DiscoverRunner(verbosity=0, interactive=False)
            with self.assertRaises(SystemCheckError):
                runner.run_tests(['test_runner_apps.sample.tests_sample.TestDjangoTestCase'])
            self.assertTrue(teardown_databases.called)
            self.assertTrue(teardown_test_environment.called)

    def test_run_checks_raises_and_teardown_raises(self):
        """
        SystemCheckError is surfaced when run_checks() raises SystemCheckError
        and teardown databases() raises ValueError.
        """
        with mock.patch('django.test.runner.DiscoverRunner.setup_test_environment'), \
                mock.patch('django.test.runner.DiscoverRunner.setup_databases'), \
                mock.patch('django.test.runner.DiscoverRunner.build_suite'), \
                mock.patch('django.test.runner.DiscoverRunner.run_checks', side_effect=SystemCheckError), \
                mock.patch('django.test.runner.DiscoverRunner.teardown_databases', side_effect=ValueError) \
                as teardown_databases, \
                mock.patch('django.test.runner.DiscoverRunner.teardown_test_environment') as teardown_test_environment:
            runner = DiscoverRunner(verbosity=0, interactive=False)
            with self.assertRaises(SystemCheckError):
                runner.run_tests(['test_runner_apps.sample.tests_sample.TestDjangoTestCase'])
            self.assertTrue(teardown_databases.called)
            self.assertFalse(teardown_test_environment.called)

    def test_run_checks_passes_and_teardown_raises(self):
        """
        Exceptions on teardown are surfaced if no exceptions happen during
        run_checks().
        """
        with mock.patch('django.test.runner.DiscoverRunner.setup_test_environment'), \
                mock.patch('django.test.runner.DiscoverRunner.setup_databases'), \
                mock.patch('django.test.runner.DiscoverRunner.build_suite'), \
                mock.patch('django.test.runner.DiscoverRunner.run_checks'), \
                mock.patch('django.test.runner.DiscoverRunner.teardown_databases', side_effect=ValueError) \
                as teardown_databases, \
                mock.patch('django.test.runner.DiscoverRunner.teardown_test_environment') as teardown_test_environment:
            runner = DiscoverRunner(verbosity=0, interactive=False)
            with self.assertRaises(ValueError):
                # Suppress the output when running TestDjangoTestCase.
                with mock.patch('sys.stderr'):
                    runner.run_tests(['test_runner_apps.sample.tests_sample.TestDjangoTestCase'])
            self.assertTrue(teardown_databases.called)
            self.assertFalse(teardown_test_environment.called)


# RemovedInDjango50Warning
class NoOpTestRunner(DiscoverRunner):
    def setup_test_environment(self, **kwargs):
        return

    def setup_databases(self, **kwargs):
        return

    def run_checks(self, databases):
        return

    def teardown_databases(self, old_config, **kwargs):
        return

    def teardown_test_environment(self, **kwargs):
        return


class DiscoverRunnerExtraTestsDeprecationTests(SimpleTestCase):
    msg = 'The extra_tests argument is deprecated.'

    def get_runner(self):
        return NoOpTestRunner(verbosity=0, interactive=False)

    def test_extra_tests_build_suite(self):
        runner = self.get_runner()
        with self.assertWarnsMessage(RemovedInDjango50Warning, self.msg):
            runner.build_suite(extra_tests=[])

    def test_extra_tests_run_tests(self):
        runner = self.get_runner()
        with captured_stderr():
            with self.assertWarnsMessage(RemovedInDjango50Warning, self.msg):
                runner.run_tests(
                    test_labels=['test_runner_apps.sample.tests_sample.EmptyTestCase'],
                    extra_tests=[],
                )