diff --git a/tests/runtests.py b/tests/runtests.py index a52152b7e5..516ec012cd 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os, sys, traceback +import os, subprocess, sys, traceback import unittest import django.contrib as contrib @@ -85,16 +85,17 @@ class InvalidModelTestCase(unittest.TestCase): self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected)) self.assert_(not missing, "Missing Errors: " + '\n'.join(missing)) -def django_tests(verbosity, interactive, failfast, test_labels): +def setup(verbosity, test_labels): from django.conf import settings - - old_installed_apps = settings.INSTALLED_APPS - old_root_urlconf = getattr(settings, "ROOT_URLCONF", "") - old_template_dirs = settings.TEMPLATE_DIRS - old_use_i18n = settings.USE_I18N - old_login_url = settings.LOGIN_URL - old_language_code = settings.LANGUAGE_CODE - old_middleware_classes = settings.MIDDLEWARE_CLASSES + state = { + 'INSTALLED_APPS': settings.INSTALLED_APPS, + 'ROOT_URLCONF': getattr(settings, "ROOT_URLCONF", ""), + 'TEMPLATE_DIRS': settings.TEMPLATE_DIRS, + 'USE_I18N': settings.USE_I18N, + 'LOGIN_URL': settings.LOGIN_URL, + 'LANGUAGE_CODE': settings.LANGUAGE_CODE, + 'MIDDLEWARE_CLASSES': settings.MIDDLEWARE_CLASSES, + } # Redirect some settings for the duration of these tests. settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS @@ -136,6 +137,18 @@ def django_tests(verbosity, interactive, failfast, test_labels): if model_label not in settings.INSTALLED_APPS: settings.INSTALLED_APPS.append(model_label) + return state + +def teardown(state): + from django.conf import settings + # Restore the old settings. + for key, value in state.items(): + setattr(settings, key, value) + +def django_tests(verbosity, interactive, failfast, test_labels): + from django.conf import settings + state = setup(verbosity, test_labels) + # Add tests for invalid models. extra_tests = [] for model_dir, model_name in get_invalid_models(): @@ -169,17 +182,105 @@ def django_tests(verbosity, interactive, failfast, test_labels): test_runner = TestRunner(verbosity=verbosity, interactive=interactive, failfast=failfast) failures = test_runner.run_tests(test_labels, extra_tests=extra_tests) - if failures: - sys.exit(bool(failures)) + teardown(state) + return failures - # Restore the old settings. - settings.INSTALLED_APPS = old_installed_apps - settings.ROOT_URLCONF = old_root_urlconf - settings.TEMPLATE_DIRS = old_template_dirs - settings.USE_I18N = old_use_i18n - settings.LANGUAGE_CODE = old_language_code - settings.LOGIN_URL = old_login_url - settings.MIDDLEWARE_CLASSES = old_middleware_classes + +def bisect_tests(bisection_label, options, test_labels): + state = setup(int(options.verbosity), test_labels) + + if not test_labels: + # Get the full list of test labels to use for bisection + from django.db.models.loading import get_apps + test_labels = [app.__name__.split('.')[-2] for app in get_apps()] + + print '***** Bisecting test suite:',' '.join(test_labels) + + # Make sure the bisection point isn't in the test list + # Also remove tests that need to be run in specific combinations + for label in [bisection_label, 'model_inheritance_same_model_name']: + try: + test_labels.remove(label) + except ValueError: + pass + + subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings] + if options.failfast: + subprocess_args.append('--failfast') + if options.verbosity: + subprocess_args.append('--verbosity=%s' % options.verbosity) + if not options.interactive: + subprocess_args.append('--noinput') + + iteration = 1 + while len(test_labels) > 1: + midpoint = len(test_labels)/2 + test_labels_a = test_labels[:midpoint] + [bisection_label] + test_labels_b = test_labels[midpoint:] + [bisection_label] + print '***** Pass %da: Running the first half of the test suite' % iteration + print '***** Test labels:',' '.join(test_labels_a) + failures_a = subprocess.call(subprocess_args + test_labels_a) + + print '***** Pass %db: Running the second half of the test suite' % iteration + print '***** Test labels:',' '.join(test_labels_b) + print + failures_b = subprocess.call(subprocess_args + test_labels_b) + + if failures_a and not failures_b: + print "***** Problem found in first half. Bisecting again..." + iteration = iteration + 1 + test_labels = test_labels_a[:-1] + elif failures_b and not failures_a: + print "***** Problem found in second half. Bisecting again..." + iteration = iteration + 1 + test_labels = test_labels_b[:-1] + elif failures_a and failures_b: + print "***** Multiple sources of failure found" + break + else: + print "***** No source of failure found... try pair execution (--pair)" + break + + if len(test_labels) == 1: + print "***** Source of error:",test_labels[0] + teardown(state) + +def paired_tests(paired_test, options, test_labels): + state = setup(int(options.verbosity), test_labels) + + if not test_labels: + print "" + # Get the full list of test labels to use for bisection + from django.db.models.loading import get_apps + test_labels = [app.__name__.split('.')[-2] for app in get_apps()] + + print '***** Trying paired execution' + + # Make sure the bisection point isn't in the test list + # Also remove tests that need to be run in specific combinations + for label in [paired_test, 'model_inheritance_same_model_name']: + try: + test_labels.remove(label) + except ValueError: + pass + + subprocess_args = ['python','runtests.py', '--settings=%s' % options.settings] + if options.failfast: + subprocess_args.append('--failfast') + if options.verbosity: + subprocess_args.append('--verbosity=%s' % options.verbosity) + if not options.interactive: + subprocess_args.append('--noinput') + + for i, label in enumerate(test_labels): + print '***** %d of %d: Check test pairing with %s' % (i+1, len(test_labels), label) + failures = subprocess.call(subprocess_args + [label, paired_test]) + if failures: + print '***** Found problem pair with',label + return + + print '***** No problem pair found' + teardown(state) if __name__ == "__main__": from optparse import OptionParser @@ -194,10 +295,22 @@ if __name__ == "__main__": help='Tells Django to stop running the test suite after first failed test.') parser.add_option('--settings', help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.') + parser.add_option('--bisect', action='store', dest='bisect', default=None, + help="Bisect the test suite to discover a test that causes a test failure when combined with the named test.") + parser.add_option('--pair', action='store', dest='pair', default=None, + help="Run the test suite in pairs with the named test to find problem pairs.") options, args = parser.parse_args() if options.settings: os.environ['DJANGO_SETTINGS_MODULE'] = options.settings elif "DJANGO_SETTINGS_MODULE" not in os.environ: parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. " "Set it or use --settings.") - django_tests(int(options.verbosity), options.interactive, options.failfast, args) + + if options.bisect: + bisect_tests(options.bisect, options, args) + elif options.pair: + paired_tests(options.pair, options, args) + else: + failures = django_tests(int(options.verbosity), options.interactive, options.failfast, args) + if failures: + sys.exit(bool(failures))