From c1c163b42717ed5e051098ebf0e2f5c77810f20e Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 10 Sep 2017 15:34:18 +0100 Subject: [PATCH] Fixed #28574 -- Added QuerySet.explain(). --- django/db/backends/base/features.py | 12 +++ django/db/backends/base/operations.py | 18 ++++ django/db/backends/mysql/features.py | 7 ++ django/db/backends/mysql/operations.py | 13 +++ django/db/backends/postgresql/features.py | 2 + django/db/backends/postgresql/operations.py | 15 ++++ django/db/backends/sqlite3/operations.py | 1 + django/db/models/query.py | 3 + django/db/models/sql/compiler.py | 16 ++++ django/db/models/sql/query.py | 12 +++ docs/ref/models/querysets.txt | 43 +++++++++ docs/releases/2.1.txt | 3 + tests/basic/tests.py | 1 + tests/queries/test_explain.py | 99 +++++++++++++++++++++ tests/runtests.py | 8 ++ 15 files changed, 253 insertions(+) create mode 100644 tests/queries/test_explain.py diff --git a/django/db/backends/base/features.py b/django/db/backends/base/features.py index 74f7829e3f..d5b142383a 100644 --- a/django/db/backends/base/features.py +++ b/django/db/backends/base/features.py @@ -250,9 +250,21 @@ class BaseDatabaseFeatures: # Convert CharField results from bytes to str in database functions. db_functions_convert_bytes_to_str = False + # What formats does the backend EXPLAIN syntax support? + supported_explain_formats = set() + + # Does DatabaseOperations.explain_query_prefix() raise ValueError if + # unknown kwargs are passed to QuerySet.explain()? + validates_explain_options = True + def __init__(self, connection): self.connection = connection + @cached_property + def supports_explaining_query_execution(self): + """Does this backend support explaining query execution?""" + return self.connection.ops.explain_prefix is not None + @cached_property def supports_transactions(self): """Confirm support for transactions.""" diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py index 465ac70b7b..1fe7fe827c 100644 --- a/django/db/backends/base/operations.py +++ b/django/db/backends/base/operations.py @@ -45,6 +45,9 @@ class BaseDatabaseOperations: UNBOUNDED_FOLLOWING = 'UNBOUNDED ' + FOLLOWING CURRENT_ROW = 'CURRENT ROW' + # Prefix for EXPLAIN queries, or None EXPLAIN isn't supported. + explain_prefix = None + def __init__(self, connection): self.connection = connection self._cache = None @@ -652,3 +655,18 @@ class BaseDatabaseOperations: def window_frame_range_start_end(self, start=None, end=None): return self.window_frame_rows_start_end(start, end) + + def explain_query_prefix(self, format=None, **options): + if not self.connection.features.supports_explaining_query_execution: + raise NotSupportedError('This backend does not support explaining query execution.') + if format: + supported_formats = self.connection.features.supported_explain_formats + normalized_format = format.upper() + if normalized_format not in supported_formats: + msg = '%s is not a recognized format.' % normalized_format + if supported_formats: + msg += ' Allowed formats: %s' % ', '.join(sorted(supported_formats)) + raise ValueError(msg) + if options: + raise ValueError('Unknown options: %s' % ', '.join(sorted(options.keys()))) + return self.explain_prefix diff --git a/django/db/backends/mysql/features.py b/django/db/backends/mysql/features.py index 691fba49ad..57f4c8ee76 100644 --- a/django/db/backends/mysql/features.py +++ b/django/db/backends/mysql/features.py @@ -49,6 +49,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): END; """ db_functions_convert_bytes_to_str = True + # Alias MySQL's TRADITIONAL to TEXT for consistency with other backends. + supported_explain_formats = {'JSON', 'TEXT', 'TRADITIONAL'} @cached_property def _mysql_storage_engine(self): @@ -81,6 +83,11 @@ class DatabaseFeatures(BaseDatabaseFeatures): def supports_over_clause(self): return self.connection.mysql_version >= (8, 0, 2) + @cached_property + def needs_explain_extended(self): + # EXTENDED is deprecated (and not required) in 5.7 and removed in 8.0. + return self.connection.mysql_version < (5, 7) + @cached_property def supports_transactions(self): """ diff --git a/django/db/backends/mysql/operations.py b/django/db/backends/mysql/operations.py index 2b85be9c56..e5c0c5c71b 100644 --- a/django/db/backends/mysql/operations.py +++ b/django/db/backends/mysql/operations.py @@ -26,6 +26,7 @@ class DatabaseOperations(BaseDatabaseOperations): 'PositiveSmallIntegerField': 'unsigned integer', } cast_char_field_without_max_length = 'char' + explain_prefix = 'EXPLAIN' def date_extract_sql(self, lookup_type, field_name): # http://dev.mysql.com/doc/mysql/en/date-and-time-functions.html @@ -269,3 +270,15 @@ class DatabaseOperations(BaseDatabaseOperations): ) % {'lhs': lhs_sql, 'rhs': rhs_sql}, lhs_params * 2 + rhs_params * 2 else: return "TIMESTAMPDIFF(MICROSECOND, %s, %s)" % (rhs_sql, lhs_sql), rhs_params + lhs_params + + def explain_query_prefix(self, format=None, **options): + # Alias MySQL's TRADITIONAL to TEXT for consistency with other backends. + if format and format.upper() == 'TEXT': + format = 'TRADITIONAL' + prefix = super().explain_query_prefix(format, **options) + if format: + prefix += ' FORMAT=%s' % format + if self.connection.features.needs_explain_extended and format is None: + # EXTENDED and FORMAT are mutually exclusive options. + prefix += ' EXTENDED' + return prefix diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index a51383df8f..06b0303bac 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -50,6 +50,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): $$ LANGUAGE plpgsql;""" supports_over_clause = True supports_aggregate_filter_clause = True + supported_explain_formats = {'JSON', 'TEXT', 'XML', 'YAML'} + validates_explain_options = False # A query will error on invalid options. @cached_property def is_postgresql_9_5(self): diff --git a/django/db/backends/postgresql/operations.py b/django/db/backends/postgresql/operations.py index 6f48cfa228..80a28bce46 100644 --- a/django/db/backends/postgresql/operations.py +++ b/django/db/backends/postgresql/operations.py @@ -7,6 +7,7 @@ from django.db.backends.base.operations import BaseDatabaseOperations class DatabaseOperations(BaseDatabaseOperations): cast_char_field_without_max_length = 'varchar' + explain_prefix = 'EXPLAIN' def unification_cast_sql(self, output_field): internal_type = output_field.get_internal_type() @@ -258,3 +259,17 @@ class DatabaseOperations(BaseDatabaseOperations): 'and FOLLOWING.' ) return start_, end_ + + def explain_query_prefix(self, format=None, **options): + prefix = super().explain_query_prefix(format) + extra = {} + if format: + extra['FORMAT'] = format + if options: + extra.update({ + name.upper(): 'true' if value else 'false' + for name, value in options.items() + }) + if extra: + prefix += ' (%s)' % ', '.join('%s %s' % i for i in extra.items()) + return prefix diff --git a/django/db/backends/sqlite3/operations.py b/django/db/backends/sqlite3/operations.py index 10b064d966..ee197d34b4 100644 --- a/django/db/backends/sqlite3/operations.py +++ b/django/db/backends/sqlite3/operations.py @@ -19,6 +19,7 @@ class DatabaseOperations(BaseDatabaseOperations): 'DateField': 'TEXT', 'DateTimeField': 'TEXT', } + explain_prefix = 'EXPLAIN QUERY PLAN' def bulk_batch_size(self, fields, objs): """ diff --git a/django/db/models/query.py b/django/db/models/query.py index 2db6690c44..1c6c3aae35 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -717,6 +717,9 @@ class QuerySet: prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups) self._prefetch_done = True + def explain(self, *, format=None, **options): + return self.query.explain(using=self.db, format=format, **options) + ################################################## # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS # ################################################## diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index cb5e3136e2..f5222f21be 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -530,6 +530,12 @@ class SQLCompiler: result.append('HAVING %s' % having) params.extend(h_params) + if self.query.explain_query: + result.insert(0, self.connection.ops.explain_query_prefix( + self.query.explain_format, + **self.query.explain_options + )) + if order_by: ordering = [] for _, (o_sql, o_params, _) in order_by: @@ -1101,6 +1107,16 @@ class SQLCompiler: sql, params = self.as_sql() return 'EXISTS (%s)' % sql, params + def explain_query(self): + result = list(self.execute_sql()) + # Some backends return 1 item tuples with strings, and others return + # tuples with integers and strings. Flatten them out into strings. + for row in result[0]: + if not isinstance(row, str): + yield ' '.join(str(c) for c in row) + else: + yield row + class SQLInsertCompiler(SQLCompiler): return_id = False diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 3756ecbb5d..995e89564d 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -223,6 +223,10 @@ class Query: self._filtered_relations = {} + self.explain_query = False + self.explain_format = None + self.explain_options = {} + @property def extra(self): if self._extra is None: @@ -511,6 +515,14 @@ class Query: compiler = q.get_compiler(using=using) return compiler.has_results() + def explain(self, using, format=None, **options): + q = self.clone() + q.explain_query = True + q.explain_format = format + q.explain_options = options + compiler = q.get_compiler(using=using) + return '\n'.join(compiler.explain_query()) + def combine(self, rhs, connector): """ Merge the 'rhs' query into the current one (with any 'rhs' effects diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index d623966b3a..b2f430d7bc 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -2476,6 +2476,49 @@ Class method that returns an instance of :class:`~django.db.models.Manager` with a copy of the ``QuerySet``’s methods. See :ref:`create-manager-with-queryset-methods` for more details. +``explain()`` +~~~~~~~~~~~~~ + +.. versionadded:: 2.1 + +.. method:: explain(format=None, **options) + +Returns a string of the ``QuerySet``’s execution plan, which details how the +database would execute the query, including any indexes or joins that would be +used. Knowing these details may help you improve the performance of slow +queries. + +For example, when using PostgreSQL:: + + >>> print(Blog.objects.filter(title='My Blog').explain()) + Seq Scan on blog (cost=0.00..35.50 rows=10 width=12) + Filter: (title = 'My Blog'::bpchar) + +The output differs significantly between databases. + +``explain()`` is supported by all built-in database backends except Oracle +because an implementation there isn't straightforward. + +The ``format`` parameter changes the output format from the databases's default, +usually text-based. PostgreSQL supports ``'TEXT'``, ``'JSON'``, ``'YAML'``, and +``'XML'``. MySQL supports ``'TEXT'`` (also called ``'TRADITIONAL'``) and +``'JSON'``. + +Some databases accept flags that can return more information about the query. +Pass these flags as keyword arguments. For example, when using PostgreSQL:: + + >>> print(Blog.objects.filter(title='My Blog').explain(verbose=True)) + Seq Scan on public.blog (cost=0.00..35.50 rows=10 width=12) (actual time=0.004..0.004 rows=10 loops=1) + Output: id, title + Filter: (blog.title = 'My Blog'::bpchar) + Planning time: 0.064 ms + Execution time: 0.058 ms + +On some databases, flags may cause the query to be executed which could have +adverse effects on your database. For example, PostgreSQL's ``ANALYZE`` flag +could result in changes to data if there are triggers or if a function is +called, even for a ``SELECT`` query. + .. _field-lookups: ``Field`` lookups diff --git a/docs/releases/2.1.txt b/docs/releases/2.1.txt index 9044c3cd70..7b65b90b9c 100644 --- a/docs/releases/2.1.txt +++ b/docs/releases/2.1.txt @@ -236,6 +236,9 @@ Models encouraged instead of :class:`~django.db.models.NullBooleanField`, which will likely be deprecated in the future. +* The new :meth:`.QuerySet.explain` method displays the database's execution + plan of a queryset's query. + Requests and Responses ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/basic/tests.py b/tests/basic/tests.py index 1a5e95e9d6..76d92a7591 100644 --- a/tests/basic/tests.py +++ b/tests/basic/tests.py @@ -555,6 +555,7 @@ class ManagerTest(SimpleTestCase): 'only', 'using', 'exists', + 'explain', '_insert', '_update', 'raw', diff --git a/tests/queries/test_explain.py b/tests/queries/test_explain.py new file mode 100644 index 0000000000..26baf6fb30 --- /dev/null +++ b/tests/queries/test_explain.py @@ -0,0 +1,99 @@ +import unittest + +from django.db import NotSupportedError, connection, transaction +from django.db.models import Count +from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature +from django.test.utils import CaptureQueriesContext + +from .models import Tag + + +@skipUnlessDBFeature('supports_explaining_query_execution') +class ExplainTests(TestCase): + + def test_basic(self): + querysets = [ + Tag.objects.filter(name='test'), + Tag.objects.filter(name='test').select_related('parent'), + Tag.objects.filter(name='test').prefetch_related('children'), + Tag.objects.filter(name='test').annotate(Count('children')), + Tag.objects.filter(name='test').values_list('name'), + Tag.objects.order_by().union(Tag.objects.order_by().filter(name='test')), + Tag.objects.all().select_for_update().filter(name='test'), + ] + supported_formats = connection.features.supported_explain_formats + all_formats = (None,) + tuple(supported_formats) + tuple(f.lower() for f in supported_formats) + for idx, queryset in enumerate(querysets): + for format in all_formats: + with self.subTest(format=format, queryset=idx): + if connection.vendor == 'mysql': + # This does a query and caches the result. + connection.features.needs_explain_extended + with self.assertNumQueries(1), CaptureQueriesContext(connection) as captured_queries: + result = queryset.explain(format=format) + self.assertTrue(captured_queries[0]['sql'].startswith(connection.ops.explain_prefix)) + self.assertIsInstance(result, str) + self.assertTrue(result) + + @skipUnlessDBFeature('validates_explain_options') + def test_unknown_options(self): + with self.assertRaisesMessage(ValueError, 'Unknown options: test, test2'): + Tag.objects.all().explain(test=1, test2=1) + + def test_unknown_format(self): + msg = 'DOES NOT EXIST is not a recognized format.' + if connection.features.supported_explain_formats: + msg += ' Allowed formats: %s' % ', '.join(sorted(connection.features.supported_explain_formats)) + with self.assertRaisesMessage(ValueError, msg): + Tag.objects.all().explain(format='does not exist') + + @unittest.skipUnless(connection.vendor == 'postgresql', 'PostgreSQL specific') + def test_postgres_options(self): + qs = Tag.objects.filter(name='test') + test_options = [ + {'COSTS': False, 'BUFFERS': True, 'ANALYZE': True}, + {'costs': False, 'buffers': True, 'analyze': True}, + {'verbose': True, 'timing': True, 'analyze': True}, + {'verbose': False, 'timing': False, 'analyze': True}, + ] + if connection.pg_version >= 100000: + test_options.append({'summary': True}) + for options in test_options: + with self.subTest(**options), transaction.atomic(): + with CaptureQueriesContext(connection) as captured_queries: + qs.explain(format='text', **options) + self.assertEqual(len(captured_queries), 1) + for name, value in options.items(): + option = '{} {}'.format(name.upper(), 'true' if value else 'false') + self.assertIn(option, captured_queries[0]['sql']) + + @unittest.skipUnless(connection.vendor == 'mysql', 'MySQL specific') + def test_mysql_text_to_traditional(self): + with CaptureQueriesContext(connection) as captured_queries: + Tag.objects.filter(name='test').explain(format='text') + self.assertEqual(len(captured_queries), 1) + self.assertIn('FORMAT=TRADITIONAL', captured_queries[0]['sql']) + + @unittest.skipUnless(connection.vendor == 'mysql', 'MySQL < 5.7 specific') + def test_mysql_extended(self): + # Inner skip to avoid module level query for MySQL version. + if not connection.features.needs_explain_extended: + raise unittest.SkipTest('MySQL < 5.7 specific') + qs = Tag.objects.filter(name='test') + with CaptureQueriesContext(connection) as captured_queries: + qs.explain(format='json') + self.assertEqual(len(captured_queries), 1) + self.assertNotIn('EXTENDED', captured_queries[0]['sql']) + with CaptureQueriesContext(connection) as captured_queries: + qs.explain(format='text') + self.assertEqual(len(captured_queries), 1) + self.assertNotIn('EXTENDED', captured_queries[0]['sql']) + + +@skipIfDBFeature('supports_explaining_query_execution') +class ExplainUnsupportedTests(TestCase): + + def test_message(self): + msg = 'This backend does not support explaining query execution.' + with self.assertRaisesMessage(NotSupportedError, msg): + Tag.objects.filter(name='test').explain() diff --git a/tests/runtests.py b/tests/runtests.py index 0302137dbb..c3d5ee427f 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -20,6 +20,14 @@ from django.test.utils import get_runner from django.utils.deprecation import RemovedInDjango30Warning from django.utils.log import DEFAULT_LOGGING +try: + import MySQLdb +except ImportError: + pass +else: + # Ignore informational warnings from QuerySet.explain(). + warnings.filterwarnings('ignore', '\(1003, *', category=MySQLdb.Warning) + # Make deprecation warnings errors to ensure no usage of deprecated features. warnings.simplefilter("error", RemovedInDjango30Warning) # Make runtime warning errors to ensure no usage of error prone patterns.