From d9521f66b1851b0eacd55bc78f801dc64123e333 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 2 May 2015 15:54:17 +0200 Subject: [PATCH] Removed global timezone-aware datetime adapters. Refs #23820. Fixed #19738. Refs #17755. In order not to introduce a regression for raw queries, parameters are passed through the connection.ops.value_to_db_* methods, depending on their type. --- django/db/backends/base/operations.py | 23 +++++++++++ django/db/backends/mysql/base.py | 20 ++++----- django/db/backends/oracle/base.py | 16 ++++---- django/db/backends/oracle/operations.py | 13 ------ django/db/backends/sqlite3/base.py | 20 ++++----- django/db/models/sql/query.py | 24 +++++++++-- docs/internals/deprecation.txt | 5 +++ docs/releases/1.9.txt | 54 ++++++++++++++++++------- tests/timezones/tests.py | 26 +++++++++++- 9 files changed, 141 insertions(+), 60 deletions(-) diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py index df6d844297..485abaa40d 100644 --- a/django/db/backends/base/operations.py +++ b/django/db/backends/base/operations.py @@ -433,6 +433,25 @@ class BaseDatabaseOperations(object): """ return value + def value_to_db_unknown(self, value): + """ + Transforms a value to something compatible with the backend driver. + + This method only depends on the type of the value. It's designed for + cases where the target type isn't known, such as .raw() SQL queries. + As a consequence it may not work perfectly in all circumstances. + """ + if isinstance(value, datetime.datetime): # must be before date + return self.value_to_db_datetime(value) + elif isinstance(value, datetime.date): + return self.value_to_db_date(value) + elif isinstance(value, datetime.time): + return self.value_to_db_time(value) + elif isinstance(value, decimal.Decimal): + return self.value_to_db_decimal(value) + else: + return value + def value_to_db_date(self, value): """ Transforms a date value to an object compatible with what is expected @@ -486,6 +505,8 @@ class BaseDatabaseOperations(object): """ first = datetime.date(value, 1, 1) second = datetime.date(value, 12, 31) + first = self.value_to_db_date(first) + second = self.value_to_db_date(second) return [first, second] def year_lookup_bounds_for_datetime_field(self, value): @@ -502,6 +523,8 @@ class BaseDatabaseOperations(object): tz = timezone.get_current_timezone() first = timezone.make_aware(first, tz) second = timezone.make_aware(second, tz) + first = self.value_to_db_datetime(first) + second = self.value_to_db_datetime(second) return [first, second] def get_db_converters(self, expression): diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 4c8809fdb7..e603c75eae 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -16,6 +16,7 @@ from django.db import utils from django.db.backends import utils as backend_utils from django.db.backends.base.base import BaseDatabaseWrapper from django.utils import six, timezone +from django.utils.deprecation import RemovedInDjango21Warning from django.utils.encoding import force_str from django.utils.functional import cached_property from django.utils.safestring import SafeBytes, SafeText @@ -52,15 +53,14 @@ DatabaseError = Database.DatabaseError IntegrityError = Database.IntegrityError -def adapt_datetime_with_timezone_support(value, conv): - # Equivalent to DateTimeField.get_db_prep_value. Used only by raw SQL. - if settings.USE_TZ: - if timezone.is_naive(value): - warnings.warn("MySQL received a naive datetime (%s)" - " while time zone support is active." % value, - RuntimeWarning) - default_timezone = timezone.get_default_timezone() - value = timezone.make_aware(value, default_timezone) +def adapt_datetime_warn_on_aware_datetime(value, conv): + # Remove this function and rely on the default adapter in Django 2.1. + if settings.USE_TZ and timezone.is_aware(value): + warnings.warn( + "The MySQL database adapter received an aware datetime (%s), " + "probably from cursor.execute(). Update your code to pass a " + "naive datetime in the database connection's time zone (UTC by " + "default).", RemovedInDjango21Warning) value = value.astimezone(timezone.utc).replace(tzinfo=None) return Thing2Literal(value.strftime("%Y-%m-%d %H:%M:%S.%f"), conv) @@ -74,7 +74,7 @@ django_conversions.update({ FIELD_TYPE.TIME: backend_utils.typecast_time, FIELD_TYPE.DECIMAL: backend_utils.typecast_decimal, FIELD_TYPE.NEWDECIMAL: backend_utils.typecast_decimal, - datetime.datetime: adapt_datetime_with_timezone_support, + datetime.datetime: adapt_datetime_warn_on_aware_datetime, }) # This should match the numerical portion of the version numbers (we can treat diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index b84b28b97b..f88527e349 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -17,6 +17,7 @@ from django.db import utils from django.db.backends.base.base import BaseDatabaseWrapper from django.db.backends.base.validation import BaseDatabaseValidation from django.utils import six, timezone +from django.utils.deprecation import RemovedInDjango21Warning from django.utils.duration import duration_string from django.utils.encoding import force_bytes, force_text from django.utils.functional import cached_property @@ -336,13 +337,14 @@ class OracleParam(object): # without being converted by DateTimeField.get_db_prep_value. if settings.USE_TZ and (isinstance(param, datetime.datetime) and not isinstance(param, Oracle_datetime)): - if timezone.is_naive(param): - warnings.warn("Oracle received a naive datetime (%s)" - " while time zone support is active." % param, - RuntimeWarning) - default_timezone = timezone.get_default_timezone() - param = timezone.make_aware(param, default_timezone) - param = Oracle_datetime.from_datetime(param.astimezone(timezone.utc)) + if timezone.is_aware(param): + warnings.warn( + "The Oracle database adapter received an aware datetime (%s), " + "probably from cursor.execute(). Update your code to pass a " + "naive datetime in the database connection's time zone (UTC by " + "default).", RemovedInDjango21Warning) + param = param.astimezone(timezone.utc).replace(tzinfo=None) + param = Oracle_datetime.from_datetime(param) if isinstance(param, datetime.timedelta): param = duration_string(param) diff --git a/django/db/backends/oracle/operations.py b/django/db/backends/oracle/operations.py index 2b3ffb3a8c..8b6dbf2fdc 100644 --- a/django/db/backends/oracle/operations.py +++ b/django/db/backends/oracle/operations.py @@ -419,19 +419,6 @@ WHEN (new.%(col_name)s IS NULL) return Oracle_datetime(1900, 1, 1, value.hour, value.minute, value.second, value.microsecond) - def year_lookup_bounds_for_date_field(self, value): - # Create bounds as real date values - first = datetime.date(value, 1, 1) - last = datetime.date(value, 12, 31) - return [first, last] - - def year_lookup_bounds_for_datetime_field(self, value): - # cx_Oracle doesn't support tz-aware datetimes - bounds = super(DatabaseOperations, self).year_lookup_bounds_for_datetime_field(value) - if settings.USE_TZ: - bounds = [b.astimezone(timezone.utc) for b in bounds] - return [Oracle_datetime.from_datetime(b) for b in bounds] - def combine_expression(self, connector, sub_expressions): "Oracle requires special cases for %% and & operators in query expressions" if connector == '%%': diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 4cc6266893..dea41a8ca6 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -20,6 +20,7 @@ from django.utils import six, timezone from django.utils.dateparse import ( parse_date, parse_datetime, parse_duration, parse_time, ) +from django.utils.deprecation import RemovedInDjango21Warning from django.utils.encoding import force_text from django.utils.safestring import SafeBytes @@ -49,15 +50,14 @@ DatabaseError = Database.DatabaseError IntegrityError = Database.IntegrityError -def adapt_datetime_with_timezone_support(value): - # Equivalent to DateTimeField.get_db_prep_value. Used only by raw SQL. - if settings.USE_TZ: - if timezone.is_naive(value): - warnings.warn("SQLite received a naive datetime (%s)" - " while time zone support is active." % value, - RuntimeWarning) - default_timezone = timezone.get_default_timezone() - value = timezone.make_aware(value, default_timezone) +def adapt_datetime_warn_on_aware_datetime(value): + # Remove this function and rely on the default adapter in Django 2.1. + if settings.USE_TZ and timezone.is_aware(value): + warnings.warn( + "The SQLite database adapter received an aware datetime (%s), " + "probably from cursor.execute(). Update your code to pass a " + "naive datetime in the database connection's time zone (UTC by " + "default).", RemovedInDjango21Warning) value = value.astimezone(timezone.utc).replace(tzinfo=None) return value.isoformat(str(" ")) @@ -77,7 +77,7 @@ Database.register_converter(str("timestamp"), decoder(parse_datetime)) Database.register_converter(str("TIMESTAMP"), decoder(parse_datetime)) Database.register_converter(str("decimal"), decoder(backend_utils.typecast_decimal)) -Database.register_adapter(datetime.datetime, adapt_datetime_with_timezone_support) +Database.register_adapter(datetime.datetime, adapt_datetime_warn_on_aware_datetime) Database.register_adapter(decimal.Decimal, backend_utils.rev_typecast_decimal) if six.PY2: Database.register_adapter(str, lambda s: s.decode('utf-8')) diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index d84e3aa681..52553ddfa1 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -86,13 +86,29 @@ class RawQuery(object): def __repr__(self): return "" % self + @property + def params_type(self): + return dict if isinstance(self.params, Mapping) else tuple + def __str__(self): - _type = dict if isinstance(self.params, Mapping) else tuple - return self.sql % _type(self.params) + return self.sql % self.params_type(self.params) def _execute_query(self): - self.cursor = connections[self.using].cursor() - self.cursor.execute(self.sql, self.params) + connection = connections[self.using] + + # Adapt parameters to the database, as much as possible considering + # that the target type isn't known. See #17755. + params_type = self.params_type + adapter = connection.ops.value_to_db_unknown + if params_type is tuple: + params = tuple(adapter(val) for val in self.params) + elif params_type is dict: + params = dict((key, adapter(val)) for key, val in six.iteritems(self.params)) + else: + raise RuntimeError("Unexpected params type: %s" % params_type) + + self.cursor = connection.cursor() + self.cursor.execute(self.sql, params) class Query(object): diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 6580d2796c..1aa4d79524 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -32,6 +32,11 @@ details on these changes. * ``django.db.models.fields.add_lazy_relation()`` will be removed. +* When time zone support is enabled, database backends that don't support time + zones won't convert aware datetimes to naive values in UTC anymore when such + values are passed as parameters to SQL queries executed outside of the ORM, + e.g. with ``cursor.execute()``. + * The ``django.contrib.auth.tests.utils.skipIfCustomUser()`` decorator will be removed. diff --git a/docs/releases/1.9.txt b/docs/releases/1.9.txt index 28c953b212..f0c8a345bc 100644 --- a/docs/releases/1.9.txt +++ b/docs/releases/1.9.txt @@ -313,11 +313,15 @@ Database backend API doesn't implement this. You may want to review the implementation on the backends that Django includes for reference (:ticket:`24245`). -* The recommended way to add time zone information to datetimes fetched from - databases that don't support time zones is to register a converter for - ``DateTimeField``. Do this in ``DatabaseOperations.get_db_converters()``. - Registering a global converter at the level of the DB-API module is - discouraged because it can conflict with other libraries. +* Registering a global adapter or converter at the level of the DB-API module + to handle time zone information of :class:`~datetime.datetime` values passed + as query parameters or returned as query results on databases that don't + support time zones is discouraged. It can conflict with other libraries. + + The recommended way to add a time zone to :class:`~datetime.datetime` values + fetched from the database is to register a converter for ``DateTimeField`` + in ``DatabaseOperations.get_db_converters()``. + Default settings that were tuples are now lists ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -418,20 +422,40 @@ console, for example. If you are overriding Django's default logging, you should check to see how your configuration merges with the new defaults. -Removal of time zone aware global converters for datetimes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Removal of time zone aware global adapters and converters for datetimes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Django no longer registers global converters for returning time zone aware -datetimes in database query results when :setting:`USE_TZ` is ``True``. -Instead the ORM adds suitable time zone information. +Django no longer registers global adapters and converters for managing time +zone information on :class:`~datetime.datetime` values sent to the database as +query parameters or read from the database in query results. This change +affects projects that meet all the following conditions: -As a consequence, SQL queries executed outside of the ORM, for instance with -``cursor.execute(query, params)``, now return naive datetimes instead of aware -datetimes on databases that do not support time zones: SQLite, MySQL, and -Oracle. Since these datetimes are in UTC, you can make them aware as follows:: +* The :setting:`USE_TZ` setting is ``True``. +* The database is SQLite, MySQL, Oracle, or a third-party database that + doesn't support time zones. In doubt, you can check the value of + ``connection.features.supports_timezones``. +* The code queries the database outside of the ORM, typically with + ``cursor.execute(sql, params)``. + +If you're passing aware :class:`~datetime.datetime` parameters to such +queries, you should turn them into naive datetimes in UTC:: from django.utils import timezone - value = value.replace(tzinfo=timezone.utc) + param = timezone.make_naive(param, timezone.utc) + +If you fail to do so, Django 1.9 and 2.0 will perform the conversion like +earlier versions but emit a deprecation warning. Django 2.1 won't perform any +conversion, which may result in data corruption. + +If you're reading :class:`~datetime.datetime` values from the results, they +will be naive instead of aware. You can compensate as follows:: + + from django.utils import timezone + value = timezone.make_aware(value, timezone.utc) + +You don't need any of this if you're querying the database through the ORM, +even if you're using :meth:`raw() ` +queries. The ORM takes care of managing time zone information. Miscellaneous ~~~~~~~~~~~~~ diff --git a/tests/timezones/tests.py b/tests/timezones/tests.py index 39158badc1..dc2d358364 100644 --- a/tests/timezones/tests.py +++ b/tests/timezones/tests.py @@ -266,6 +266,13 @@ class LegacyDatabaseTests(TestCase): [event], transform=lambda d: d) + def test_cursor_execute_accepts_naive_datetime(self): + dt = datetime.datetime(2011, 9, 1, 13, 20, 30) + with connection.cursor() as cursor: + cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [dt]) + event = Event.objects.get() + self.assertEqual(event.dt, dt) + def test_cursor_execute_returns_naive_datetime(self): dt = datetime.datetime(2011, 9, 1, 13, 20, 30) Event.objects.create(dt=dt) @@ -564,6 +571,23 @@ class NewDatabaseTests(TestCase): [event], transform=lambda d: d) + @skipUnlessDBFeature('supports_timezones') + def test_cursor_execute_accepts_aware_datetime(self): + dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT) + with connection.cursor() as cursor: + cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [dt]) + event = Event.objects.get() + self.assertEqual(event.dt, dt) + + @skipIfDBFeature('supports_timezones') + def test_cursor_execute_accepts_naive_datetime(self): + dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT) + utc_naive_dt = timezone.make_naive(dt, timezone.utc) + with connection.cursor() as cursor: + cursor.execute('INSERT INTO timezones_event (dt) VALUES (%s)', [utc_naive_dt]) + event = Event.objects.get() + self.assertEqual(event.dt, dt) + @skipUnlessDBFeature('supports_timezones') def test_cursor_execute_returns_aware_datetime(self): dt = datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT) @@ -578,7 +602,7 @@ class NewDatabaseTests(TestCase): utc_naive_dt = timezone.make_naive(dt, timezone.utc) Event.objects.create(dt=dt) with connection.cursor() as cursor: - cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [dt]) + cursor.execute('SELECT dt FROM timezones_event WHERE dt = %s', [utc_naive_dt]) self.assertEqual(cursor.fetchall()[0][0], utc_naive_dt) @requires_tz_support