diff --git a/django/db/backends/base/features.py b/django/db/backends/base/features.py index 095164770f..8005b6e6b4 100644 --- a/django/db/backends/base/features.py +++ b/django/db/backends/base/features.py @@ -64,6 +64,10 @@ class BaseDatabaseFeatures(object): # Is there a true datatype for timedeltas? has_native_duration_field = False + # Does the database driver supports same type temporal data subtraction + # by returning the type used to store duration field? + supports_temporal_subtraction = False + # Does the database driver support timedeltas as arguments? # This is only relevant when there is a native duration field. # Specifically, there is a bug with cx_Oracle: diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py index e7ecbca4e4..907a59f269 100644 --- a/django/db/backends/base/operations.py +++ b/django/db/backends/base/operations.py @@ -590,3 +590,10 @@ class BaseDatabaseOperations(object): range of the column type bound to the field. """ return self.integer_field_ranges[internal_type] + + def subtract_temporals(self, internal_type, lhs, rhs): + if self.connection.features.supports_temporal_subtraction: + lhs_sql, lhs_params = lhs + rhs_sql, rhs_params = rhs + return "(%s - %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params + raise NotImplementedError("This backend does not support %s subtraction." % internal_type) diff --git a/django/db/backends/mysql/features.py b/django/db/backends/mysql/features.py index 075b68a872..ea5fd0d9e9 100644 --- a/django/db/backends/mysql/features.py +++ b/django/db/backends/mysql/features.py @@ -32,6 +32,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): atomic_transactions = False supports_column_check_constraints = False can_clone_databases = True + supports_temporal_subtraction = True @cached_property def _mysql_storage_engine(self): diff --git a/django/db/backends/mysql/operations.py b/django/db/backends/mysql/operations.py index f81bf683b3..e71de6c4a2 100644 --- a/django/db/backends/mysql/operations.py +++ b/django/db/backends/mysql/operations.py @@ -210,3 +210,21 @@ class DatabaseOperations(BaseDatabaseOperations): if value is not None: value = uuid.UUID(value) return value + + def subtract_temporals(self, internal_type, lhs, rhs): + lhs_sql, lhs_params = lhs + rhs_sql, rhs_params = rhs + if self.connection.features.supports_microsecond_precision: + if internal_type == 'TimeField': + return ( + "((TIME_TO_SEC(%(lhs)s) * POW(10, 6) + MICROSECOND(%(lhs)s)) -" + " (TIME_TO_SEC(%(rhs)s) * POW(10, 6) + MICROSECOND(%(rhs)s)))" + ) % {'lhs': lhs_sql, 'rhs': rhs_sql}, lhs_params * 2 + rhs_params * 2 + else: + return "TIMESTAMPDIFF(MICROSECOND, %s, %s)" % (rhs_sql, lhs_sql), rhs_params + lhs_params + elif internal_type == 'TimeField': + return ( + "(TIME_TO_SEC(%s) * POW(10, 6) - TIME_TO_SEC(%s) * POW(10, 6))" + ) % (lhs_sql, rhs_sql), lhs_params + rhs_params + else: + return "(TIMESTAMPDIFF(SECOND, %s, %s) * POW(10, 6))" % (rhs_sql, lhs_sql), rhs_params + lhs_params diff --git a/django/db/backends/oracle/features.py b/django/db/backends/oracle/features.py index 1ef0f232e5..ee300f3530 100644 --- a/django/db/backends/oracle/features.py +++ b/django/db/backends/oracle/features.py @@ -39,6 +39,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): uppercases_column_names = True # select for update with limit can be achieved on Oracle, but not with the current backend. supports_select_for_update_with_limit = False + supports_temporal_subtraction = True def introspected_boolean_field_type(self, field=None, created_separately=False): """ diff --git a/django/db/backends/oracle/operations.py b/django/db/backends/oracle/operations.py index 3eebd7d979..c101fcd870 100644 --- a/django/db/backends/oracle/operations.py +++ b/django/db/backends/oracle/operations.py @@ -449,3 +449,10 @@ WHEN (new.%(col_name)s IS NULL) "SELECT %s FROM DUAL" % ", ".join(row) for row in placeholder_rows ) + + def subtract_temporals(self, internal_type, lhs, rhs): + if internal_type == 'DateField': + lhs_sql, lhs_params = lhs + rhs_sql, rhs_params = rhs + return "NUMTODSINTERVAL(%s - %s, 'DAY')" % (lhs_sql, rhs_sql), lhs_params + rhs_params + return super(DatabaseOperations, self).subtract_temporals(internal_type, lhs, rhs) diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index 9130a4521e..4465ab1cd1 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -29,3 +29,4 @@ class DatabaseFeatures(BaseDatabaseFeatures): requires_sqlparse_for_splitting = False greatest_least_ignores_nulls = True can_clone_databases = True + supports_temporal_subtraction = True diff --git a/django/db/backends/postgresql/operations.py b/django/db/backends/postgresql/operations.py index 3624c9cf56..5bd433c639 100644 --- a/django/db/backends/postgresql/operations.py +++ b/django/db/backends/postgresql/operations.py @@ -239,3 +239,10 @@ class DatabaseOperations(BaseDatabaseOperations): if value: return Inet(value) return None + + def subtract_temporals(self, internal_type, lhs, rhs): + if internal_type == 'DateField': + lhs_sql, lhs_params = lhs + rhs_sql, rhs_params = rhs + return "age(%s, %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params + return super(DatabaseOperations, self).subtract_temporals(internal_type, lhs, rhs) diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 55b97e8bd2..2330a98a53 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -213,6 +213,8 @@ class DatabaseWrapper(BaseDatabaseWrapper): conn.create_function("django_datetime_extract", 3, _sqlite_datetime_extract) conn.create_function("django_datetime_trunc", 3, _sqlite_datetime_trunc) conn.create_function("django_time_extract", 2, _sqlite_time_extract) + conn.create_function("django_time_diff", 2, _sqlite_time_diff) + conn.create_function("django_timestamp_diff", 2, _sqlite_timestamp_diff) conn.create_function("regexp", 2, _sqlite_regexp) conn.create_function("django_format_dtdelta", 3, _sqlite_format_dtdelta) conn.create_function("django_power", 2, _sqlite_power) @@ -444,6 +446,27 @@ def _sqlite_format_dtdelta(conn, lhs, rhs): return str(out) +def _sqlite_time_diff(lhs, rhs): + left = backend_utils.typecast_time(lhs) + right = backend_utils.typecast_time(rhs) + return ( + (left.hour * 60 * 60 * 1000000) + + (left.minute * 60 * 1000000) + + (left.second * 1000000) + + (left.microsecond) - + (right.hour * 60 * 60 * 1000000) - + (right.minute * 60 * 1000000) - + (right.second * 1000000) - + (right.microsecond) + ) + + +def _sqlite_timestamp_diff(lhs, rhs): + left = backend_utils.typecast_timestamp(lhs) + right = backend_utils.typecast_timestamp(rhs) + return (left - right).total_seconds() * 1000000 + + def _sqlite_regexp(re_pattern, re_string): return bool(re.search(re_pattern, force_text(re_string))) if re_string is not None else False diff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py index dffc48afb8..acf20d569f 100644 --- a/django/db/backends/sqlite3/features.py +++ b/django/db/backends/sqlite3/features.py @@ -38,6 +38,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_paramstyle_pyformat = False supports_sequence_reset = False can_clone_databases = True + supports_temporal_subtraction = True @cached_property def uses_savepoints(self): diff --git a/django/db/backends/sqlite3/operations.py b/django/db/backends/sqlite3/operations.py index 2af08469fb..1daa38fe50 100644 --- a/django/db/backends/sqlite3/operations.py +++ b/django/db/backends/sqlite3/operations.py @@ -263,3 +263,10 @@ class DatabaseOperations(BaseDatabaseOperations): def integer_field_range(self, internal_type): # SQLite doesn't enforce any integer constraints return (None, None) + + def subtract_temporals(self, internal_type, lhs, rhs): + lhs_sql, lhs_params = lhs + rhs_sql, rhs_params = rhs + if internal_type == 'TimeField': + return "django_time_diff(%s, %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params + return "django_timestamp_diff(%s, %s)" % (lhs_sql, rhs_sql), lhs_params + rhs_params diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 0f9c42009d..02174ef101 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -398,6 +398,10 @@ class CombinedExpression(Expression): ((lhs_output and lhs_output.get_internal_type() == 'DurationField') or (rhs_output and rhs_output.get_internal_type() == 'DurationField'))): return DurationExpression(self.lhs, self.connector, self.rhs).as_sql(compiler, connection) + if (lhs_output and rhs_output and self.connector == self.SUB and + lhs_output.get_internal_type() in {'DateField', 'DateTimeField', 'TimeField'} and + lhs_output.get_internal_type() == lhs_output.get_internal_type()): + return TemporalSubtraction(self.lhs, self.rhs).as_sql(compiler, connection) expressions = [] expression_params = [] sql, params = compiler.compile(self.lhs) @@ -448,6 +452,17 @@ class DurationExpression(CombinedExpression): return expression_wrapper % sql, expression_params +class TemporalSubtraction(CombinedExpression): + def __init__(self, lhs, rhs): + super(TemporalSubtraction, self).__init__(lhs, self.SUB, rhs, output_field=fields.DurationField()) + + def as_sql(self, compiler, connection): + connection.ops.check_expression_support(self) + lhs = compiler.compile(self.lhs, connection) + rhs = compiler.compile(self.rhs, connection) + return connection.ops.subtract_temporals(self.lhs.output_field.get_internal_type(), lhs, rhs) + + class F(Combinable): """ An object capable of resolving references to existing query objects. diff --git a/docs/releases/1.10.txt b/docs/releases/1.10.txt index 7ea4a5e4d5..1220d9402c 100644 --- a/docs/releases/1.10.txt +++ b/docs/releases/1.10.txt @@ -201,7 +201,7 @@ CSRF Database backends ~~~~~~~~~~~~~~~~~ -* ... +* Temporal data subtraction was unified on all backends. Email ~~~~~ @@ -398,6 +398,13 @@ Database backend API from the database are now converted to ``float`` to make it easier to combine them with values used by the GIS libraries. +* In order to enable temporal subtraction you must set the + ``supports_temporal_subtraction`` database feature flag to ``True`` and + implement the ``DatabaseOperations.subtract_temporals()`` method. This + method should return the SQL and parameters required to compute the + difference in microseconds between the ``lhs`` and ``rhs`` arguments in the + datatype used to store :class:`~django.db.models.DurationField`. + ``select_related()`` prohibits non-relational fields for nested relations ------------------------------------------------------------------------- diff --git a/tests/expressions/tests.py b/tests/expressions/tests.py index 21e53341c5..7a4f4adbc7 100644 --- a/tests/expressions/tests.py +++ b/tests/expressions/tests.py @@ -870,8 +870,42 @@ class FTimeDeltaTests(TestCase): Experiment.objects.filter(end__gte=F('start') + F('estimated_time') + datetime.timedelta(hours=1))] self.assertEqual(delta_math, ['e4']) - @skipUnlessDBFeature("has_native_duration_field") + @skipUnlessDBFeature('supports_temporal_subtraction') def test_date_subtraction(self): + queryset = Experiment.objects.annotate( + completion_duration=ExpressionWrapper( + F('completed') - F('assigned'), output_field=models.DurationField() + ) + ) + + at_least_5_days = {e.name for e in queryset.filter(completion_duration__gte=datetime.timedelta(days=5))} + self.assertEqual(at_least_5_days, {'e3', 'e4'}) + + less_than_5_days = {e.name for e in queryset.filter(completion_duration__lt=datetime.timedelta(days=5))} + expected = {'e0', 'e2'} + if connection.features.supports_microsecond_precision: + expected.add('e1') + self.assertEqual(less_than_5_days, expected) + + @skipUnlessDBFeature('supports_temporal_subtraction') + def test_time_subtraction(self): + if connection.features.supports_microsecond_precision: + time = datetime.time(12, 30, 15, 2345) + timedelta = datetime.timedelta(hours=1, minutes=15, seconds=15, microseconds=2345) + else: + time = datetime.time(12, 30, 15) + timedelta = datetime.timedelta(hours=1, minutes=15, seconds=15) + Time.objects.create(time=time) + queryset = Time.objects.annotate( + difference=ExpressionWrapper( + F('time') - Value(datetime.time(11, 15, 0), output_field=models.TimeField()), + output_field=models.DurationField(), + ) + ) + self.assertEqual(queryset.get().difference, timedelta) + + @skipUnlessDBFeature('supports_temporal_subtraction') + def test_datetime_subtraction(self): under_estimate = [e.name for e in Experiment.objects.filter(estimated_time__gt=F('end') - F('start'))] self.assertEqual(under_estimate, ['e2'])