From b1f6a4d66fd7f987a41c1e1aaa43907d9d347ba6 Mon Sep 17 00:00:00 2001 From: Karen Tracey Date: Wed, 22 Dec 2010 03:34:04 +0000 Subject: [PATCH] Fixed #10154: Allow combining F expressions with timedelta values. git-svn-id: http://code.djangoproject.com/svn/django/trunk@15018 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/db/backends/__init__.py | 9 + django/db/backends/mysql/base.py | 4 + django/db/backends/oracle/base.py | 14 ++ django/db/backends/postgresql/operations.py | 17 ++ django/db/backends/sqlite3/base.py | 32 +++ django/db/models/expressions.py | 43 +++- django/db/models/sql/expressions.py | 10 + docs/releases/1.3.txt | 3 + docs/topics/db/queries.txt | 12 +- .../expressions_regress/models.py | 13 ++ .../expressions_regress/tests.py | 207 +++++++++++++++++- 11 files changed, 360 insertions(+), 4 deletions(-) diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 23df66a762..971e3622ed 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -154,6 +154,9 @@ class BaseDatabaseFeatures(object): # deferred can_defer_constraint_checks = False + # date_interval_sql can properly handle mixed Date/DateTime fields and timedeltas + supports_mixed_date_datetime_comparisons = True + # Features that need to be confirmed at runtime # Cache whether the confirmation has been performed. _confirmed = False @@ -220,6 +223,12 @@ class BaseDatabaseOperations(object): """ raise NotImplementedError() + def date_interval_sql(self, sql, connector, timedelta): + """ + Implements the date interval functionality for expressions + """ + raise NotImplementedError() + def date_trunc_sql(self, lookup_type, field_name): """ Given a lookup_type of 'year', 'month' or 'day', returns the SQL that diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 500c1ac495..9b49a8f0ea 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -158,6 +158,10 @@ class DatabaseOperations(BaseDatabaseOperations): sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str) return sql + def date_interval_sql(self, sql, connector, timedelta): + return "(%s %s INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND)" % (sql, connector, + timedelta.days, timedelta.seconds, timedelta.microseconds) + def drop_foreignkey_sql(self): return "DROP FOREIGN KEY" diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index e10706e683..231f4d5d62 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -118,6 +118,20 @@ WHEN (new.%(col_name)s IS NULL) else: return "EXTRACT(%s FROM %s)" % (lookup_type, field_name) + def date_interval_sql(self, sql, connector, timedelta): + """ + Implements the interval functionality for expressions + format for Oracle: + (datefield + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)) + """ + minutes, seconds = divmod(timedelta.seconds, 60) + hours, minutes = divmod(minutes, 60) + days = str(timedelta.days) + day_precision = len(days) + fmt = "(%s %s INTERVAL '%s %02d:%02d:%02d.%06d' DAY(%d) TO SECOND(6))" + return fmt % (sql, connector, days, hours, minutes, seconds, + timedelta.microseconds, day_precision) + def date_trunc_sql(self, lookup_type, field_name): # Oracle uses TRUNC() for both dates and numbers. # http://download-east.oracle.com/docs/cd/B10501_01/server.920/a96540/functions155a.htm#SQLRF06151 diff --git a/django/db/backends/postgresql/operations.py b/django/db/backends/postgresql/operations.py index e8ce3f242b..83fe7c21ec 100644 --- a/django/db/backends/postgresql/operations.py +++ b/django/db/backends/postgresql/operations.py @@ -27,6 +27,23 @@ class DatabaseOperations(BaseDatabaseOperations): else: return "EXTRACT('%s' FROM %s)" % (lookup_type, field_name) + def date_interval_sql(self, sql, connector, timedelta): + """ + implements the interval functionality for expressions + format for Postgres: + (datefield + interval '3 days 200 seconds 5 microseconds') + """ + modifiers = [] + if timedelta.days: + modifiers.append(u'%s days' % timedelta.days) + if timedelta.seconds: + modifiers.append(u'%s seconds' % timedelta.seconds) + if timedelta.microseconds: + modifiers.append(u'%s microseconds' % timedelta.microseconds) + mods = u' '.join(modifiers) + conn = u' %s ' % connector + return u'(%s)' % conn.join([sql, u'interval \'%s\'' % mods]) + def date_trunc_sql(self, lookup_type, field_name): # http://www.postgresql.org/docs/8.0/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name) diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 339135c266..8344bad2a0 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -9,6 +9,7 @@ standard library. import re import sys +import datetime from django.db import utils from django.db.backends import * @@ -63,6 +64,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): test_db_allows_multiple_connections = False supports_unspecified_pk = True supports_1000_query_parameters = False + supports_mixed_date_datetime_comparisons = False def _supports_stddev(self): """Confirm support for STDDEV and related stats functions @@ -90,6 +92,16 @@ class DatabaseOperations(BaseDatabaseOperations): # cause a collision with a field name). return "django_extract('%s', %s)" % (lookup_type.lower(), field_name) + def date_interval_sql(self, sql, connector, timedelta): + # It would be more straightforward if we could use the sqlite strftime + # function, but it does not allow for keeping six digits of fractional + # second information, nor does it allow for formatting date and datetime + # values differently. So instead we register our own function that + # formats the datetime combined with the delta in a manner suitable + # for comparisons. + return u'django_format_dtdelta(%s, "%s", "%d", "%d", "%d")' % (sql, + connector, timedelta.days, timedelta.seconds, timedelta.microseconds) + def date_trunc_sql(self, lookup_type, field_name): # sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined # function django_date_trunc that's registered in connect(). Note that @@ -197,6 +209,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): self.connection.create_function("django_extract", 2, _sqlite_extract) self.connection.create_function("django_date_trunc", 2, _sqlite_date_trunc) self.connection.create_function("regexp", 2, _sqlite_regexp) + self.connection.create_function("django_format_dtdelta", 5, _sqlite_format_dtdelta) connection_created.send(sender=self.__class__, connection=self) return self.connection.cursor(factory=SQLiteCursorWrapper) @@ -260,6 +273,25 @@ def _sqlite_date_trunc(lookup_type, dt): elif lookup_type == 'day': return "%i-%02i-%02i 00:00:00" % (dt.year, dt.month, dt.day) +def _sqlite_format_dtdelta(dt, conn, days, secs, usecs): + try: + dt = util.typecast_timestamp(dt) + delta = datetime.timedelta(int(days), int(secs), int(usecs)) + if conn.strip() == '+': + dt = dt + delta + else: + dt = dt - delta + except (ValueError, TypeError): + return None + + if isinstance(dt, datetime.datetime): + rv = dt.strftime("%Y-%m-%d %H:%M:%S") + if dt.microsecond: + rv = "%s.%0.6d" % (rv, dt.microsecond) + else: + rv = dt.strftime("%Y-%m-%d") + return rv + def _sqlite_regexp(re_pattern, re_string): import re try: diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index f760e4c5f3..8b9c934bbb 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -1,4 +1,4 @@ -from datetime import datetime +import datetime from django.utils import tree from django.utils.copycompat import deepcopy @@ -26,6 +26,9 @@ class ExpressionNode(tree.Node): super(ExpressionNode, self).__init__(children, connector, negated) def _combine(self, other, connector, reversed, node=None): + if isinstance(other, datetime.timedelta): + return DateModifierNode([self, other], connector) + if reversed: obj = ExpressionNode([other], connector) obj.add(node or self, connector) @@ -111,3 +114,41 @@ class F(ExpressionNode): def evaluate(self, evaluator, qn, connection): return evaluator.evaluate_leaf(self, qn, connection) + +class DateModifierNode(ExpressionNode): + """ + Node that implements the following syntax: + filter(end_date__gt=F('start_date') + datetime.timedelta(days=3, seconds=200)) + + which translates into: + POSTGRES: + WHERE end_date > (start_date + INTERVAL '3 days 200 seconds') + + MYSQL: + WHERE end_date > (start_date + INTERVAL '3 0:0:200:0' DAY_MICROSECOND) + + ORACLE: + WHERE end_date > (start_date + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)) + + SQLITE: + WHERE end_date > django_format_dtdelta(start_date, "+" "3", "200", "0") + (A custom function is used in order to preserve six digits of fractional + second information on sqlite, and to format both date and datetime values.) + + Note that microsecond comparisons are not well supported with MySQL, since + MySQL does not store microsecond information. + + Only adding and subtracting timedeltas is supported, attempts to use other + operations raise a TypeError. + """ + def __init__(self, children, connector, negated=False): + if len(children) != 2: + raise TypeError('Must specify a node and a timedelta.') + if not isinstance(children[1], datetime.timedelta): + raise TypeError('Second child must be a timedelta.') + if connector not in (self.ADD, self.SUB): + raise TypeError('Connector must be + or -, not %s' % connector) + super(DateModifierNode, self).__init__(children, connector, negated) + + def evaluate(self, evaluator, qn, connection): + return evaluator.evaluate_date_modifier_node(self, qn, connection) diff --git a/django/db/models/sql/expressions.py b/django/db/models/sql/expressions.py index fffbba085c..1bbf742b5c 100644 --- a/django/db/models/sql/expressions.py +++ b/django/db/models/sql/expressions.py @@ -85,3 +85,13 @@ class SQLEvaluator(object): return col.as_sql(qn, connection), () else: return '%s.%s' % (qn(col[0]), qn(col[1])), () + + def evaluate_date_modifier_node(self, node, qn, connection): + timedelta = node.children.pop() + sql, params = self.evaluate_node(node, qn, connection) + + if timedelta.days == 0 and timedelta.seconds == 0 and \ + timedelta.microseconds == 0: + return sql, params + + return connection.ops.date_interval_sql(sql, node.connector, timedelta), params diff --git a/docs/releases/1.3.txt b/docs/releases/1.3.txt index 0bf212452e..7992c95e5f 100644 --- a/docs/releases/1.3.txt +++ b/docs/releases/1.3.txt @@ -235,6 +235,9 @@ requests. These include: providing a :class:`~django.template.RequestContext` by default. + * Support for combining :ref:`F() expressions ` + with timedelta values when retrieving or updating database values. + .. _HTTPOnly: http://www.owasp.org/index.php/HTTPOnly .. _backwards-incompatible-changes-1.3: diff --git a/docs/topics/db/queries.txt b/docs/topics/db/queries.txt index 2141aa6e91..cbbf073d39 100644 --- a/docs/topics/db/queries.txt +++ b/docs/topics/db/queries.txt @@ -36,6 +36,7 @@ models, which comprise a Weblog application: headline = models.CharField(max_length=255) body_text = models.TextField() pub_date = models.DateTimeField() + mod_date = models.DateTimeField() authors = models.ManyToManyField(Author) n_comments = models.IntegerField() n_pingbacks = models.IntegerField() @@ -566,10 +567,19 @@ You can also use the double underscore notation to span relationships in an ``F()`` object. An ``F()`` object with a double underscore will introduce any joins needed to access the related object. For example, to retrieve all the entries where the author's name is the same as the blog name, we could -issue the query: +issue the query:: >>> Entry.objects.filter(authors__name=F('blog__name')) +.. versionadded:: 1.3 + +For date and date/time fields, you can add or subtract a ``datetime.timedelta`` +object. The following would return all entries that were modified more than 3 days +after they were published:: + + >>> from datetime import timedelta + >>> Entry.objects.filter(mod_date__gt=F('pub_date') + timedelta(days=3)) + The pk lookup shortcut ---------------------- diff --git a/tests/regressiontests/expressions_regress/models.py b/tests/regressiontests/expressions_regress/models.py index f2997ee5be..56c3746b91 100644 --- a/tests/regressiontests/expressions_regress/models.py +++ b/tests/regressiontests/expressions_regress/models.py @@ -10,3 +10,16 @@ class Number(models.Model): def __unicode__(self): return u'%i, %.3f' % (self.integer, self.float) +class Experiment(models.Model): + name = models.CharField(max_length=24) + assigned = models.DateField() + completed = models.DateField() + start = models.DateTimeField() + end = models.DateTimeField() + + class Meta: + ordering = ('name',) + + def duration(self): + return self.end - self.start + diff --git a/tests/regressiontests/expressions_regress/tests.py b/tests/regressiontests/expressions_regress/tests.py index 67632f48bb..a900c6c5fd 100644 --- a/tests/regressiontests/expressions_regress/tests.py +++ b/tests/regressiontests/expressions_regress/tests.py @@ -1,12 +1,14 @@ """ Spanning tests for all the operations that F() expressions can perform. """ +import datetime + from django.conf import settings -from django.db import models, DEFAULT_DB_ALIAS +from django.db import models, connection from django.db.models import F from django.test import TestCase, Approximate, skipUnlessDBFeature -from regressiontests.expressions_regress.models import Number +from regressiontests.expressions_regress.models import Number, Experiment class ExpressionsRegressTests(TestCase): @@ -193,3 +195,204 @@ class ExpressionOperatorTests(TestCase): self.assertEqual(Number.objects.get(pk=self.n.pk).integer, 47) self.assertEqual(Number.objects.get(pk=self.n.pk).float, Approximate(15.500, places=3)) +class FTimeDeltaTests(TestCase): + + def setUp(self): + sday = datetime.date(2010, 6, 25) + stime = datetime.datetime(2010, 6, 25, 12, 15, 30, 747000) + midnight = datetime.time(0) + + delta0 = datetime.timedelta(0) + delta1 = datetime.timedelta(microseconds=253000) + delta2 = datetime.timedelta(seconds=44) + delta3 = datetime.timedelta(hours=21, minutes=8) + delta4 = datetime.timedelta(days=10) + + # Test data is set so that deltas and delays will be + # strictly increasing. + self.deltas = [] + self.delays = [] + self.days_long = [] + + # e0: started same day as assigned, zero duration + end = stime+delta0 + e0 = Experiment.objects.create(name='e0', assigned=sday, start=stime, + end=end, completed=end.date()) + self.deltas.append(delta0) + self.delays.append(e0.start- + datetime.datetime.combine(e0.assigned, midnight)) + self.days_long.append(e0.completed-e0.assigned) + + # e1: started one day after assigned, tiny duration, data + # set so that end time has no fractional seconds, which + # tests an edge case on sqlite. This Experiment is only + # included in the test data when the DB supports microsecond + # precision. + if connection.features.supports_microsecond_precision: + delay = datetime.timedelta(1) + end = stime + delay + delta1 + e1 = Experiment.objects.create(name='e1', assigned=sday, + start=stime+delay, end=end, completed=end.date()) + self.deltas.append(delta1) + self.delays.append(e1.start- + datetime.datetime.combine(e1.assigned, midnight)) + self.days_long.append(e1.completed-e1.assigned) + + # e2: started three days after assigned, small duration + end = stime+delta2 + e2 = Experiment.objects.create(name='e2', + assigned=sday-datetime.timedelta(3), start=stime, end=end, + completed=end.date()) + self.deltas.append(delta2) + self.delays.append(e2.start- + datetime.datetime.combine(e2.assigned, midnight)) + self.days_long.append(e2.completed-e2.assigned) + + # e3: started four days after assigned, medium duration + delay = datetime.timedelta(4) + end = stime + delay + delta3 + e3 = Experiment.objects.create(name='e3', + assigned=sday, start=stime+delay, end=end, completed=end.date()) + self.deltas.append(delta3) + self.delays.append(e3.start- + datetime.datetime.combine(e3.assigned, midnight)) + self.days_long.append(e3.completed-e3.assigned) + + # e4: started 10 days after assignment, long duration + end = stime + delta4 + e4 = Experiment.objects.create(name='e4', + assigned=sday-datetime.timedelta(10), start=stime, end=end, + completed=end.date()) + self.deltas.append(delta4) + self.delays.append(e4.start- + datetime.datetime.combine(e4.assigned, midnight)) + self.days_long.append(e4.completed-e4.assigned) + self.expnames = [e.name for e in Experiment.objects.all()] + + def test_delta_add(self): + for i in range(len(self.deltas)): + delta = self.deltas[i] + test_set = [e.name for e in + Experiment.objects.filter(end__lt=F('start')+delta)] + self.assertEqual(test_set, self.expnames[:i]) + + test_set = [e.name for e in + Experiment.objects.filter(end__lte=F('start')+delta)] + self.assertEqual(test_set, self.expnames[:i+1]) + + def test_delta_subtract(self): + for i in range(len(self.deltas)): + delta = self.deltas[i] + test_set = [e.name for e in + Experiment.objects.filter(start__gt=F('end')-delta)] + self.assertEqual(test_set, self.expnames[:i]) + + test_set = [e.name for e in + Experiment.objects.filter(start__gte=F('end')-delta)] + self.assertEqual(test_set, self.expnames[:i+1]) + + def test_exclude(self): + for i in range(len(self.deltas)): + delta = self.deltas[i] + test_set = [e.name for e in + Experiment.objects.exclude(end__lt=F('start')+delta)] + self.assertEqual(test_set, self.expnames[i:]) + + test_set = [e.name for e in + Experiment.objects.exclude(end__lte=F('start')+delta)] + self.assertEqual(test_set, self.expnames[i+1:]) + + def test_date_comparison(self): + for i in range(len(self.days_long)): + days = self.days_long[i] + test_set = [e.name for e in + Experiment.objects.filter(completed__lt=F('assigned')+days)] + self.assertEqual(test_set, self.expnames[:i]) + + test_set = [e.name for e in + Experiment.objects.filter(completed__lte=F('assigned')+days)] + self.assertEqual(test_set, self.expnames[:i+1]) + + @skipUnlessDBFeature("supports_mixed_date_datetime_comparisons") + def test_mixed_comparisons1(self): + for i in range(len(self.delays)): + delay = self.delays[i] + if not connection.features.supports_microsecond_precision: + delay = datetime.timedelta(delay.days, delay.seconds) + test_set = [e.name for e in + Experiment.objects.filter(assigned__gt=F('start')-delay)] + self.assertEqual(test_set, self.expnames[:i]) + + test_set = [e.name for e in + Experiment.objects.filter(assigned__gte=F('start')-delay)] + self.assertEqual(test_set, self.expnames[:i+1]) + + def test_mixed_comparisons2(self): + delays = [datetime.timedelta(delay.days) for delay in self.delays] + for i in range(len(delays)): + delay = delays[i] + test_set = [e.name for e in + Experiment.objects.filter(start__lt=F('assigned')+delay)] + self.assertEqual(test_set, self.expnames[:i]) + + test_set = [e.name for e in + Experiment.objects.filter(start__lte=F('assigned')+delay+ + datetime.timedelta(1))] + self.assertEqual(test_set, self.expnames[:i+1]) + + def test_delta_update(self): + for i in range(len(self.deltas)): + delta = self.deltas[i] + exps = Experiment.objects.all() + expected_durations = [e.duration() for e in exps] + expected_starts = [e.start+delta for e in exps] + expected_ends = [e.end+delta for e in exps] + + Experiment.objects.update(start=F('start')+delta, end=F('end')+delta) + exps = Experiment.objects.all() + new_starts = [e.start for e in exps] + new_ends = [e.end for e in exps] + new_durations = [e.duration() for e in exps] + self.assertEqual(expected_starts, new_starts) + self.assertEqual(expected_ends, new_ends) + self.assertEqual(expected_durations, new_durations) + + def test_delta_invalid_op_mult(self): + raised = False + try: + r = repr(Experiment.objects.filter(end__lt=F('start')*self.deltas[0])) + except TypeError: + raised = True + self.assertTrue(raised, "TypeError not raised on attempt to multiply datetime by timedelta.") + + def test_delta_invalid_op_div(self): + raised = False + try: + r = repr(Experiment.objects.filter(end__lt=F('start')/self.deltas[0])) + except TypeError: + raised = True + self.assertTrue(raised, "TypeError not raised on attempt to divide datetime by timedelta.") + + def test_delta_invalid_op_mod(self): + raised = False + try: + r = repr(Experiment.objects.filter(end__lt=F('start')%self.deltas[0])) + except TypeError: + raised = True + self.assertTrue(raised, "TypeError not raised on attempt to modulo divide datetime by timedelta.") + + def test_delta_invalid_op_and(self): + raised = False + try: + r = repr(Experiment.objects.filter(end__lt=F('start')&self.deltas[0])) + except TypeError: + raised = True + self.assertTrue(raised, "TypeError not raised on attempt to binary and a datetime with a timedelta.") + + def test_delta_invalid_op_or(self): + raised = False + try: + r = repr(Experiment.objects.filter(end__lt=F('start')|self.deltas[0])) + except TypeError: + raised = True + self.assertTrue(raised, "TypeError not raised on attempt to binary or a datetime with a timedelta.")