p31829507
/
django
mirror of https://github.com/django/django.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refs #25774, #26348 -- Allowed Trunc functions to operate with time fields. 

Thanks Josh for the amazing testing setup and Tim for the review.
This commit is contained in:
Simon Charette 2016-06-18 23:38:24 -04:00
parent 90468079ec
commit 082c52dbed
11 changed files with 245 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
django/db/backends/base/operations.py
View File

@ -113,6 +113,14 @@ class BaseDatabaseOperations(object):
"""
raise NotImplementedError('subclasses of BaseDatabaseOperations may require a datetime_trunk_sql() method')
def time_trunc_sql(self, lookup_type, field_name):
"""
Given a lookup_type of 'hour', 'minute' or 'second', returns the SQL
that truncates the given time field field_name to a time object with
only the given specificity.
"""
raise NotImplementedError('subclasses of BaseDatabaseOperations may require a time_trunc_sql() method')
def time_extract_sql(self, lookup_type, field_name):
"""
Given a lookup_type of 'hour', 'minute' or 'second', returns the SQL

12
django/db/backends/mysql/operations.py
View File

@ -70,6 +70,18 @@ class DatabaseOperations(BaseDatabaseOperations):
sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str)
return sql, params
def time_trunc_sql(self, lookup_type, field_name):
fields = {
'hour': '%%H:00:00',
'minute': '%%H:%%i:00',
'second': '%%H:%%i:%%s',
} # Use double percents to escape.
if lookup_type in fields:
format_str = fields[lookup_type]
return "CAST(DATE_FORMAT(%s, '%s') AS TIME)" % (field_name, format_str)
else:
return "TIME(%s)" % (field_name)
def date_interval_sql(self, timedelta):
return "INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND" % (
timedelta.days, timedelta.seconds, timedelta.microseconds), []

12
django/db/backends/oracle/operations.py
View File

@ -148,6 +148,18 @@ WHEN (new.%(col_name)s IS NULL)
sql = "CAST(%s AS DATE)" % field_name # Cast to DATE removes sub-second precision.
return sql, []
def time_trunc_sql(self, lookup_type, field_name):
# The implementation is similar to `datetime_trunc_sql` as both
# `DateTimeField` and `TimeField` are stored as TIMESTAMP where
# the date part of the later is ignored.
if lookup_type == 'hour':
sql = "TRUNC(%s, 'HH24')" % field_name
elif lookup_type == 'minute':
sql = "TRUNC(%s, 'MI')" % field_name
elif lookup_type == 'second':
sql = "CAST(%s AS DATE)" % field_name # Cast to DATE removes sub-second precision.
return sql
def get_db_converters(self, expression):
converters = super(DatabaseOperations, self).get_db_converters(expression)
internal_type = expression.output_field.get_internal_type()

3
django/db/backends/postgresql/operations.py
View File

@ -56,6 +56,9 @@ class DatabaseOperations(BaseDatabaseOperations):
sql = "DATE_TRUNC('%s', %s)" % (lookup_type, field_name)
return sql, params
def time_trunc_sql(self, lookup_type, field_name):
return "DATE_TRUNC('%s', %s)::time" % (lookup_type, field_name)
def deferrable_sql(self):
return " DEFERRABLE INITIALLY DEFERRED"

14
django/db/backends/sqlite3/base.py
View File

@ -213,6 +213,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
conn.create_function("django_datetime_extract", 3, _sqlite_datetime_extract)
conn.create_function("django_datetime_trunc", 3, _sqlite_datetime_trunc)
conn.create_function("django_time_extract", 2, _sqlite_time_extract)
conn.create_function("django_time_trunc", 2, _sqlite_time_trunc)
conn.create_function("django_time_diff", 2, _sqlite_time_diff)
conn.create_function("django_timestamp_diff", 2, _sqlite_timestamp_diff)
conn.create_function("regexp", 2, _sqlite_regexp)
@ -370,6 +371,19 @@ def _sqlite_date_trunc(lookup_type, dt):
return "%i-%02i-%02i" % (dt.year, dt.month, dt.day)
def _sqlite_time_trunc(lookup_type, dt):
try:
dt = backend_utils.typecast_time(dt)
except (ValueError, TypeError):
return None
if lookup_type == 'hour':
return "%02i:00:00" % dt.hour
elif lookup_type == 'minute':
return "%02i:%02i:00" % (dt.hour, dt.minute)
elif lookup_type == 'second':
return "%02i:%02i:%02i" % (dt.hour, dt.minute, dt.second)
def _sqlite_datetime_parse(dt, tzname):
if dt is None:
return None

7
django/db/backends/sqlite3/operations.py
View File

@ -70,6 +70,13 @@ class DatabaseOperations(BaseDatabaseOperations):
# cause a collision with a field name).
return "django_date_trunc('%s', %s)" % (lookup_type.lower(), field_name)
def time_trunc_sql(self, lookup_type, field_name):
# sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined
# function django_date_trunc that's registered in connect(). Note that
# single quotes are used because this is a string (and could otherwise
# cause a collision with a field name).
return "django_time_trunc('%s', %s)" % (lookup_type.lower(), field_name)
def _require_pytz(self):
if settings.USE_TZ and pytz is None:
raise ImproperlyConfigured("This query requires pytz, but it isn't installed.")

49
django/db/models/functions/datetime.py
View File

@ -151,26 +151,38 @@ class TruncBase(TimezoneMixin, Transform):
elif isinstance(self.output_field, DateField):
sql = connection.ops.date_trunc_sql(self.kind, inner_sql)
params = []
elif isinstance(self.output_field, TimeField):
sql = connection.ops.time_trunc_sql(self.kind, inner_sql)
params = []
else:
raise ValueError('Trunc only valid on DateField or DateTimeField.')
raise ValueError('Trunc only valid on DateField, TimeField, or DateTimeField.')
return sql, inner_params + params
def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
copy = super(TruncBase, self).resolve_expression(query, allow_joins, reuse, summarize, for_save)
field = copy.lhs.output_field
# DateTimeField is a subclass of DateField so this works for both.
assert isinstance(field, DateField), (
"%r isn't a DateField or DateTimeField." % field.name
assert isinstance(field, (DateField, TimeField)), (
"%r isn't a DateField, TimeField, or DateTimeField." % field.name
)
# If self.output_field was None, then accessing the field will trigger
# the resolver to assign it to self.lhs.output_field.
if not isinstance(copy.output_field, (DateField, DateTimeField)):
raise ValueError('output_field must be either DateField or DateTimeField')
# Passing dates to functions expecting datetimes is most likely a
# mistake.
if not isinstance(copy.output_field, (DateField, DateTimeField, TimeField)):
raise ValueError('output_field must be either DateField, TimeField, or DateTimeField')
# Passing dates or times to functions expecting datetimes is most
# likely a mistake.
output_field = copy.output_field
explicit_output_field = field.__class__ != copy.output_field.__class__
if type(field) == DateField and (
isinstance(copy.output_field, DateTimeField) or copy.kind in ('hour', 'minute', 'second')):
raise ValueError("Cannot truncate DateField '%s' to DateTimeField. " % field.name)
isinstance(output_field, DateTimeField) or copy.kind in ('hour', 'minute', 'second', 'time')):
raise ValueError("Cannot truncate DateField '%s' to %s. " % (
field.name, output_field.__class__.__name__ if explicit_output_field else 'DateTimeField'
))
elif isinstance(field, TimeField) and (
isinstance(output_field, DateTimeField) or copy.kind in ('year', 'month', 'day', 'date')):
raise ValueError("Cannot truncate TimeField '%s' to %s. " % (
field.name, output_field.__class__.__name__ if explicit_output_field else 'DateTimeField'
))
return copy
def convert_value(self, value, expression, connection, context):
@ -184,8 +196,10 @@ class TruncBase(TimezoneMixin, Transform):
value = value.replace(tzinfo=None)
value = timezone.make_aware(value, self.tzinfo)
elif isinstance(value, datetime):
# self.output_field is definitely a DateField here.
value = value.date()
if isinstance(self.output_field, DateField):
value = value.date()
elif isinstance(self.output_field, TimeField):
value = value.time()
return value
@ -209,6 +223,7 @@ class TruncDay(TruncBase):
class TruncDate(TruncBase):
kind = 'date'
lookup_name = 'date'
@cached_property
@ -227,25 +242,13 @@ class TruncDate(TruncBase):
class TruncHour(TruncBase):
kind = 'hour'
@cached_property
def output_field(self):
return DateTimeField()
class TruncMinute(TruncBase):
kind = 'minute'
@cached_property
def output_field(self):
return DateTimeField()
class TruncSecond(TruncBase):
kind = 'second'
@cached_property
def output_field(self):
return DateTimeField()
DateTimeField.register_lookup(TruncDate)

72
docs/ref/models/database-functions.txt
View File

@ -288,8 +288,10 @@ We'll be using the following model in examples of each function::
class Experiment(models.Model):
start_datetime = models.DateTimeField()
start_date = models.DateField(null=True, blank=True)
start_time = models.TimeField(null=True, blank=True)
end_datetime = models.DateTimeField(null=True, blank=True)
end_date = models.DateField(null=True, blank=True)
end_time = models.TimeField(null=True, blank=True)
``Extract``
-----------
@ -500,13 +502,14 @@ but not the exact second, then ``Trunc`` (and its subclasses) can be useful to
filter or aggregate your data. For example, you can use ``Trunc`` to calculate
the number of sales per day.
``Trunc`` takes a single ``expression``, representing a ``DateField`` or
``DateTimeField``, a ``kind`` representing a date part, and an ``output_field``
that's either ``DateTimeField()`` or ``DateField()``. It returns a datetime or
date, depending on ``output_field``, with fields up to ``kind`` set to their
minimum value. If ``output_field`` is omitted, it will default to the
``output_field`` of ``expression``. A ``tzinfo`` subclass, usually provided by
``pytz``, can be passed to truncate a value in a specific timezone.
``Trunc`` takes a single ``expression``, representing a ``DateField``,
``TimeField``, or ``DateTimeField``, a ``kind`` representing a date or time
part, and an ``output_field`` that's either ``DateTimeField()``,
``TimeField()``, or ``DateField()``. It returns a datetime, date, or time
depending on ``output_field``, with fields up to ``kind`` set to their minimum
value. If ``output_field`` is omitted, it will default to the ``output_field``
of ``expression``. A ``tzinfo`` subclass, usually provided by ``pytz``, can be
passed to truncate a value in a specific timezone.
Given the datetime ``2015-06-15 14:30:50.000321+00:00``, the built-in ``kind``\s
return:
@ -616,6 +619,61 @@ that deal with date-parts can be used with ``DateField``::
2016-01-01 00:00:00+11:00 1
2014-06-01 00:00:00+10:00 1
``TimeField`` truncation
~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 1.11
.. class:: TruncHour(expression, output_field=None, tzinfo=None, **extra)
.. attribute:: kind = 'hour'
.. class:: TruncMinute(expression, output_field=None, tzinfo=None, **extra)
.. attribute:: kind = 'minute'
.. class:: TruncSecond(expression, output_field=None, tzinfo=None, **extra)
.. attribute:: kind = 'second'
These are logically equivalent to ``Trunc('time_field', kind)``. They truncate
all parts of the time up to ``kind`` which allows grouping or filtering times
with less precision. ``expression`` can have an ``output_field`` of either
``TimeField`` or ``DateTimeField``.
Since ``TimeField``\s don't have a date component, only ``Trunc`` subclasses
that deal with time-parts can be used with ``TimeField``::
>>> from datetime import datetime
>>> from django.db.models import Count, TimeField
>>> from django.db.models.functions import TruncHour
>>> from django.utils import timezone
>>> start1 = datetime(2014, 6, 15, 14, 30, 50, 321, tzinfo=timezone.utc)
>>> start2 = datetime(2014, 6, 15, 14, 40, 2, 123, tzinfo=timezone.utc)
>>> start3 = datetime(2015, 12, 31, 17, 5, 27, 999, tzinfo=timezone.utc)
>>> Experiment.objects.create(start_datetime=start1, start_time=start1.time())
>>> Experiment.objects.create(start_datetime=start2, start_time=start2.time())
>>> Experiment.objects.create(start_datetime=start3, start_time=start3.time())
>>> experiments_per_hour = Experiment.objects.annotate(
... hour=TruncHour('start_datetime', output_field=TimeField()),
... ).values('hour').annotate(experiments=Count('id'))
>>> for exp in experiments_per_hour:
... print(exp['hour'], exp['experiments'])
...
14:00:00 2
17:00:00 1
>>> import pytz
>>> melb = pytz.timezone('Australia/Melbourne')
>>> experiments_per_hour = Experiment.objects.annotate(
... hour=TruncHour('start_datetime', tzinfo=melb),
... ).values('hour').annotate(experiments=Count('id'))
>>> for exp in experiments_per_hour:
... print(exp['hour'], exp['experiments'])
...
2014-06-16 00:00:00+10:00 2
2016-01-01 04:00:00+11:00 1
``DateTimeField`` truncation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

10
docs/releases/1.11.txt
View File

@ -198,6 +198,9 @@ Models
* :class:`~django.db.models.ImageField` now has a default
:data:`~django.core.validators.validate_image_file_extension` validator.
* Added support for time truncation to
:class:`~django.db.models.functions.datetime.Trunc` functions.
Requests and Responses
~~~~~~~~~~~~~~~~~~~~~~
@ -263,7 +266,12 @@ Backwards incompatible changes in 1.11
Database backend API
--------------------
* ...
* The ``DatabaseOperations.time_trunc_sql()`` method is added to support
``TimeField`` truncation. It accepts a ``lookup_type`` and ``field_name``
arguments and returns the appropriate SQL to truncate the given time field
``field_name`` to a time object with only the given specificity. The
``lookup_type`` argument can be either ``'hour'``, ``'minute'``, or
``'second'``.
Dropped support for PostgreSQL 9.2 and PostGIS 2.0
--------------------------------------------------

91
tests/db_functions/test_datetime.py
View File

@ -5,7 +5,7 @@ from unittest import skipIf
from django.conf import settings
from django.db import connection
from django.db.models import DateField, DateTimeField, IntegerField
from django.db.models import DateField, DateTimeField, IntegerField, TimeField
from django.db.models.functions import (
Extract, ExtractDay, ExtractHour, ExtractMinute, ExtractMonth,
ExtractSecond, ExtractWeekDay, ExtractYear, Trunc, TruncDate, TruncDay,
@ -353,18 +353,25 @@ class DateFunctionTests(TestCase):
self.create_model(start_datetime, end_datetime)
self.create_model(end_datetime, start_datetime)
with self.assertRaisesMessage(ValueError, 'output_field must be either DateField or DateTimeField'):
msg = 'output_field must be either DateField, TimeField, or DateTimeField'
with self.assertRaisesMessage(ValueError, msg):
list(DTModel.objects.annotate(truncated=Trunc('start_datetime', 'year', output_field=IntegerField())))
with self.assertRaisesMessage(AssertionError, "'name' isn't a DateField or DateTimeField."):
with self.assertRaisesMessage(AssertionError, "'name' isn't a DateField, TimeField, or DateTimeField."):
list(DTModel.objects.annotate(truncated=Trunc('name', 'year', output_field=DateTimeField())))
with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
list(DTModel.objects.annotate(truncated=Trunc('start_date', 'second')))
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=Trunc('start_time', 'month')))
with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
list(DTModel.objects.annotate(truncated=Trunc('start_date', 'month', output_field=DateTimeField())))
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=Trunc('start_time', 'second', output_field=DateTimeField())))
def test_datetime_kind(kind):
self.assertQuerysetEqual(
DTModel.objects.annotate(
@ -389,9 +396,24 @@ class DateFunctionTests(TestCase):
lambda m: (m.start_datetime, m.truncated)
)
def test_time_kind(kind):
self.assertQuerysetEqual(
DTModel.objects.annotate(
truncated=Trunc('start_time', kind, output_field=TimeField())
).order_by('start_datetime'),
[
(start_datetime, truncate_to(start_datetime.time(), kind)),
(end_datetime, truncate_to(end_datetime.time(), kind))
],
lambda m: (m.start_datetime, m.truncated)
)
test_date_kind('year')
test_date_kind('month')
test_date_kind('day')
test_time_kind('hour')
test_time_kind('minute')
test_time_kind('second')
test_datetime_kind('year')
test_datetime_kind('month')
test_datetime_kind('day')
@ -428,6 +450,12 @@ class DateFunctionTests(TestCase):
)
self.assertEqual(DTModel.objects.filter(start_datetime=TruncYear('start_datetime')).count(), 1)
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=TruncYear('start_time')))
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=TruncYear('start_time', output_field=TimeField())))
def test_trunc_month_func(self):
start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'month')
@ -454,6 +482,12 @@ class DateFunctionTests(TestCase):
)
self.assertEqual(DTModel.objects.filter(start_datetime=TruncMonth('start_datetime')).count(), 1)
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=TruncMonth('start_time')))
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=TruncMonth('start_time', output_field=TimeField())))
def test_trunc_date_func(self):
start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
end_datetime = microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123))
@ -472,6 +506,12 @@ class DateFunctionTests(TestCase):
)
self.assertEqual(DTModel.objects.filter(start_datetime__date=TruncDate('start_datetime')).count(), 2)
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateField"):
list(DTModel.objects.annotate(truncated=TruncDate('start_time')))
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateField"):
list(DTModel.objects.annotate(truncated=TruncDate('start_time', output_field=TimeField())))
def test_trunc_day_func(self):
start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'day')
@ -490,6 +530,12 @@ class DateFunctionTests(TestCase):
)
self.assertEqual(DTModel.objects.filter(start_datetime=TruncDay('start_datetime')).count(), 1)
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=TruncDay('start_time')))
with self.assertRaisesMessage(ValueError, "Cannot truncate TimeField 'start_time' to DateTimeField"):
list(DTModel.objects.annotate(truncated=TruncDay('start_time', output_field=TimeField())))
def test_trunc_hour_func(self):
start_datetime = microsecond_support(datetime(2015, 6, 15, 14, 30, 50, 321))
end_datetime = truncate_to(microsecond_support(datetime(2016, 6, 15, 14, 10, 50, 123)), 'hour')
@ -506,6 +552,14 @@ class DateFunctionTests(TestCase):
],
lambda m: (m.start_datetime, m.extracted)
)
self.assertQuerysetEqual(
DTModel.objects.annotate(extracted=TruncHour('start_time')).order_by('start_datetime'),
[
(start_datetime, truncate_to(start_datetime.time(), 'hour')),
(end_datetime, truncate_to(end_datetime.time(), 'hour')),
],
lambda m: (m.start_datetime, m.extracted)
)
self.assertEqual(DTModel.objects.filter(start_datetime=TruncHour('start_datetime')).count(), 1)
with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
@ -530,6 +584,14 @@ class DateFunctionTests(TestCase):
],
lambda m: (m.start_datetime, m.extracted)
)
self.assertQuerysetEqual(
DTModel.objects.annotate(extracted=TruncMinute('start_time')).order_by('start_datetime'),
[
(start_datetime, truncate_to(start_datetime.time(), 'minute')),
(end_datetime, truncate_to(end_datetime.time(), 'minute')),
],
lambda m: (m.start_datetime, m.extracted)
)
self.assertEqual(DTModel.objects.filter(start_datetime=TruncMinute('start_datetime')).count(), 1)
with self.assertRaisesMessage(ValueError, "Cannot truncate DateField 'start_date' to DateTimeField"):
@ -554,6 +616,14 @@ class DateFunctionTests(TestCase):
],
lambda m: (m.start_datetime, m.extracted)
)
self.assertQuerysetEqual(
DTModel.objects.annotate(extracted=TruncSecond('start_time')).order_by('start_datetime'),
[
(start_datetime, truncate_to(start_datetime.time(), 'second')),
(end_datetime, truncate_to(end_datetime.time(), 'second'))
],
lambda m: (m.start_datetime, m.extracted)
)
result = 1 if connection.features.supports_microsecond_precision else 2
self.assertEqual(DTModel.objects.filter(start_datetime=TruncSecond('start_datetime')).count(), result)
@ -680,9 +750,24 @@ class DateFunctionWithTimeZoneTests(DateFunctionTests):
lambda m: (m.start_datetime, m.truncated)
)
def test_time_kind(kind, tzinfo=melb):
self.assertQuerysetEqual(
DTModel.objects.annotate(
truncated=Trunc('start_time', kind, output_field=TimeField(), tzinfo=melb)
).order_by('start_datetime'),
[
(start_datetime, truncate_to(start_datetime.time(), kind)),
(end_datetime, truncate_to(end_datetime.time(), kind))
],
lambda m: (m.start_datetime, m.truncated)
)
test_date_kind('year')
test_date_kind('month')
test_date_kind('day')
test_time_kind('hour')
test_time_kind('minute')
test_time_kind('second')
test_datetime_kind('year')
test_datetime_kind('month')
test_datetime_kind('day')

2
tests/queries/tests.py
View File

@ -1312,7 +1312,7 @@ class Queries3Tests(BaseQuerysetTest):
def test_ticket8683(self):
# An error should be raised when QuerySet.datetimes() is passed the
# wrong type of field.
with self.assertRaisesMessage(AssertionError, "'name' isn't a DateField or DateTimeField."):
with self.assertRaisesMessage(AssertionError, "'name' isn't a DateField, TimeField, or DateTimeField."):
Item.objects.datetimes('name', 'month')
def test_ticket22023(self):