diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index bdf841cbd9..104744b220 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -1,5 +1,6 @@ from collections import deque import datetime +import decimal import time import warnings @@ -18,6 +19,7 @@ from django.db.backends.signals import connection_created from django.db.backends import utils from django.db.transaction import TransactionManagementError from django.db.utils import DatabaseError, DatabaseErrorWrapper, ProgrammingError +from django.utils.dateparse import parse_duration from django.utils.deprecation import RemovedInDjango19Warning from django.utils.functional import cached_property from django.utils import six @@ -575,6 +577,9 @@ class BaseDatabaseFeatures(object): supports_binary_field = True + # Is there a true datatype for timedeltas? + has_native_duration_field = False + # Do time/datetime fields have microsecond precision? supports_microsecond_precision = True @@ -1251,8 +1256,16 @@ class BaseDatabaseOperations(object): Some field types on some backends do not provide data in the correct format, this is the hook for coverter functions. """ + if not self.connection.features.has_native_duration_field and internal_type == 'DurationField': + return [self.convert_durationfield_value] return [] + def convert_durationfield_value(self, value, field): + if value is not None: + value = str(decimal.Decimal(value) / decimal.Decimal(1000000)) + value = parse_duration(value) + return value + def check_aggregate_support(self, aggregate_func): """Check that the backend supports the provided aggregate @@ -1272,6 +1285,9 @@ class BaseDatabaseOperations(object): conn = ' %s ' % connector return conn.join(sub_expressions) + def combine_duration_expression(self, connector, sub_expressions): + return self.combine_expression(connector, sub_expressions) + def modify_insert_params(self, placeholders, params): """Allow modification of insert parameters. Needed for Oracle Spatial backend due to #10888. diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 566ce6c0a7..8fcc9e0db1 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -289,9 +289,12 @@ class DatabaseOperations(BaseDatabaseOperations): sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str) return sql, params - def date_interval_sql(self, sql, connector, timedelta): - return "(%s %s INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND)" % (sql, connector, - timedelta.days, timedelta.seconds, timedelta.microseconds) + def date_interval_sql(self, timedelta): + return "INTERVAL '%d 0:0:%d:%d' DAY_MICROSECOND" % ( + timedelta.days, timedelta.seconds, timedelta.microseconds), [] + + def format_for_duration_arithmetic(self, sql): + return 'INTERVAL %s MICROSECOND' % sql def drop_foreignkey_sql(self): return "DROP FOREIGN KEY" diff --git a/django/db/backends/mysql/creation.py b/django/db/backends/mysql/creation.py index 40d9251235..efa0ea06e7 100644 --- a/django/db/backends/mysql/creation.py +++ b/django/db/backends/mysql/creation.py @@ -16,6 +16,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'date', 'DateTimeField': 'datetime', 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)', + 'DurationField': 'bigint', 'FileField': 'varchar(%(max_length)s)', 'FilePathField': 'varchar(%(max_length)s)', 'FloatField': 'double precision', diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 51abca9137..db64803c0a 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -198,19 +198,22 @@ WHEN (new.%(col_name)s IS NULL) # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions050.htm return "EXTRACT(%s FROM %s)" % (lookup_type.upper(), field_name) - def date_interval_sql(self, sql, connector, timedelta): + def date_interval_sql(self, timedelta): """ Implements the interval functionality for expressions format for Oracle: - (datefield + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)) + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6) """ minutes, seconds = divmod(timedelta.seconds, 60) hours, minutes = divmod(minutes, 60) days = str(timedelta.days) day_precision = len(days) - fmt = "(%s %s INTERVAL '%s %02d:%02d:%02d.%06d' DAY(%d) TO SECOND(6))" - return fmt % (sql, connector, days, hours, minutes, seconds, - timedelta.microseconds, day_precision) + fmt = "INTERVAL '%s %02d:%02d:%02d.%06d' DAY(%d) TO SECOND(6)" + return fmt % (days, hours, minutes, seconds, timedelta.microseconds, + day_precision), [] + + def format_for_duration_arithmetic(self, sql): + return "NUMTODSINTERVAL(%s / 100000, 'SECOND')" % sql def date_trunc_sql(self, lookup_type, field_name): # http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions230.htm#i1002084 diff --git a/django/db/backends/oracle/creation.py b/django/db/backends/oracle/creation.py index d470d5ac7c..ae4f5f42ec 100644 --- a/django/db/backends/oracle/creation.py +++ b/django/db/backends/oracle/creation.py @@ -29,6 +29,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'DATE', 'DateTimeField': 'TIMESTAMP', 'DecimalField': 'NUMBER(%(max_digits)s, %(decimal_places)s)', + 'DurationField': 'NUMBER(19)', 'FileField': 'NVARCHAR2(%(max_length)s)', 'FilePathField': 'NVARCHAR2(%(max_length)s)', 'FloatField': 'DOUBLE PRECISION', diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index 66200c0064..5fb5d68b5b 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -47,6 +47,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): needs_datetime_string_cast = False can_return_id_from_insert = True has_real_datatype = True + has_native_duration_field = True can_defer_constraint_checks = True has_select_for_update = True has_select_for_update_nowait = True diff --git a/django/db/backends/postgresql_psycopg2/creation.py b/django/db/backends/postgresql_psycopg2/creation.py index 9573107271..45ce939165 100644 --- a/django/db/backends/postgresql_psycopg2/creation.py +++ b/django/db/backends/postgresql_psycopg2/creation.py @@ -16,6 +16,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'date', 'DateTimeField': 'timestamp with time zone', 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)', + 'DurationField': 'interval', 'FileField': 'varchar(%(max_length)s)', 'FilePathField': 'varchar(%(max_length)s)', 'FloatField': 'double precision', diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py index 0e512b91fa..3447bfe682 100644 --- a/django/db/backends/postgresql_psycopg2/operations.py +++ b/django/db/backends/postgresql_psycopg2/operations.py @@ -16,23 +16,6 @@ class DatabaseOperations(BaseDatabaseOperations): else: return "EXTRACT('%s' FROM %s)" % (lookup_type, field_name) - def date_interval_sql(self, sql, connector, timedelta): - """ - implements the interval functionality for expressions - format for Postgres: - (datefield + interval '3 days 200 seconds 5 microseconds') - """ - modifiers = [] - if timedelta.days: - modifiers.append('%s days' % timedelta.days) - if timedelta.seconds: - modifiers.append('%s seconds' % timedelta.seconds) - if timedelta.microseconds: - modifiers.append('%s microseconds' % timedelta.microseconds) - mods = ' '.join(modifiers) - conn = ' %s ' % connector - return '(%s)' % conn.join([sql, 'interval \'%s\'' % mods]) - def date_trunc_sql(self, lookup_type, field_name): # http://www.postgresql.org/docs/current/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name) diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index db2ac07ce9..5e23474db1 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -21,7 +21,8 @@ from django.db.backends.sqlite3.creation import DatabaseCreation from django.db.backends.sqlite3.introspection import DatabaseIntrospection from django.db.backends.sqlite3.schema import DatabaseSchemaEditor from django.db.models import fields, aggregates -from django.utils.dateparse import parse_date, parse_datetime, parse_time +from django.utils.dateparse import parse_date, parse_datetime, parse_time, parse_duration +from django.utils.duration import duration_string from django.utils.encoding import force_text from django.utils.functional import cached_property from django.utils.safestring import SafeBytes @@ -175,15 +176,12 @@ class DatabaseOperations(BaseDatabaseOperations): # cause a collision with a field name). return "django_date_extract('%s', %s)" % (lookup_type.lower(), field_name) - def date_interval_sql(self, sql, connector, timedelta): - # It would be more straightforward if we could use the sqlite strftime - # function, but it does not allow for keeping six digits of fractional - # second information, nor does it allow for formatting date and datetime - # values differently. So instead we register our own function that - # formats the datetime combined with the delta in a manner suitable - # for comparisons. - return 'django_format_dtdelta(%s, "%s", "%d", "%d", "%d")' % (sql, - connector, timedelta.days, timedelta.seconds, timedelta.microseconds) + def date_interval_sql(self, timedelta): + return "'%s'" % duration_string(timedelta), [] + + def format_for_duration_arithmetic(self, sql): + """Do nothing here, we will handle it in the custom function.""" + return sql def date_trunc_sql(self, lookup_type, field_name): # sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined @@ -314,6 +312,14 @@ class DatabaseOperations(BaseDatabaseOperations): return 'django_power(%s)' % ','.join(sub_expressions) return super(DatabaseOperations, self).combine_expression(connector, sub_expressions) + def combine_duration_expression(self, connector, sub_expressions): + if connector not in ['+', '-']: + raise utils.DatabaseError('Invalid connector for timedelta: %s.' % connector) + fn_params = ["'%s'" % connector] + sub_expressions + if len(fn_params) > 3: + raise ValueError('Too many params for timedelta operations.') + return "django_format_dtdelta(%s)" % ', '.join(fn_params) + def integer_field_range(self, internal_type): # SQLite doesn't enforce any integer constraints return (None, None) @@ -408,7 +414,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): conn.create_function("django_datetime_extract", 3, _sqlite_datetime_extract) conn.create_function("django_datetime_trunc", 3, _sqlite_datetime_trunc) conn.create_function("regexp", 2, _sqlite_regexp) - conn.create_function("django_format_dtdelta", 5, _sqlite_format_dtdelta) + conn.create_function("django_format_dtdelta", 3, _sqlite_format_dtdelta) conn.create_function("django_power", 2, _sqlite_power) return conn @@ -585,19 +591,33 @@ def _sqlite_datetime_trunc(lookup_type, dt, tzname): return "%i-%02i-%02i %02i:%02i:%02i" % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) -def _sqlite_format_dtdelta(dt, conn, days, secs, usecs): +def _sqlite_format_dtdelta(conn, lhs, rhs): + """ + LHS and RHS can be either: + - An integer number of microseconds + - A string representing a timedelta object + - A string representing a datetime + """ try: - dt = backend_utils.typecast_timestamp(dt) - delta = datetime.timedelta(int(days), int(secs), int(usecs)) + if isinstance(lhs, int): + lhs = str(decimal.Decimal(lhs) / decimal.Decimal(1000000)) + real_lhs = parse_duration(lhs) + if real_lhs is None: + real_lhs = backend_utils.typecast_timestamp(lhs) + if isinstance(rhs, int): + rhs = str(decimal.Decimal(rhs) / decimal.Decimal(1000000)) + real_rhs = parse_duration(rhs) + if real_rhs is None: + real_rhs = backend_utils.typecast_timestamp(rhs) if conn.strip() == '+': - dt = dt + delta + out = real_lhs + real_rhs else: - dt = dt - delta + out = real_lhs - real_rhs except (ValueError, TypeError): return None # typecast_timestamp returns a date or a datetime without timezone. # It will be formatted as "%Y-%m-%d" or "%Y-%m-%d %H:%M:%S[.%f]" - return str(dt) + return str(out) def _sqlite_regexp(re_pattern, re_string): diff --git a/django/db/backends/sqlite3/creation.py b/django/db/backends/sqlite3/creation.py index ea91c4a2b7..78a1cb5463 100644 --- a/django/db/backends/sqlite3/creation.py +++ b/django/db/backends/sqlite3/creation.py @@ -18,6 +18,7 @@ class DatabaseCreation(BaseDatabaseCreation): 'DateField': 'date', 'DateTimeField': 'datetime', 'DecimalField': 'decimal', + 'DurationField': 'bigint', 'FileField': 'varchar(%(max_length)s)', 'FilePathField': 'varchar(%(max_length)s)', 'FloatField': 'real', diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 81b81673d7..de0d339af7 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -34,12 +34,12 @@ class CombinableMixin(object): BITOR = '|' def _combine(self, other, connector, reversed, node=None): - if isinstance(other, datetime.timedelta): - return DateModifierNode(self, connector, other) - if not hasattr(other, 'resolve_expression'): # everything must be resolvable to an expression - other = Value(other) + if isinstance(other, datetime.timedelta): + other = DurationValue(other, output_field=fields.DurationField()) + else: + other = Value(other) if reversed: return Expression(other, connector, self) @@ -333,6 +333,18 @@ class Expression(ExpressionNode): self.lhs, self.rhs = exprs def as_sql(self, compiler, connection): + try: + lhs_output = self.lhs.output_field + except FieldError: + lhs_output = None + try: + rhs_output = self.rhs.output_field + except FieldError: + rhs_output = None + if (not connection.features.has_native_duration_field and + ((lhs_output and lhs_output.get_internal_type() == 'DurationField') + or (rhs_output and rhs_output.get_internal_type() == 'DurationField'))): + return DurationExpression(self.lhs, self.connector, self.rhs).as_sql(compiler, connection) expressions = [] expression_params = [] sql, params = compiler.compile(self.lhs) @@ -354,45 +366,31 @@ class Expression(ExpressionNode): return c -class DateModifierNode(Expression): - """ - Node that implements the following syntax: - filter(end_date__gt=F('start_date') + datetime.timedelta(days=3, seconds=200)) - - which translates into: - POSTGRES: - WHERE end_date > (start_date + INTERVAL '3 days 200 seconds') - - MYSQL: - WHERE end_date > (start_date + INTERVAL '3 0:0:200:0' DAY_MICROSECOND) - - ORACLE: - WHERE end_date > (start_date + INTERVAL '3 00:03:20.000000' DAY(1) TO SECOND(6)) - - SQLITE: - WHERE end_date > django_format_dtdelta(start_date, "+" "3", "200", "0") - (A custom function is used in order to preserve six digits of fractional - second information on sqlite, and to format both date and datetime values.) - - Note that microsecond comparisons are not well supported with MySQL, since - MySQL does not store microsecond information. - - Only adding and subtracting timedeltas is supported, attempts to use other - operations raise a TypeError. - """ - def __init__(self, lhs, connector, rhs): - if not isinstance(rhs, datetime.timedelta): - raise TypeError('rhs must be a timedelta.') - if connector not in (self.ADD, self.SUB): - raise TypeError('Connector must be + or -, not %s' % connector) - super(DateModifierNode, self).__init__(lhs, connector, Value(rhs)) +class DurationExpression(Expression): + def compile(self, side, compiler, connection): + if not isinstance(side, DurationValue): + try: + output = side.output_field + except FieldError: + pass + if output.get_internal_type() == 'DurationField': + sql, params = compiler.compile(side) + return connection.ops.format_for_duration_arithmetic(sql), params + return compiler.compile(side) def as_sql(self, compiler, connection): - timedelta = self.rhs.value - sql, params = compiler.compile(self.lhs) - if (timedelta.days == timedelta.seconds == timedelta.microseconds == 0): - return sql, params - return connection.ops.date_interval_sql(sql, self.connector, timedelta), params + expressions = [] + expression_params = [] + sql, params = self.compile(self.lhs, compiler, connection) + expressions.append(sql) + expression_params.extend(params) + sql, params = self.compile(self.rhs, compiler, connection) + expressions.append(sql) + expression_params.extend(params) + # order of precedence + expression_wrapper = '(%s)' + sql = connection.ops.combine_duration_expression(self.connector, expressions) + return expression_wrapper % sql, expression_params class F(CombinableMixin): @@ -488,6 +486,13 @@ class Value(ExpressionNode): return '%s', [self.value] +class DurationValue(Value): + def as_sql(self, compiler, connection): + if connection.features.has_native_duration_field: + return super(DurationValue, self).as_sql(compiler, connection) + return connection.ops.date_interval_sql(self.value) + + class Col(ExpressionNode): def __init__(self, alias, target, source=None): if source is None: diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index de754c83e5..92ad769e4c 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -19,8 +19,9 @@ from django.conf import settings from django import forms from django.core import exceptions, validators, checks from django.utils.datastructures import DictWrapper -from django.utils.dateparse import parse_date, parse_datetime, parse_time +from django.utils.dateparse import parse_date, parse_datetime, parse_time, parse_duration from django.utils.deprecation import RemovedInDjango19Warning +from django.utils.duration import duration_string from django.utils.functional import cached_property, curry, total_ordering, Promise from django.utils.text import capfirst from django.utils import timezone @@ -36,8 +37,8 @@ from django.utils.itercompat import is_iterable __all__ = [str(x) for x in ( 'AutoField', 'BLANK_CHOICE_DASH', 'BigIntegerField', 'BinaryField', 'BooleanField', 'CharField', 'CommaSeparatedIntegerField', 'DateField', - 'DateTimeField', 'DecimalField', 'EmailField', 'Empty', 'Field', - 'FieldDoesNotExist', 'FilePathField', 'FloatField', + 'DateTimeField', 'DecimalField', 'DurationField', 'EmailField', 'Empty', + 'Field', 'FieldDoesNotExist', 'FilePathField', 'FloatField', 'GenericIPAddressField', 'IPAddressField', 'IntegerField', 'NOT_PROVIDED', 'NullBooleanField', 'PositiveIntegerField', 'PositiveSmallIntegerField', 'SlugField', 'SmallIntegerField', 'TextField', 'TimeField', 'URLField', @@ -1573,6 +1574,50 @@ class DecimalField(Field): return super(DecimalField, self).formfield(**defaults) +class DurationField(Field): + """Stores timedelta objects. + + Uses interval on postgres, bigint of microseconds on other databases. + """ + empty_strings_allowed = False + default_error_messages = { + 'invalid': _("'%(value)s' value has an invalid format. It must be in " + "[DD] [HH:[MM:]]ss[.uuuuuu] format.") + } + description = _("Duration") + + def get_internal_type(self): + return "DurationField" + + def to_python(self, value): + if value is None: + return value + if isinstance(value, datetime.timedelta): + return value + try: + parsed = parse_duration(value) + except ValueError: + pass + else: + if parsed is not None: + return parsed + + raise exceptions.ValidationError( + self.error_messages['invalid'], + code='invalid', + params={'value': value}, + ) + + def get_db_prep_value(self, value, connection, prepared=False): + if connection.features.has_native_duration_field: + return value + return value.total_seconds() * 1000000 + + def value_to_string(self, obj): + val = self._get_val_from_obj(obj) + return '' if val is None else duration_string(val) + + class EmailField(CharField): default_validators = [validators.validate_email] description = _("Email address") diff --git a/django/forms/fields.py b/django/forms/fields.py index 6f0dc16c35..891d8ccd6e 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -26,7 +26,9 @@ from django.forms.widgets import ( from django.utils import formats from django.utils.encoding import smart_text, force_str, force_text from django.utils.ipv6 import clean_ipv6_address +from django.utils.dateparse import parse_duration from django.utils.deprecation import RemovedInDjango19Warning, RemovedInDjango20Warning, RenameMethodsBase +from django.utils.duration import duration_string from django.utils import six from django.utils.six.moves.urllib.parse import urlsplit, urlunsplit from django.utils.translation import ugettext_lazy as _, ungettext_lazy @@ -37,7 +39,7 @@ from django.core.validators import EMPTY_VALUES # NOQA __all__ = ( 'Field', 'CharField', 'IntegerField', - 'DateField', 'TimeField', 'DateTimeField', + 'DateField', 'TimeField', 'DateTimeField', 'DurationField', 'RegexField', 'EmailField', 'FileField', 'ImageField', 'URLField', 'BooleanField', 'NullBooleanField', 'ChoiceField', 'MultipleChoiceField', 'ComboField', 'MultiValueField', 'FloatField', 'DecimalField', @@ -518,6 +520,25 @@ class DateTimeField(BaseTemporalField): return datetime.datetime.strptime(force_str(value), format) +class DurationField(Field): + default_error_messages = { + 'invalid': _('Enter a valid duration.'), + } + + def prepare_value(self, value): + return duration_string(value) + + def to_python(self, value): + if value in self.empty_values: + return None + if isinstance(value, datetime.timedelta): + return value + value = parse_duration(value) + if value is None: + raise ValidationError(self.error_messages['invalid'], code='invalid') + return value + + class RegexField(CharField): def __init__(self, regex, max_length=None, min_length=None, error_message=None, *args, **kwargs): """ diff --git a/django/utils/dateparse.py b/django/utils/dateparse.py index 422f55b9c5..2c749f015c 100644 --- a/django/utils/dateparse.py +++ b/django/utils/dateparse.py @@ -27,6 +27,29 @@ datetime_re = re.compile( r'(?PZ|[+-]\d{2}(?::?\d{2})?)?$' ) +standard_duration_re = re.compile( + r'^' + r'(?:(?P-?\d+) )?' + r'((?:(?P\d+):)(?=\d+:\d+))?' + r'(?:(?P\d+):)?' + r'(?P\d+)' + r'(?:\.(?P\d{1,6})\d{0,6})?' + r'$' +) + +# Support the sections of ISO 8601 date representation that are accepted by +# timedelta +iso8601_duration_re = re.compile( + r'^P' + r'(?:(?P\d+(.\d+)?)D)?' + r'(?:T' + r'(?:(?P\d+(.\d+)?)H)?' + r'(?:(?P\d+(.\d+)?)M)?' + r'(?:(?P\d+(.\d+)?)S)?' + r')?' + r'$' +) + def parse_date(value): """Parses a string and return a datetime.date. @@ -84,3 +107,21 @@ def parse_datetime(value): kw = {k: int(v) for k, v in six.iteritems(kw) if v is not None} kw['tzinfo'] = tzinfo return datetime.datetime(**kw) + + +def parse_duration(value): + """Parses a duration string and returns a datetime.timedelta. + + The preferred format for durations in Django is '%d %H:%M:%S.%f'. + + Also supports ISO 8601 representation. + """ + match = standard_duration_re.match(value) + if not match: + match = iso8601_duration_re.match(value) + if match: + kw = match.groupdict() + if kw.get('microseconds'): + kw['microseconds'] = kw['microseconds'].ljust(6, '0') + kw = {k: float(v) for k, v in six.iteritems(kw) if v is not None} + return datetime.timedelta(**kw) diff --git a/django/utils/duration.py b/django/utils/duration.py new file mode 100644 index 0000000000..c37c885b91 --- /dev/null +++ b/django/utils/duration.py @@ -0,0 +1,21 @@ +"""Version of str(timedelta) which is not English specific.""" + + +def duration_string(duration): + days = duration.days + seconds = duration.seconds + microseconds = duration.microseconds + + minutes = seconds // 60 + seconds = seconds % 60 + + hours = minutes // 60 + minutes = minutes % 60 + + string = '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds) + if days: + string = '{} '.format(days) + string + if microseconds: + string += '.{:06d}'.format(microseconds) + + return string diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 0f13adc249..b63826d9b1 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -546,6 +546,23 @@ For each field, we describe the default widget used if you don't specify The maximum number of decimal places permitted. +``DurationField`` +~~~~~~~~~~~~~~~~~ + +.. versionadded:: 1.8 + +.. class:: DurationField(**kwargs) + + * Default widget: :class:`TextInput` + * Empty value: ``None`` + * Normalizes to: A Python :class:`~python:datetime.timedelta`. + * Validates that the given value is a string which can be converted into a + ``timedelta``. + * Error message keys: ``required``, ``invalid``. + + Accepts any format understood by + :func:`~django.utils.dateparse.parse_duration`. + ``EmailField`` ~~~~~~~~~~~~~~ diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 2429406d68..203241ae5f 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -542,6 +542,23 @@ The default form widget for this field is a :class:`~django.forms.TextInput`. :class:`FloatField` and :class:`DecimalField` classes, please see :ref:`FloatField vs. DecimalField `. +``DurationField`` +----------------- + +.. versionadded:: 1.8 + +.. class:: DurationField([**options]) + +A field for storing periods of time - modeled in Python by +:class:`~python:datetime.timedelta`. When used on PostgreSQL, the data type +used is an ``interval``, otherwise a ``bigint`` of microseconds is used. + +.. note:: + + Arithmetic with ``DurationField`` works in most cases. However on all + databases other than PostgreSQL, comparing the value of a ``DurationField`` + to arithmetic on ``DateTimeField`` instances will not work as expected. + ``EmailField`` -------------- diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 5939ff6fb1..327f44bbfc 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -158,6 +158,15 @@ The functions defined in this module share the following properties: ``tzinfo`` attribute is a :class:`~django.utils.tzinfo.FixedOffset` instance. +.. function:: parse_duration(value) + + .. versionadded:: 1.8 + + Parses a string and returns a :class:`datetime.timedelta`. + + Expects data in the format ``"DD HH:MM:SS.uuuuuu"`` or as specified by ISO + 8601 (e.g. ``P4DT1H15M20S`` which is equivalent to ``4 1:15:20``). + ``django.utils.decorators`` =========================== diff --git a/docs/releases/1.8.txt b/docs/releases/1.8.txt index fccf50bc70..3a7cfea8e7 100644 --- a/docs/releases/1.8.txt +++ b/docs/releases/1.8.txt @@ -54,9 +54,16 @@ New data types ~~~~~~~~~~~~~~ * Django now has a :class:`~django.db.models.UUIDField` for storing - universally unique identifiers. There is a corresponding :class:`form field - `. It is stored as the native ``uuid`` data type on - PostgreSQL and as a fixed length character field on other backends. + universally unique identifiers. It is stored as the native ``uuid`` data type + on PostgreSQL and as a fixed length character field on other backends. There + is a corresponding :class:`form field `. + +* Django now has a :class:`~django.db.models.DurationField` for storing periods + of time - modeled in Python by :class:`~python:datetime.timedelta`. It is + stored in the native ``interval`` data type on PostgreSQL and as a ``bigint`` + of microseconds on other backends. Date and time related arithmetic has also + been improved on all backends. There is a corresponding :class:`form field + `. Query Expressions ~~~~~~~~~~~~~~~~~ diff --git a/tests/expressions/models.py b/tests/expressions/models.py index 3a25e0862e..53eb54ec48 100644 --- a/tests/expressions/models.py +++ b/tests/expressions/models.py @@ -47,6 +47,7 @@ class Experiment(models.Model): name = models.CharField(max_length=24) assigned = models.DateField() completed = models.DateField() + estimated_time = models.DurationField() start = models.DateTimeField() end = models.DateTimeField() diff --git a/tests/expressions/tests.py b/tests/expressions/tests.py index 6949dbf43a..0e9bd57e91 100644 --- a/tests/expressions/tests.py +++ b/tests/expressions/tests.py @@ -4,7 +4,7 @@ from copy import deepcopy import datetime from django.core.exceptions import FieldError -from django.db import connection, transaction +from django.db import connection, transaction, DatabaseError from django.db.models import F from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature from django.test.utils import Approximate @@ -602,7 +602,7 @@ class FTimeDeltaTests(TestCase): # e0: started same day as assigned, zero duration end = stime + delta0 e0 = Experiment.objects.create(name='e0', assigned=sday, start=stime, - end=end, completed=end.date()) + end=end, completed=end.date(), estimated_time=delta0) self.deltas.append(delta0) self.delays.append(e0.start - datetime.datetime.combine(e0.assigned, midnight)) @@ -617,7 +617,7 @@ class FTimeDeltaTests(TestCase): delay = datetime.timedelta(1) end = stime + delay + delta1 e1 = Experiment.objects.create(name='e1', assigned=sday, - start=stime + delay, end=end, completed=end.date()) + start=stime + delay, end=end, completed=end.date(), estimated_time=delta1) self.deltas.append(delta1) self.delays.append(e1.start - datetime.datetime.combine(e1.assigned, midnight)) @@ -627,7 +627,7 @@ class FTimeDeltaTests(TestCase): end = stime + delta2 e2 = Experiment.objects.create(name='e2', assigned=sday - datetime.timedelta(3), start=stime, end=end, - completed=end.date()) + completed=end.date(), estimated_time=datetime.timedelta(hours=1)) self.deltas.append(delta2) self.delays.append(e2.start - datetime.datetime.combine(e2.assigned, midnight)) @@ -637,7 +637,7 @@ class FTimeDeltaTests(TestCase): delay = datetime.timedelta(4) end = stime + delay + delta3 e3 = Experiment.objects.create(name='e3', - assigned=sday, start=stime + delay, end=end, completed=end.date()) + assigned=sday, start=stime + delay, end=end, completed=end.date(), estimated_time=delta3) self.deltas.append(delta3) self.delays.append(e3.start - datetime.datetime.combine(e3.assigned, midnight)) @@ -647,7 +647,7 @@ class FTimeDeltaTests(TestCase): end = stime + delta4 e4 = Experiment.objects.create(name='e4', assigned=sday - datetime.timedelta(10), start=stime, end=end, - completed=end.date()) + completed=end.date(), estimated_time=delta4 - datetime.timedelta(1)) self.deltas.append(delta4) self.delays.append(e4.start - datetime.datetime.combine(e4.assigned, midnight)) @@ -675,6 +675,10 @@ class FTimeDeltaTests(TestCase): Experiment.objects.filter(end__lt=F('start') + delta)] self.assertEqual(test_set, self.expnames[:i]) + test_set = [e.name for e in + Experiment.objects.filter(end__lt=delta + F('start'))] + self.assertEqual(test_set, self.expnames[:i]) + test_set = [e.name for e in Experiment.objects.filter(end__lte=F('start') + delta)] self.assertEqual(test_set, self.expnames[:i + 1]) @@ -756,42 +760,29 @@ class FTimeDeltaTests(TestCase): self.assertEqual(expected_ends, new_ends) self.assertEqual(expected_durations, new_durations) - def test_delta_invalid_op_mult(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start') * self.deltas[0])) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to multiply datetime by timedelta.") + def test_invalid_operator(self): + with self.assertRaises(DatabaseError): + list(Experiment.objects.filter(start=F('start') * datetime.timedelta(0))) - def test_delta_invalid_op_div(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start') / self.deltas[0])) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to divide datetime by timedelta.") + def test_durationfield_add(self): + zeros = [e.name for e in + Experiment.objects.filter(start=F('start') + F('estimated_time'))] + self.assertEqual(zeros, ['e0']) - def test_delta_invalid_op_mod(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start') % self.deltas[0])) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to modulo divide datetime by timedelta.") + end_less = [e.name for e in + Experiment.objects.filter(end__lt=F('start') + F('estimated_time'))] + self.assertEqual(end_less, ['e2']) - def test_delta_invalid_op_and(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start').bitand(self.deltas[0]))) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to binary and a datetime with a timedelta.") + delta_math = [e.name for e in + Experiment.objects.filter(end__gte=F('start') + F('estimated_time') + datetime.timedelta(hours=1))] + self.assertEqual(delta_math, ['e4']) - def test_delta_invalid_op_or(self): - raised = False - try: - repr(Experiment.objects.filter(end__lt=F('start').bitor(self.deltas[0]))) - except TypeError: - raised = True - self.assertTrue(raised, "TypeError not raised on attempt to binary or a datetime with a timedelta.") + @skipUnlessDBFeature("has_native_duration_field") + def test_date_subtraction(self): + under_estimate = [e.name for e in + Experiment.objects.filter(estimated_time__gt=F('end') - F('start'))] + self.assertEqual(under_estimate, ['e2']) + + over_estimate = [e.name for e in + Experiment.objects.filter(estimated_time__lt=F('end') - F('start'))] + self.assertEqual(over_estimate, ['e4']) diff --git a/tests/forms_tests/tests/test_fields.py b/tests/forms_tests/tests/test_fields.py index 3caf902349..b2f0869c47 100644 --- a/tests/forms_tests/tests/test_fields.py +++ b/tests/forms_tests/tests/test_fields.py @@ -43,11 +43,12 @@ except ImportError: from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import ( BooleanField, CharField, ChoiceField, ComboField, DateField, DateTimeField, - DecimalField, EmailField, Field, FileField, FilePathField, FloatField, - Form, forms, HiddenInput, ImageField, IntegerField, MultipleChoiceField, - NullBooleanField, NumberInput, PasswordInput, RadioSelect, RegexField, - SplitDateTimeField, TextInput, Textarea, TimeField, TypedChoiceField, - TypedMultipleChoiceField, URLField, UUIDField, ValidationError, Widget, + DecimalField, DurationField, EmailField, Field, FileField, FilePathField, + FloatField, Form, forms, HiddenInput, ImageField, IntegerField, + MultipleChoiceField, NullBooleanField, NumberInput, PasswordInput, + RadioSelect, RegexField, SplitDateTimeField, TextInput, Textarea, + TimeField, TypedChoiceField, TypedMultipleChoiceField, URLField, UUIDField, + ValidationError, Widget, ) from django.test import SimpleTestCase from django.utils import formats @@ -611,6 +612,32 @@ class FieldsTests(SimpleTestCase): # RegexField ################################################################## + def test_durationfield_1(self): + f = DurationField() + self.assertEqual(datetime.timedelta(seconds=30), f.clean('30')) + self.assertEqual( + datetime.timedelta(minutes=15, seconds=30), + f.clean('15:30') + ) + self.assertEqual( + datetime.timedelta(hours=1, minutes=15, seconds=30), + f.clean('1:15:30') + ) + self.assertEqual( + datetime.timedelta( + days=1, hours=1, minutes=15, seconds=30, milliseconds=300), + f.clean('1 1:15:30.3') + ) + + def test_durationfield_2(self): + class DurationForm(Form): + duration = DurationField(initial=datetime.timedelta(hours=1)) + f = DurationForm() + self.assertHTMLEqual( + '', + str(f['duration']) + ) + def test_regexfield_1(self): f = RegexField('^[0-9][A-F][0-9]$') self.assertEqual('2A2', f.clean('2A2')) diff --git a/tests/model_fields/models.py b/tests/model_fields/models.py index bb75208905..e9e287f04f 100644 --- a/tests/model_fields/models.py +++ b/tests/model_fields/models.py @@ -121,6 +121,10 @@ class DateTimeModel(models.Model): t = models.TimeField() +class DurationModel(models.Model): + field = models.DurationField() + + class PrimaryKeyCharModel(models.Model): string = models.CharField(max_length=10, primary_key=True) diff --git a/tests/model_fields/test_durationfield.py b/tests/model_fields/test_durationfield.py new file mode 100644 index 0000000000..fc2c22af61 --- /dev/null +++ b/tests/model_fields/test_durationfield.py @@ -0,0 +1,67 @@ +import datetime +import json + +from django.core import exceptions, serializers +from django.db import models +from django.test import TestCase + +from .models import DurationModel + + +class TestSaveLoad(TestCase): + + def test_simple_roundtrip(self): + duration = datetime.timedelta(days=123, seconds=123, microseconds=123) + DurationModel.objects.create(field=duration) + loaded = DurationModel.objects.get() + self.assertEqual(loaded.field, duration) + + +class TestQuerying(TestCase): + + @classmethod + def setUpTestData(cls): + cls.objs = [ + DurationModel.objects.create(field=datetime.timedelta(days=1)), + DurationModel.objects.create(field=datetime.timedelta(seconds=1)), + DurationModel.objects.create(field=datetime.timedelta(seconds=-1)), + ] + + def test_exact(self): + self.assertSequenceEqual( + DurationModel.objects.filter(field=datetime.timedelta(days=1)), + [self.objs[0]] + ) + + def test_gt(self): + self.assertSequenceEqual( + DurationModel.objects.filter(field__gt=datetime.timedelta(days=0)), + [self.objs[0], self.objs[1]] + ) + + +class TestSerialization(TestCase): + test_data = '[{"fields": {"field": "1 01:00:00"}, "model": "model_fields.durationmodel", "pk": null}]' + + def test_dumping(self): + instance = DurationModel(field=datetime.timedelta(days=1, hours=1)) + data = serializers.serialize('json', [instance]) + self.assertEqual(json.loads(data), json.loads(self.test_data)) + + def test_loading(self): + instance = list(serializers.deserialize('json', self.test_data))[0].object + self.assertEqual(instance.field, datetime.timedelta(days=1, hours=1)) + + +class TestValidation(TestCase): + + def test_invalid_string(self): + field = models.DurationField() + with self.assertRaises(exceptions.ValidationError) as cm: + field.clean('not a datetime', None) + self.assertEqual(cm.exception.code, 'invalid') + self.assertEqual( + cm.exception.message % cm.exception.params, + "'not a datetime' value has an invalid format. " + "It must be in [DD] [HH:[MM:]]ss[.uuuuuu] format." + ) diff --git a/tests/utils_tests/test_dateparse.py b/tests/utils_tests/test_dateparse.py index cdf91c039e..a224e3b174 100644 --- a/tests/utils_tests/test_dateparse.py +++ b/tests/utils_tests/test_dateparse.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals -from datetime import date, time, datetime +from datetime import date, time, datetime, timedelta import unittest -from django.utils.dateparse import parse_date, parse_time, parse_datetime +from django.utils.dateparse import parse_date, parse_time, parse_datetime, parse_duration from django.utils.timezone import get_fixed_timezone @@ -46,3 +46,43 @@ class DateParseTests(unittest.TestCase): # Invalid inputs self.assertEqual(parse_datetime('20120423091500'), None) self.assertRaises(ValueError, parse_datetime, '2012-04-56T09:15:90') + + +class DurationParseTests(unittest.TestCase): + def test_seconds(self): + self.assertEqual(parse_duration('30'), timedelta(seconds=30)) + + def test_minutes_seconds(self): + self.assertEqual(parse_duration('15:30'), timedelta(minutes=15, seconds=30)) + self.assertEqual(parse_duration('5:30'), timedelta(minutes=5, seconds=30)) + + def test_hours_minutes_seconds(self): + self.assertEqual(parse_duration('10:15:30'), timedelta(hours=10, minutes=15, seconds=30)) + self.assertEqual(parse_duration('1:15:30'), timedelta(hours=1, minutes=15, seconds=30)) + self.assertEqual(parse_duration('100:200:300'), timedelta(hours=100, minutes=200, seconds=300)) + + def test_days(self): + self.assertEqual(parse_duration('4 15:30'), timedelta(days=4, minutes=15, seconds=30)) + self.assertEqual(parse_duration('4 10:15:30'), timedelta(days=4, hours=10, minutes=15, seconds=30)) + + def test_fractions_of_seconds(self): + self.assertEqual(parse_duration('15:30.1'), timedelta(minutes=15, seconds=30, milliseconds=100)) + self.assertEqual(parse_duration('15:30.01'), timedelta(minutes=15, seconds=30, milliseconds=10)) + self.assertEqual(parse_duration('15:30.001'), timedelta(minutes=15, seconds=30, milliseconds=1)) + self.assertEqual(parse_duration('15:30.0001'), timedelta(minutes=15, seconds=30, microseconds=100)) + self.assertEqual(parse_duration('15:30.00001'), timedelta(minutes=15, seconds=30, microseconds=10)) + self.assertEqual(parse_duration('15:30.000001'), timedelta(minutes=15, seconds=30, microseconds=1)) + + def test_negative(self): + self.assertEqual(parse_duration('-4 15:30'), timedelta(days=-4, minutes=15, seconds=30)) + + def test_iso_8601(self): + self.assertEqual(parse_duration('P4Y'), None) + self.assertEqual(parse_duration('P4M'), None) + self.assertEqual(parse_duration('P4W'), None) + self.assertEqual(parse_duration('P4D'), timedelta(days=4)) + self.assertEqual(parse_duration('P0.5D'), timedelta(hours=12)) + self.assertEqual(parse_duration('PT5H'), timedelta(hours=5)) + self.assertEqual(parse_duration('PT5M'), timedelta(minutes=5)) + self.assertEqual(parse_duration('PT5S'), timedelta(seconds=5)) + self.assertEqual(parse_duration('PT0.000005S'), timedelta(microseconds=5)) diff --git a/tests/utils_tests/test_duration.py b/tests/utils_tests/test_duration.py new file mode 100644 index 0000000000..559d2ef16f --- /dev/null +++ b/tests/utils_tests/test_duration.py @@ -0,0 +1,43 @@ +import datetime +import unittest + +from django.utils.dateparse import parse_duration +from django.utils.duration import duration_string + + +class TestDurationString(unittest.TestCase): + + def test_simple(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5) + self.assertEqual(duration_string(duration), '01:03:05') + + def test_days(self): + duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5) + self.assertEqual(duration_string(duration), '1 01:03:05') + + def test_microseconds(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345) + self.assertEqual(duration_string(duration), '01:03:05.012345') + + def test_negative(self): + duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5) + self.assertEqual(duration_string(duration), '-1 01:03:05') + + +class TestParseDurationRoundtrip(unittest.TestCase): + + def test_simple(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5) + self.assertEqual(parse_duration(duration_string(duration)), duration) + + def test_days(self): + duration = datetime.timedelta(days=1, hours=1, minutes=3, seconds=5) + self.assertEqual(parse_duration(duration_string(duration)), duration) + + def test_microseconds(self): + duration = datetime.timedelta(hours=1, minutes=3, seconds=5, microseconds=12345) + self.assertEqual(parse_duration(duration_string(duration)), duration) + + def test_negative(self): + duration = datetime.timedelta(days=-1, hours=1, minutes=3, seconds=5) + self.assertEqual(parse_duration(duration_string(duration)), duration)