From f9c4ce51235aac4862cfe2dfaaf6836acaea1c3d Mon Sep 17 00:00:00 2001 From: Malcolm Tredinnick Date: Wed, 14 Mar 2007 12:08:19 +0000 Subject: [PATCH] Fixed #2635 -- Added improved MySQL backend support from Andy Dustman. Also added database.txt documentation; currently only describing Django-related features of MySQL versions. As of this commit, there are four known test failures due to (a) no transaction support with MyISAM storage engine and (b) MySQLdb automatically creating decimal types for Django's "float" field. git-svn-id: http://code.djangoproject.com/svn/django/trunk@4724 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/db/backends/mysql/base.py | 60 +++---- django/db/backends/mysql/client.py | 29 +++- django/db/models/fields/__init__.py | 2 +- docs/databases.txt | 162 ++++++++++++++++++ .../serializers_regress/tests.py | 2 +- 5 files changed, 210 insertions(+), 45 deletions(-) create mode 100644 docs/databases.txt diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 4bd87518e8..cbe080144c 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -10,6 +10,9 @@ try: except ImportError, e: from django.core.exceptions import ImproperlyConfigured raise ImproperlyConfigured, "Error loading MySQLdb module: %s" % e +if Database.version_info < (1,2,1,'final',2): + raise ImportError, "MySQLdb-1.2.1p2 or newer is required; you have %s" % MySQLdb.__version__ + from MySQLdb.converters import conversions from MySQLdb.constants import FIELD_TYPE import types @@ -17,11 +20,14 @@ import re DatabaseError = Database.DatabaseError +# MySQLdb-1.2.1 supports the Python boolean type, and only uses datetime +# module for time-related columns; older versions could have used mx.DateTime +# or strings if there were no datetime module. However, MySQLdb still returns +# TIME columns as timedelta -- they are more like timedelta in terms of actual +# behavior as they are signed and include days -- and Django expects time, so +# we still need to override that. django_conversions = conversions.copy() django_conversions.update({ - types.BooleanType: util.rev_typecast_boolean, - FIELD_TYPE.DATETIME: util.typecast_timestamp, - FIELD_TYPE.DATE: util.typecast_date, FIELD_TYPE.TIME: util.typecast_time, }) @@ -31,31 +37,12 @@ django_conversions.update({ # http://dev.mysql.com/doc/refman/5.0/en/news.html . server_version_re = re.compile(r'(\d{1,2})\.(\d{1,2})\.(\d{1,2})') -# This is an extra debug layer over MySQL queries, to display warnings. -# It's only used when DEBUG=True. -class MysqlDebugWrapper: - def __init__(self, cursor): - self.cursor = cursor - - def execute(self, sql, params=()): - try: - return self.cursor.execute(sql, params) - except Database.Warning, w: - self.cursor.execute("SHOW WARNINGS") - raise Database.Warning, "%s: %s" % (w, self.cursor.fetchall()) - - def executemany(self, sql, param_list): - try: - return self.cursor.executemany(sql, param_list) - except Database.Warning, w: - self.cursor.execute("SHOW WARNINGS") - raise Database.Warning, "%s: %s" % (w, self.cursor.fetchall()) - - def __getattr__(self, attr): - if self.__dict__.has_key(attr): - return self.__dict__[attr] - else: - return getattr(self.cursor, attr) +# MySQLdb-1.2.1 and newer automatically makes use of SHOW WARNINGS on +# MySQL-4.1 and newer, so the MysqlDebugWrapper is unnecessary. Since the +# point is to raise Warnings as exceptions, this can be done with the Python +# warning module, and this is setup when the connection is created, and the +# standard util.CursorDebugWrapper can be used. Also, using sql_mode +# TRADITIONAL will automatically cause most warnings to be treated as errors. try: # Only exists in Python 2.4+ @@ -83,28 +70,31 @@ class DatabaseWrapper(local): def cursor(self): from django.conf import settings + from warnings import filterwarnings if not self._valid_connection(): kwargs = { - 'user': settings.DATABASE_USER, - 'db': settings.DATABASE_NAME, - 'passwd': settings.DATABASE_PASSWORD, 'conv': django_conversions, } + if settings.DATABASE_USER: + kwargs['user'] = settings.DATABASE_USER + if settings.DATABASE_NAME: + kwargs['db'] = settings.DATABASE_NAME + if settings.DATABASE_PASSWORD: + kwargs['passwd'] = settings.DATABASE_PASSWORD if settings.DATABASE_HOST.startswith('/'): kwargs['unix_socket'] = settings.DATABASE_HOST - else: + elif settings.DATABASE_HOST: kwargs['host'] = settings.DATABASE_HOST if settings.DATABASE_PORT: kwargs['port'] = int(settings.DATABASE_PORT) kwargs.update(self.options) self.connection = Database.connect(**kwargs) cursor = self.connection.cursor() - if self.connection.get_server_info() >= '4.1': - cursor.execute("SET NAMES 'utf8'") else: cursor = self.connection.cursor() if settings.DEBUG: - return util.CursorDebugWrapper(MysqlDebugWrapper(cursor), self) + filterwarnings("error", category=Database.Warning) + return util.CursorDebugWrapper(cursor, self) return cursor def _commit(self): diff --git a/django/db/backends/mysql/client.py b/django/db/backends/mysql/client.py index f9d6297b8e..116074a9ce 100644 --- a/django/db/backends/mysql/client.py +++ b/django/db/backends/mysql/client.py @@ -3,12 +3,25 @@ import os def runshell(): args = [''] - args += ["--user=%s" % settings.DATABASE_USER] - if settings.DATABASE_PASSWORD: - args += ["--password=%s" % settings.DATABASE_PASSWORD] - if settings.DATABASE_HOST: - args += ["--host=%s" % settings.DATABASE_HOST] - if settings.DATABASE_PORT: - args += ["--port=%s" % settings.DATABASE_PORT] - args += [settings.DATABASE_NAME] + db = settings.DATABASE_OPTIONS.get('db', settings.DATABASE_NAME) + user = settings.DATABASE_OPTIONS.get('user', settings.DATABASE_USER) + passwd = settings.DATABASE_OPTIONS.get('passwd', settings.DATABASE_PASSWORD) + host = settings.DATABASE_OPTIONS.get('host', settings.DATABASE_HOST) + port = settings.DATABASE_OPTIONS.get('port', settings.DATABASE_PORT) + defaults_file = settings.DATABASE_OPTIONS.get('read_default_file') + # Seems to be no good way to set sql_mode with CLI + + if defaults_file: + args += ["--defaults-file=%s" % defaults_file] + if user: + args += ["--user=%s" % user] + if passwd: + args += ["--password=%s" % passwd] + if host: + args += ["--host=%s" % host] + if port: + args += ["--port=%s" % port] + if db: + args += [db] + os.execvp('mysql', args) diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 1d9f801f39..781e4cb9e9 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -826,7 +826,7 @@ class TimeField(Field): if value is not None: # MySQL will throw a warning if microseconds are given, because it # doesn't support microseconds. - if settings.DATABASE_ENGINE == 'mysql': + if settings.DATABASE_ENGINE == 'mysql' and hasattr(value, 'microsecond'): value = value.replace(microsecond=0) value = str(value) return Field.get_db_prep_save(self, value) diff --git a/docs/databases.txt b/docs/databases.txt new file mode 100644 index 0000000000..ff6abd7271 --- /dev/null +++ b/docs/databases.txt @@ -0,0 +1,162 @@ +=============================== +Notes About Supported Databases +=============================== + +Django attempts to support as many features as possible on all databases. +However, since not all database servers are identical, there is obviously +going to be some variations. This file describes some of the +features that might relevant to Django usage. It is not intended as a +replacement for server-specific documentation or reference manuals. + +MySQL Notes +=========== + +Django expects the database to support transactions, referential integrity, +and Unicode support (UTF-8 encoding). Fortunately MySQL_ has all these +features as available as far back as 3.23. While it may be possible to use +3.23 or 4.0, you will probably have less trouble if you use 4.1 or 5.0. + +MySQL-4.1 +--------- + +MySQL-4.1_ has greatly improved support for character sets. It is possible to +set different default character sets on the database, table, and column. +Previous versions have only a server-wide character set setting. It's also the +first version where the character set can be changed on the fly. 4.1 also has +support for views, but these are not currently used by Django. + +MySQL-5.0 +--------- + +MySQL-5.0_ adds the ``information_schema`` database, which contains detailed +data on all database schema. This is used for Django's ``inspectdb`` feature, +when it is available. 5.0 also has support for stored procedures, but these +are not currently used by Django. + +.. _MySQL: http://www.mysql.com/ +.. _MySQL-4.1: http://dev.mysql.com/doc/refman/4.1/en/index.html +.. _MySQL-5.0: http://dev.mysql.com/doc/refman/5.0/en/index.html + +Storage Engines +--------------- + +MySQL has several `storage engines`_ (previously called table types). You can +change the default storage engine in the server configuration. + +The default one is MyISAM_. The main drawback of MyISAM is that it does not +currently have support for transactions or foreign keys. On the plus side, it +is currently the only engine that supports full-text indexing and searching. + +The InnoDB_ engine is fully transactional and supports foreign key references. + +The BDB_ engine, like InnoDB, is also fully transactional and supports foreign +key references. However, it's use seems to be somewhat deprecated. + +`Other storage engines`_, including SolidDB_ and Falcon_, are on the horizon. +For now, InnoDB is probably your best choice. + +.. _storage engines: http://dev.mysql.com/doc/refman/5.0/en/storage-engines.html +.. _MyISAM: http://dev.mysql.com/doc/refman/5.0/en/myisam-storage-engine.html +.. _BDB: http://dev.mysql.com/doc/refman/5.0/en/bdb-storage-engine.html +.. _InnoDB: http://dev.mysql.com/doc/refman/5.0/en/innodb.html +.. _Other storage engines: http://dev.mysql.com/doc/refman/5.1/en/storage-engines-other.html +.. _SolidDB: http://forge.mysql.com/projects/view.php?id=139 +.. _Falcon: http://dev.mysql.com/doc/falcon/en/index.html + +MySQLdb +------- + +`MySQLdb`_ is the Python interface to MySQL. 1.2.1 is the first version which +has support for MySQL-4.1 and newer. If you are trying to use an older version +of MySQL, then 1.2.0 *may* work for you. + +.. _MySQLdb: http://sourceforge.net/projects/mysql-python + +Creating your database +~~~~~~~~~~~~~~~~~~~~~~ + +You can `create your database`_ using the command-line tools and this SQL:: + + CREATE DATABASE CHARACTER SET utf8; + +This ensures all tables and columns will use utf8 by default. + +.. _create your database: http://dev.mysql.com/doc/refman/5.0/en/create-database.html + +Connecting to the database +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Refer to the `settings documentation`_. + +Connection settings are used in this order: + + 1. ``DATABASE_OPTIONS`` + 2. ``DATABASE_NAME``, ``DATABASE_USER``, ``DATABASE_PASSWORD``, ``DATABASE_HOST``, + ``DATABASE_PORT`` + 3. MySQL option files. + +In other words, if you set the name of the database in ``DATABASE_OPTIONS``, +this will take precedence over ``DATABASE_NAME``, which would override +anything in a `MySQL option file`_. + +Here's a sample configuration which uses a MySQL option file:: + + # settings.py + DATABASE_ENGINE = "mysql" + DATABASE_OPTIONS = { + 'read_default_file': '/path/to/my.cnf', + } + + # my.cnf + [client] + database = DATABASE_NAME + user = DATABASE_USER + passwd = DATABASE_PASSWORD + default-character-set = utf8 + +There are several other MySQLdb connection options which may be useful, such +as ``ssl``, ``use_unicode``, ``init_command``, and ``sql_mode``; consult the +`MySQLdb documentation`_ for more details. + +.. _settings documentation: http://www.djangoproject.com/documentation/settings/#database-engine +.. _MySQL option file: http://dev.mysql.com/doc/refman/5.0/en/option-files.html +.. _MySQLdb documentation: http://mysql-python.sourceforge.net/ + +Creating your tables +~~~~~~~~~~~~~~~~~~~~ + +When Django generates the schema, it doesn't specify a storage engine, so they +will be created with whatever default `storage engine`__ your database server +is configured for. The easiest solution is to set your database server's default +storage engine to the desired engine. + +__ `storage engines`_ + +If you are using a hosting service and can't change your server's default +storage engine, you have a couple of options. + +After the tables is created, all that is needed to convert it to a new storage +engine (such as InnoDB) is:: + + ALTER TABLE ENGINE=INNODB; + +With a lot of tables, this can be tedious. + +Another option is to use the ``init_command`` option for MySQLdb prior to +creating your tables:: + + DATABASE_OPTIONS = { + ... + "init_command": "SET storage_engine=INNODB", + ... + } + +This sets the default storage engine upon connecting to the database. After +your tables are set up and running in production, you should remove this +option. + +Another method for changing the storage engine is described in +AlterModelOnSyncDB_. + +.. _AlterModelOnSyncDB: http://code.djangoproject.com/wiki/AlterModelOnSyncDB + diff --git a/tests/regressiontests/serializers_regress/tests.py b/tests/regressiontests/serializers_regress/tests.py index 05f320bfa8..008ccecc2c 100644 --- a/tests/regressiontests/serializers_regress/tests.py +++ b/tests/regressiontests/serializers_regress/tests.py @@ -54,7 +54,7 @@ def pk_create(pk, klass, data): def data_compare(testcase, pk, klass, data): instance = klass.objects.get(id=pk) testcase.assertEqual(data, instance.data, - "Objects with PK=%d not equal; expected '%s', got '%s'" % (pk,data,instance.data)) + "Objects with PK=%d not equal; expected '%s' (%s), got '%s' (%s)" % (pk,data, type(data), instance.data, type(instance.data))) def fk_compare(testcase, pk, klass, data): instance = klass.objects.get(id=pk)