Add eventlet db_pool use for mysql

This adds the use of eventlet's db_pool module so that we can make mysql
calls without blocking the whole process.

New config options are introduced:

sql_dbpool_enable -- Enables the use of eventlet's db_pool
sql_min_pool_size -- Set the minimum number of SQL connections

The default for sql_dbpool_enable is False for now, so there is
no forced behavior changes for those using mysql.  sql_min_pool_size
is defaulted to 1 to match behavior if not using db_pool.

Adds a new test module for our sqlalchemy code, testing this new option
as much as is possible without requiring mysql server to be running.

DocImpact

Change-Id: I99833f447df05c1beba5a3925b201dfccca72cae

2 files changed, 99 insertions, 7 deletions

diff --git a/nova/db/sqlalchemy/session.py b/nova/db/sqlalchemy/session.py
index 9b6c6db3d..66ff0f916 100644
--- a/nova/db/sqlalchemy/session.py
+++ b/nova/db/sqlalchemy/session.py
@@ -163,7 +163,12 @@ There are some things which it is best to avoid:
 import re
 import time
 
+from eventlet import db_pool
 from eventlet import greenthread
+try:
+    import MySQLdb
+except ImportError:
+    MySQLdb = None
 from sqlalchemy.exc import DisconnectionError, OperationalError
 import sqlalchemy.interfaces
 import sqlalchemy.orm
@@ -188,6 +193,10 @@ sql_opts = [
     cfg.BoolOpt('sqlite_synchronous',
                 default=True,
                 help='If passed, use synchronous mode for sqlite'),
+    cfg.IntOpt('sql_min_pool_size',
+               default=1,
+               help='Minimum number of SQL connections to keep open in a '
+                     'pool'),
     cfg.IntOpt('sql_max_pool_size',
                default=5,
                help='Maximum number of SQL connections to keep open in a '
@@ -209,6 +218,9 @@ sql_opts = [
     cfg.BoolOpt('sql_connection_trace',
                default=False,
                help='Add python stack traces to SQL as comment strings'),
+    cfg.IntOpt('sql_dbpool_enable',
+               default=False,
+               help="enable the use of eventlet's db_pool for MySQL"),
 ]
 
 CONF = cfg.CONF
@@ -323,6 +335,21 @@ def create_engine(sql_connection):
         if CONF.sql_connection == "sqlite://":
             engine_args["poolclass"] = StaticPool
             engine_args["connect_args"] = {'check_same_thread': False}
+    elif all((CONF.sql_dbpool_enable, MySQLdb,
+            "mysql" in connection_dict.drivername)):
+        LOG.info(_("Using mysql/eventlet db_pool."))
+        # MySQLdb won't accept 'None' in the password field
+        password = connection_dict.password or ''
+        pool_args = {
+                'db': connection_dict.database,
+                'passwd': password,
+                'host': connection_dict.host,
+                'user': connection_dict.username,
+                'min_size': CONF.sql_min_pool_size,
+                'max_size': CONF.sql_max_pool_size,
+                'max_idle': CONF.sql_idle_timeout}
+        creator = db_pool.ConnectionPool(MySQLdb, **pool_args)
+        engine_args['creator'] = creator.create
     else:
         engine_args['pool_size'] = CONF.sql_max_pool_size
         if CONF.sql_max_overflow is not None:
@@ -342,9 +369,7 @@ def create_engine(sql_connection):
 
     if (CONF.sql_connection_trace and
             engine.dialect.dbapi.__name__ == 'MySQLdb'):
-        import MySQLdb.cursors
-        _do_query = debug_mysql_do_query()
-        setattr(MySQLdb.cursors.BaseCursor, '_do_query', _do_query)
+        patch_mysqldb_with_stacktrace_comments()
 
     try:
         engine.connect()
@@ -378,8 +403,10 @@ def get_maker(engine, autocommit=True, expire_on_commit=False):
                                        expire_on_commit=expire_on_commit)
 
 
-def debug_mysql_do_query():
-    """Return a debug version of MySQLdb.cursors._do_query"""
+def patch_mysqldb_with_stacktrace_comments():
+    """Adds current stack trace as a comment in queries by patching
+    MySQLdb.cursors.BaseCursor._do_query.
+    """
     import MySQLdb.cursors
     import traceback
 
@@ -415,5 +442,4 @@ def debug_mysql_do_query():
             qq = q
         old_mysql_do_query(self, qq)
 
-    # return the new _do_query method
-    return _do_query
+    setattr(MySQLdb.cursors.BaseCursor, '_do_query', _do_query)
diff --git a/nova/tests/test_sqlalchemy.py b/nova/tests/test_sqlalchemy.py
new file mode 100644
index 000000000..f79d607f8
--- /dev/null
+++ b/nova/tests/test_sqlalchemy.py
@@ -0,0 +1,66 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# Copyright (c) 2012 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Unit tests for SQLAlchemy specific code."""
+
+from eventlet import db_pool
+try:
+    import MySQLdb
+except ImportError:
+    MySQLdb = None
+
+from nova import context
+from nova.db.sqlalchemy import session
+from nova import test
+
+
+class DbPoolTestCase(test.TestCase):
+    def setUp(self):
+        super(DbPoolTestCase, self).setUp()
+        self.flags(sql_dbpool_enable=True)
+        self.user_id = 'fake'
+        self.project_id = 'fake'
+        self.context = context.RequestContext(self.user_id, self.project_id)
+        if not MySQLdb:
+            self.skipTest("Unable to test due to lack of MySQLdb")
+
+    def test_db_pool_option(self):
+        self.flags(sql_idle_timeout=11, sql_min_pool_size=21,
+                sql_max_pool_size=42)
+
+        info = {}
+
+        class FakeConnectionPool(db_pool.ConnectionPool):
+            def __init__(self, mod_name, **kwargs):
+                info['module'] = mod_name
+                info['kwargs'] = kwargs
+                super(FakeConnectionPool, self).__init__(mod_name,
+                        **kwargs)
+
+            def connect(self, *args, **kwargs):
+                raise test.TestingException()
+
+        self.stubs.Set(db_pool, 'ConnectionPool',
+                FakeConnectionPool)
+
+        sql_connection = 'mysql://user:pass@127.0.0.1/nova'
+        self.assertRaises(test.TestingException, session.create_engine,
+                sql_connection)
+
+        self.assertEqual(info['module'], MySQLdb)
+        self.assertEqual(info['kwargs']['max_idle'], 11)
+        self.assertEqual(info['kwargs']['min_size'], 21)
+        self.assertEqual(info['kwargs']['max_size'], 42)