Move DB thread pooling to DB API.

The eventlet db_pool wrapping done in sqlalchemy is wrong.  It needs to
be around the whole DB API call.

Fixes bug 1128605

Syncs nova with the fix in oslo.  Moves nova to use the oslo version of
db/api.py to import the DB API implementation.

Renames configuration option sql_dbpool_enable to dbapi_use_tpool

NOTE: tpool will not work without a fix to eventlet for Threads + locking.

DocImpact

Change-Id: I6c75b6138d38d12261d133f2cb2f5e59c667f837

2 files changed, 100 insertions, 35 deletions

diff --git a/nova/openstack/common/db/api.py b/nova/openstack/common/db/api.py
new file mode 100644
index 000000000..edb42074f
--- /dev/null
+++ b/nova/openstack/common/db/api.py
@@ -0,0 +1,100 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Multiple DB API backend support.
+
+Supported configuration options:
+
+`db_backend`: DB backend name or full module path to DB backend module.
+`dbapi_use_tpool`: Enable thread pooling of DB API calls.
+
+A DB backend module should implement a method named 'get_backend' which
+takes no arguments.  The method can return any object that implements DB
+API methods.
+
+*NOTE*: There are bugs in eventlet when using tpool combined with
+threading locks. The python logging module happens to use such locks.  To
+work around this issue, be sure to specify thread=False with
+eventlet.monkey_patch().
+
+A bug for eventlet has been filed here:
+
+https://bitbucket.org/eventlet/eventlet/issue/137/
+"""
+import functools
+
+from nova.openstack.common import cfg
+from nova.openstack.common import lockutils
+from nova.openstack.common import importutils
+
+
+db_opts = [
+    cfg.StrOpt('db_backend',
+               default='sqlalchemy',
+               help='The backend to use for db'),
+    cfg.BoolOpt('dbapi_use_tpool',
+                default=False,
+                help='Enable the experimental use of thread pooling for '
+                     'all DB API calls')
+]
+
+CONF = cfg.CONF
+CONF.register_opts(db_opts)
+
+
+class DBAPI(object):
+    def __init__(self, backend_mapping=None):
+        if backend_mapping is None:
+            backend_mapping = {}
+        self.__backend = None
+        self.__backend_mapping = backend_mapping
+
+    @lockutils.synchronized('dbapi_backend', 'nova-')
+    def __get_backend(self):
+        """Get the actual backend.  May be a module or an instance of
+        a class.  Doesn't matter to us.  We do this synchronized as it's
+        possible multiple greenthreads started very quickly trying to do
+        DB calls and eventlet can switch threads before self.__backend gets
+        assigned.
+        """
+        if self.__backend:
+            # Another thread assigned it
+            return self.__backend
+        backend_name = CONF.db_backend
+        self.__use_tpool = CONF.dbapi_use_tpool
+        if self.__use_tpool:
+            from eventlet import tpool
+            self.__tpool = tpool
+        # Import the untranslated name if we don't have a
+        # mapping.
+        backend_path = self.__backend_mapping.get(backend_name,
+                                                  backend_name)
+        backend_mod = importutils.import_module(backend_path)
+        self.__backend = backend_mod.get_backend()
+        return self.__backend
+
+    def __getattr__(self, key):
+        backend = self.__backend or self.__get_backend()
+        attr = getattr(backend, key)
+        if not self.__use_tpool or not hasattr(attr, '__call__'):
+            return attr
+
+        def tpool_wrapper(*args, **kwargs):
+            return self.__tpool.execute(attr, *args, **kwargs)
+
+        functools.update_wrapper(tpool_wrapper, attr)
+        return tpool_wrapper
diff --git a/nova/openstack/common/db/sqlalchemy/session.py b/nova/openstack/common/db/sqlalchemy/session.py
index 93bfa9b15..fb86d9ca5 100644
--- a/nova/openstack/common/db/sqlalchemy/session.py
+++ b/nova/openstack/common/db/sqlalchemy/session.py
@@ -244,7 +244,6 @@ import os.path
 import re
 import time
 
-from eventlet import db_pool
 from eventlet import greenthread
 from oslo.config import cfg
 from sqlalchemy.exc import DisconnectionError, OperationalError, IntegrityError
@@ -253,14 +252,10 @@ import sqlalchemy.orm
 from sqlalchemy.pool import NullPool, StaticPool
 from sqlalchemy.sql.expression import literal_column
 
-from nova.openstack.common import importutils
 from nova.openstack.common import log as logging
 from nova.openstack.common.gettextutils import _
 from nova.openstack.common import timeutils
 
-MySQLdb = importutils.try_import('MySQLdb')
-if MySQLdb is not None:
-    from MySQLdb.constants import CLIENT as mysql_client_constants
 
 sql_opts = [
     cfg.StrOpt('sql_connection',
@@ -303,9 +298,6 @@ sql_opts = [
     cfg.BoolOpt('sql_connection_trace',
                 default=False,
                 help='Add python stack traces to SQL as comment strings'),
-    cfg.BoolOpt('sql_dbpool_enable',
-                default=False,
-                help="enable the use of eventlet's db_pool for MySQL"),
 ]
 
 CONF = cfg.CONF
@@ -517,33 +509,6 @@ def create_engine(sql_connection):
         if CONF.sql_connection == "sqlite://":
             engine_args["poolclass"] = StaticPool
             engine_args["connect_args"] = {'check_same_thread': False}
-    elif all((CONF.sql_dbpool_enable, MySQLdb,
-              "mysql" in connection_dict.drivername)):
-        LOG.info(_("Using mysql/eventlet db_pool."))
-        # MySQLdb won't accept 'None' in the password field
-        password = connection_dict.password or ''
-        pool_args = {
-            'db': connection_dict.database,
-            'passwd': password,
-            'host': connection_dict.host,
-            'user': connection_dict.username,
-            'min_size': CONF.sql_min_pool_size,
-            'max_size': CONF.sql_max_pool_size,
-            'max_idle': CONF.sql_idle_timeout,
-            'client_flag': mysql_client_constants.FOUND_ROWS}
-
-        pool = db_pool.ConnectionPool(MySQLdb, **pool_args)
-
-        def creator():
-            conn = pool.create()
-            if isinstance(conn, tuple):
-                # NOTE(belliott) eventlet >= 0.10 returns a tuple
-                now, now, conn = conn
-
-            return conn
-
-        engine_args['creator'] = creator
-
     else:
         engine_args['pool_size'] = CONF.sql_max_pool_size
         if CONF.sql_max_overflow is not None: