Move DB thread pooling to DB API loader

Fixes bug 1128605

The dbpool code in sqlalchemy session is the wrong place to implement
thread pooling as it wraps each individual SQL call to run in its own
thread.  When combined with SQL server locking, all threads can be eaten
waiting on locks with none available to run a 'COMMIT'.

The correct place to do thread pooling is around each DB API call.

This patch removes dbpool from sqlalchemy and creates a common DB API
loader for all openstack projects which implements the following
configuration options:

db_backend: Full path to DB API backend module (or a known short name if
            a project chooses to implement a mapping)
dbapi_use_tpool: True or False whether to use thread pooling around all
                 DB API calls.

DB backend modules must implement a 'get_backend()' method.

Example usage for nova/db/api.py would be:

"""
from nova.openstack.common.db import api as db_api

_KNOWN_BACKENDS = {'sqlalchemy': 'nova.db.sqlalchemy.api'}

IMPL = db_api.DBAPI(backend_mapping=_KNOWN_BACKENDS)
"""

NOTE: Enabling thread pooling will be broken until this issue is
resolved in eventlet _OR_ until we modify our eventlet.monkey_patch()
calls to include 'thread=False':

https://bitbucket.org/eventlet/eventlet/issue/137/

Change-Id: Idf14563ea07cf8ccf2a77b3f53659d8528927fc7

2 files changed, 87 insertions, 40 deletions

diff --git a/tests/unit/db/sqlalchemy/test_sqlalchemy.py b/tests/unit/db/sqlalchemy/test_sqlalchemy.py
index c063f0d..6b84f10 100644
--- a/tests/unit/db/sqlalchemy/test_sqlalchemy.py
+++ b/tests/unit/db/sqlalchemy/test_sqlalchemy.py
@@ -16,8 +16,6 @@
 
 """Unit tests for SQLAlchemy specific code."""
 
-from eventlet import db_pool
-
 from sqlalchemy import Column, MetaData, Table, UniqueConstraint
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import DateTime, Integer
@@ -34,44 +32,6 @@ class TestException(Exception):
     pass
 
 
-class DbPoolTestCase(test_utils.BaseTestCase):
-    def setUp(self):
-        super(DbPoolTestCase, self).setUp()
-        if MySQLdb is None:
-            self.skipTest("Required module MySQLdb missing.")
-        self.config(sql_dbpool_enable=True)
-        self.user_id = 'fake'
-        self.project_id = 'fake'
-
-    def test_db_pool_option(self):
-        self.config(sql_idle_timeout=11, sql_min_pool_size=21,
-                    sql_max_pool_size=42)
-
-        info = {}
-
-        class FakeConnectionPool(db_pool.ConnectionPool):
-            def __init__(self, mod_name, **kwargs):
-                info['module'] = mod_name
-                info['kwargs'] = kwargs
-                super(FakeConnectionPool, self).__init__(mod_name,
-                                                         **kwargs)
-
-            def connect(self, *args, **kwargs):
-                raise TestException()
-
-        self.stubs.Set(db_pool, 'ConnectionPool',
-                       FakeConnectionPool)
-
-        sql_connection = 'mysql://user:pass@127.0.0.1/nova'
-        self.assertRaises(TestException, session.create_engine,
-                          sql_connection)
-
-        self.assertEqual(info['module'], MySQLdb)
-        self.assertEqual(info['kwargs']['max_idle'], 11)
-        self.assertEqual(info['kwargs']['min_size'], 21)
-        self.assertEqual(info['kwargs']['max_size'], 42)
-
-
 BASE = declarative_base()
 _TABLE_NAME = '__tmp__test__tmp__'
 
diff --git a/tests/unit/db/test_api.py b/tests/unit/db/test_api.py
new file mode 100644
index 0000000..a31ffd0
--- /dev/null
+++ b/tests/unit/db/test_api.py
@@ -0,0 +1,87 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Unit tests for DB API."""
+
+from eventlet import tpool
+
+from openstack.common.db import api
+from tests import utils as test_utils
+
+
+def get_backend():
+    return DBAPI()
+
+
+class DBAPI(object):
+    def api_class_call1(_self, *args, **kwargs):
+        return args, kwargs
+
+
+class DBAPITestCase(test_utils.BaseTestCase):
+    def test_dbapi_api_class_method_and_tpool_false(self):
+        backend_mapping = {'test_known': 'tests.unit.db.test_api'}
+        self.config(db_backend='test_known', dbapi_use_tpool=False)
+
+        info = dict(tpool=False)
+        orig_execute = tpool.execute
+
+        def our_execute(*args, **kwargs):
+            info['tpool'] = True
+            return orig_execute(*args, **kwargs)
+
+        self.stubs.Set(tpool, 'execute', our_execute)
+
+        dbapi = api.DBAPI(backend_mapping=backend_mapping)
+        result = dbapi.api_class_call1(1, 2, kwarg1='meow')
+        expected = ((1, 2), {'kwarg1': 'meow'})
+        self.assertEqual(expected, result)
+        self.assertFalse(info['tpool'])
+
+    def test_dbapi_api_class_method_and_tpool_true(self):
+        backend_mapping = {'test_known': 'tests.unit.db.test_api'}
+        self.config(db_backend='test_known', dbapi_use_tpool=True)
+
+        info = dict(tpool=False)
+        orig_execute = tpool.execute
+
+        def our_execute(*args, **kwargs):
+            info['tpool'] = True
+            return orig_execute(*args, **kwargs)
+
+        self.stubs.Set(tpool, 'execute', our_execute)
+
+        dbapi = api.DBAPI(backend_mapping=backend_mapping)
+        result = dbapi.api_class_call1(1, 2, kwarg1='meow')
+        expected = ((1, 2), {'kwarg1': 'meow'})
+        self.assertEqual(expected, result)
+        self.assertTrue(info['tpool'])
+
+    def test_dbapi_full_path_module_method(self):
+        self.config(db_backend='tests.unit.db.test_api')
+        dbapi = api.DBAPI()
+        result = dbapi.api_class_call1(1, 2, kwarg1='meow')
+        expected = ((1, 2), {'kwarg1': 'meow'})
+        self.assertEqual(expected, result)
+
+    def test_dbapi_unknown_invalid_backend(self):
+        self.config(db_backend='tests.unit.db.not_existant')
+        dbapi = api.DBAPI()
+
+        def call_it():
+            dbapi.api_class_call1(1, 2, kwarg1='meow')
+
+        self.assertRaises(ImportError, call_it)