Merge "Make sqlite in-memory-db usable to unittest"

6 files changed, 114 insertions, 71 deletions

diff --git a/nova/db/sqlalchemy/migration.py b/nova/db/sqlalchemy/migration.py
index 16177cbcf..14a111f9c 100644
--- a/nova/db/sqlalchemy/migration.py
+++ b/nova/db/sqlalchemy/migration.py
@@ -16,14 +16,46 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import distutils.version as dist_version
 import os
 import sys
 
+from nova.db.sqlalchemy.session import get_engine
 from nova import exception
 from nova import flags
 
 import sqlalchemy
+import migrate
+from migrate.versioning import util as migrate_util
+
+
+MIGRATE_PKG_VER = dist_version.StrictVersion(migrate.__version__)
+USE_MIGRATE_PATCH = MIGRATE_PKG_VER < dist_version.StrictVersion('0.7.3')
+
+
+@migrate_util.decorator
+def patched_with_engine(f, *a, **kw):
+    url = a[0]
+    engine = migrate_util.construct_engine(url, **kw)
+
+    try:
+        kw['engine'] = engine
+        return f(*a, **kw)
+    finally:
+        if isinstance(engine, migrate_util.Engine) and engine is not url:
+            migrate_util.log.debug('Disposing SQLAlchemy engine %s', engine)
+            engine.dispose()
+
+
+# TODO(jkoelker) When migrate 0.7.3 is released and nova depends
+#                on that version or higher, this can be removed
+if USE_MIGRATE_PATCH:
+    migrate_util.with_engine = patched_with_engine
+
+
+# NOTE(jkoelker) Delay importing migrate until we are patched
 from migrate.versioning import api as versioning_api
+from migrate.versioning.repository import Repository
 
 try:
     from migrate.versioning import exceptions as versioning_exceptions
@@ -37,6 +69,8 @@ except ImportError:
 
 FLAGS = flags.FLAGS
 
+_REPOSITORY = None
+
 
 def db_sync(version=None):
     if version is not None:
@@ -46,24 +80,24 @@ def db_sync(version=None):
             raise exception.Error(_("version should be an integer"))
 
     current_version = db_version()
-    repo_path = _find_migrate_repo()
+    repository = _find_migrate_repo()
     if version is None or version > current_version:
-        return versioning_api.upgrade(FLAGS.sql_connection, repo_path, version)
+        return versioning_api.upgrade(get_engine(), repository, version)
     else:
-        return versioning_api.downgrade(FLAGS.sql_connection, repo_path,
+        return versioning_api.downgrade(get_engine(), repository,
                                         version)
 
 
 def db_version():
-    repo_path = _find_migrate_repo()
+    repository = _find_migrate_repo()
     try:
-        return versioning_api.db_version(FLAGS.sql_connection, repo_path)
+        return versioning_api.db_version(get_engine(), repository)
     except versioning_exceptions.DatabaseNotControlledError:
         # If we aren't version controlled we may already have the database
         # in the state from before we started version control, check for that
         # and set up version_control appropriately
         meta = sqlalchemy.MetaData()
-        engine = sqlalchemy.create_engine(FLAGS.sql_connection, echo=False)
+        engine = get_engine()
         meta.reflect(bind=engine)
         try:
             for table in ('auth_tokens', 'zones', 'export_devices',
@@ -85,14 +119,17 @@ def db_version():
 
 
 def db_version_control(version=None):
-    repo_path = _find_migrate_repo()
-    versioning_api.version_control(FLAGS.sql_connection, repo_path, version)
+    repository = _find_migrate_repo()
+    versioning_api.version_control(get_engine(), repository, version)
     return version
 
 
 def _find_migrate_repo():
     """Get the path for the migrate repository."""
+    global _REPOSITORY
     path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                         'migrate_repo')
     assert os.path.exists(path)
-    return path
+    if _REPOSITORY is None:
+        _REPOSITORY = Repository(path)
+    return _REPOSITORY
diff --git a/nova/db/sqlalchemy/session.py b/nova/db/sqlalchemy/session.py
index 52983134a..fe1b44c41 100644
--- a/nova/db/sqlalchemy/session.py
+++ b/nova/db/sqlalchemy/session.py
@@ -23,6 +23,8 @@ import time
 import sqlalchemy.interfaces
 import sqlalchemy.orm
 from sqlalchemy.exc import DisconnectionError
+from sqlalchemy.pool import NullPool, StaticPool
+import time
 
 import nova.exception
 import nova.flags as flags
@@ -38,11 +40,11 @@ _MAKER = None
 
 def get_session(autocommit=True, expire_on_commit=False):
     """Return a SQLAlchemy session."""
-    global _ENGINE, _MAKER
+    global _MAKER
 
-    if _MAKER is None or _ENGINE is None:
-        _ENGINE = get_engine()
-        _MAKER = get_maker(_ENGINE, autocommit, expire_on_commit)
+    if _MAKER is None:
+        engine = get_engine()
+        _MAKER = get_maker(engine, autocommit, expire_on_commit)
 
     session = _MAKER()
     session.query = nova.exception.wrap_db_error(session.query)
@@ -81,23 +83,32 @@ class MySQLPingListener(object):
 
 def get_engine():
     """Return a SQLAlchemy engine."""
-    connection_dict = sqlalchemy.engine.url.make_url(FLAGS.sql_connection)
+    global _ENGINE
+    if _ENGINE is None:
+        connection_dict = sqlalchemy.engine.url.make_url(FLAGS.sql_connection)
+
+        engine_args = {
+            "pool_recycle": FLAGS.sql_idle_timeout,
+            "echo": False,
+            'convert_unicode': True,
+        }
+
+        if "sqlite" in connection_dict.drivername:
+            engine_args["poolclass"] = NullPool
+
+            if FLAGS.sql_connection == "sqlite://":
+                engine_args["poolclass"] = StaticPool
+                engine_args["connect_args"] = {'check_same_thread': False}
 
-    engine_args = {
-        "pool_recycle": FLAGS.sql_idle_timeout,
-        "echo": False,
-        'convert_unicode': True,
-    }
+            if not FLAGS.sqlite_synchronous:
+                engine_args["listeners"] = [SynchronousSwitchListener()]
 
-    if "sqlite" in connection_dict.drivername:
-        engine_args["poolclass"] = sqlalchemy.pool.NullPool
-        if not FLAGS.sqlite_synchronous:
-            engine_args["listeners"] = [SynchronousSwitchListener()]
+        if 'mysql' in connection_dict.drivername:
+            engine_args['listeners'] = [MySQLPingListener()]
 
-    if 'mysql' in connection_dict.drivername:
-        engine_args['listeners'] = [MySQLPingListener()]
+        _ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
 
-    return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
+    return _ENGINE
 
 
 def get_maker(engine, autocommit=True, expire_on_commit=False):
diff --git a/nova/test.py b/nova/test.py
index 87e1ce322..e44ad57ea 100644
--- a/nova/test.py
+++ b/nova/test.py
@@ -40,6 +40,7 @@ from nova.openstack.common import cfg
 from nova import utils
 from nova import service
 from nova.testing.fake import rabbit
+from nova.tests import reset_db
 from nova.virt import fake
 
 
@@ -129,8 +130,7 @@ class TestCase(unittest.TestCase):
         #             now that we have some required db setup for the system
         #             to work properly.
         self.start = utils.utcnow()
-        shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db),
-                        os.path.join(FLAGS.state_path, FLAGS.sqlite_db))
+        reset_db()
 
         # emulate some of the mox stuff, we can't use the metaclass
         # because it screws with our generators
diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py
index 02bc77898..fee29da6c 100644
--- a/nova/tests/__init__.py
+++ b/nova/tests/__init__.py
@@ -34,25 +34,44 @@
 # The code below enables nosetests to work with i18n _() blocks
 import __builtin__
 setattr(__builtin__, '_', lambda x: x)
+import os
+import shutil
+
+from nova.db.sqlalchemy.session import get_engine
+from nova import flags
+
+FLAGS = flags.FLAGS
+
+_DB = None
+
+
+def reset_db():
+    if FLAGS.sql_connection == "sqlite://":
+        engine = get_engine()
+        engine.dispose()
+        conn = engine.connect()
+        conn.connection.executescript(_DB)
+    else:
+        shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db),
+                        os.path.join(FLAGS.state_path, FLAGS.sqlite_db))
 
 
 def setup():
     import mox  # Fail fast if you don't have mox. Workaround for bug 810424
-    import os
-    import shutil
 
     from nova import context
-    from nova import flags
     from nova import db
     from nova.db import migration
     from nova.network import manager as network_manager
     from nova.tests import fake_flags
 
-    FLAGS = flags.FLAGS
-
-    testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db)
-    if os.path.exists(testdb):
-        return
+    if FLAGS.sql_connection == "sqlite://":
+        if migration.db_version() > 1:
+            return
+    else:
+        testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db)
+        if os.path.exists(testdb):
+            return
     migration.db_sync()
     ctxt = context.get_admin_context()
     network = network_manager.VlanManager()
@@ -74,5 +93,11 @@ def setup():
     for net in db.network_get_all(ctxt):
         network.set_network_host(ctxt, net)
 
-    cleandb = os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db)
-    shutil.copyfile(testdb, cleandb)
+    if FLAGS.sql_connection == "sqlite://":
+        global _DB
+        engine = get_engine()
+        conn = engine.connect()
+        _DB = "".join(line for line in conn.connection.iterdump())
+    else:
+        cleandb = os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db)
+        shutil.copyfile(testdb, cleandb)
diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py
index e8771b99c..0da6ff69d 100644
--- a/nova/tests/fake_flags.py
+++ b/nova/tests/fake_flags.py
@@ -37,7 +37,7 @@ FLAGS.set_default('image_service', 'nova.image.fake.FakeImageService')
 flags.DECLARE('iscsi_num_targets', 'nova.volume.driver')
 FLAGS.set_default('iscsi_num_targets', 8)
 FLAGS.set_default('verbose', True)
-FLAGS.set_default('sqlite_db', "tests.sqlite")
+FLAGS.set_default('sql_connection', "sqlite://")
 FLAGS.set_default('use_ipv6', True)
 FLAGS.set_default('flat_network_bridge', 'br100')
 FLAGS.set_default('sqlite_synchronous', False)
diff --git a/nova/tests/test_migrations.py b/nova/tests/test_migrations.py
index b9d3edb95..9ffaeacfa 100644
--- a/nova/tests/test_migrations.py
+++ b/nova/tests/test_migrations.py
@@ -26,50 +26,20 @@ if possible.
 
 import ConfigParser
 import commands
-import distutils.version as dist_version
 import os
 import unittest
 import urlparse
 
-import migrate
-from migrate.versioning import util as migrate_util
+from migrate.versioning import repository
 import sqlalchemy
 
 import nova.db.sqlalchemy.migrate_repo
+from nova.db.sqlalchemy.migration import versioning_api as migration_api
 from nova import log as logging
 from nova import test
 
-
 LOG = logging.getLogger('nova.tests.test_migrations')
 
-MIGRATE_PKG_VER = dist_version.StrictVersion(migrate.__version__)
-USE_MIGRATE_PATCH = MIGRATE_PKG_VER < dist_version.StrictVersion('0.7.3')
-
-
-@migrate_util.decorator
-def patched_with_engine(f, *a, **kw):
-    url = a[0]
-    engine = migrate_util.construct_engine(url, **kw)
-
-    try:
-        kw['engine'] = engine
-        return f(*a, **kw)
-    finally:
-        if isinstance(engine, migrate_util.Engine) and engine is not url:
-            migrate_util.log.debug('Disposing SQLAlchemy engine %s', engine)
-            engine.dispose()
-
-
-# TODO(jkoelker) When migrate 0.7.3 is released and nova depends
-#                on that version or higher, this can be removed
-if USE_MIGRATE_PATCH:
-    migrate_util.with_engine = patched_with_engine
-
-
-# NOTE(jkoelker) Delay importing migrate until we are patched
-from migrate.versioning import api as migration_api
-from migrate.versioning import repository
-
 
 class TestMigrations(unittest.TestCase):
     """Test sqlalchemy-migrate migrations"""