extension migrations

Allow each of the extensions to have their own
sql migration repository instead of mixing them into
the common repo.  db_sync must be called explicitly on the extension.

In the past, it was assumed that only migrations for backends backed in
sql would be run.  In practice, however, all of the migrations were run
every time.  The code has been modified to reflect this.

Adds parameter --extension to the cli for db_sync and db_version

to test out the migrations

bin/keystone-manage db_sync --extension example

will migrate to version 1 and

bin/keystone-manage db_sync --extension example 0

will migrate it back to 0

to check the version

bin/keystone-manage db_version --extension example

blueprint multiple-sql-migrate-repos

DocImpact

Change-Id: I6852d75bde6506c535fa3d74537e3c1bbd6578d8

10 files changed, 288 insertions, 77 deletions

diff --git a/doc/source/developing.rst b/doc/source/developing.rst
index c14ef7ab..2cf4b98e 100644
--- a/doc/source/developing.rst
+++ b/doc/source/developing.rst
@@ -71,6 +71,36 @@ place::
 
 .. _`python-keystoneclient`: https://github.com/openstack/python-keystoneclient
 
+Database Schema Migrations
+--------------------------
+
+Keystone uses SQLAlchemy-migrate
+_`SQLAlchemy-migrate`:http://code.google.com/p/sqlalchemy-migrate/ to migrate the SQL database
+between revisions.  For core components, the migrations are kept in a central
+repository under keystone/common/sql/migrate_repo.
+
+Extensions should be created as directories under `keystone/contrib`.  An
+extension that requires sql migrations should not change the common repository,
+but should instead have its own repository.  This repository must be in the
+extension's directory in `keystone/contrib/<extension>/migrate_repo.`  In
+addition it needs a subdirectory named `versions`.  For example, if the
+extension name is `my_extension` then the directory structure would be
+`keystone/contrib/my_extension/migrate_repo/versions/`.  For the migration
+o work, both the migrate_repo and versions subdirectories must have empty
+__init__.py files.  SQLAlchemy-migrate will look for a configuration file in
+the migrate_repo named migrate.cfg. This conforms to a Key/value ini file
+format.  A sample config file with the minimal set of values is::
+
+    [db_settings]
+    repository_id=my_extension
+    version_table=migrate_version
+    required_dbs=[]
+
+The directory `keystone/contrib/example` contains a sample extension migration.
+
+Migrations for extension must be explicitly run. To run a migration for a specific
+extension, run  `keystone-manage --extension <name> db_sync`.
+
 Initial Sample Data
 -------------------
 
diff --git a/keystone/cli.py b/keystone/cli.py
index 21d2ad40..18c095ce 100644
--- a/keystone/cli.py
+++ b/keystone/cli.py
@@ -20,12 +20,15 @@ import grp
 import os
 import pwd
 
+from migrate import exceptions
+
 from oslo.config import cfg
 import pbr.version
 
 from keystone.common import openssl
 from keystone.common.sql import migration
 from keystone import config
+from keystone import contrib
 from keystone.openstack.common import importutils
 from keystone.openstack.common import jsonutils
 from keystone import token
@@ -57,14 +60,35 @@ class DbSync(BaseApp):
                                   'version. If not provided, db_sync will '
                                   'migrate the database to the latest known '
                                   'version.'))
+        parser.add_argument('--extension', default=None,
+                            help=('Migrate the database for the specified '
+                                  'extension. If not provided, db_sync will '
+                                  'migrate the common repository.'))
+
         return parser
 
     @staticmethod
     def main():
-        for k in ['identity', 'catalog', 'policy', 'token', 'credential']:
-            driver = importutils.import_object(getattr(CONF, k).driver)
-            if hasattr(driver, 'db_sync'):
-                driver.db_sync(CONF.command.version)
+        version = CONF.command.version
+        extension = CONF.command.extension
+        if not extension:
+            migration.db_sync(version=version)
+        else:
+            package_name = "%s.%s.migrate_repo" % (contrib.__name__, extension)
+            try:
+                package = importutils.import_module(package_name)
+                repo_path = os.path.abspath(os.path.dirname(package.__file__))
+            except ImportError:
+                print _("This extension does not provide migrations.")
+                exit(0)
+            try:
+                # Register the repo with the version control API
+                # If it already knows about the repo, it will throw
+                # an exception that we can safely ignore
+                migration.db_version_control(version=None, repo_path=repo_path)
+            except exceptions.DatabaseAlreadyControlledError:
+                pass
+            migration.db_sync(version=None, repo_path=repo_path)
 
 
 class DbVersion(BaseApp):
@@ -72,9 +96,29 @@ class DbVersion(BaseApp):
 
     name = 'db_version'
 
+    @classmethod
+    def add_argument_parser(cls, subparsers):
+        parser = super(DbVersion, cls).add_argument_parser(subparsers)
+        parser.add_argument('--extension', default=None,
+                            help=('Migrate the database for the specified '
+                                  'extension. If not provided, db_sync will '
+                                  'migrate the common repository.'))
+
     @staticmethod
     def main():
-        print(migration.db_version())
+        extension = CONF.command.extension
+        if extension:
+            try:
+                package_name = ("%s.%s.migrate_repo" %
+                                (contrib.__name__, extension))
+                package = importutils.import_module(package_name)
+                repo_path = os.path.abspath(os.path.dirname(package.__file__))
+                print(migration.db_version(repo_path))
+            except ImportError:
+                print _("This extension does not provide migrations.")
+                exit(1)
+        else:
+            print(migration.db_version())
 
 
 class BaseCertificateSetup(BaseApp):
diff --git a/keystone/common/sql/migration.py b/keystone/common/sql/migration.py
index 86e0254c..3cb9cd63 100644
--- a/keystone/common/sql/migration.py
+++ b/keystone/common/sql/migration.py
@@ -39,39 +39,51 @@ except ImportError:
         sys.exit('python-migrate is not installed. Exiting.')
 
 
-def db_sync(version=None):
+def migrate_repository(version, current_version, repo_path):
+    if version is None or version > current_version:
+        result = versioning_api.upgrade(CONF.sql.connection,
+                                        repo_path, version)
+    else:
+        result = versioning_api.downgrade(
+            CONF.sql.connection, repo_path, version)
+    return result
+
+
+def db_sync(version=None, repo_path=None):
     if version is not None:
         try:
             version = int(version)
         except ValueError:
             raise Exception(_('version should be an integer'))
+    if repo_path is None:
+        repo_path = find_migrate_repo()
+    current_version = db_version(repo_path=repo_path)
+    return migrate_repository(version, current_version, repo_path)
 
-    current_version = db_version()
-    repo_path = _find_migrate_repo()
-    if version is None or version > current_version:
-        return versioning_api.upgrade(CONF.sql.connection, repo_path, version)
-    else:
-        return versioning_api.downgrade(
-            CONF.sql.connection, repo_path, version)
 
-
-def db_version():
-    repo_path = _find_migrate_repo()
+def db_version(repo_path=None):
+    if repo_path is None:
+        repo_path = find_migrate_repo()
     try:
         return versioning_api.db_version(CONF.sql.connection, repo_path)
     except versioning_exceptions.DatabaseNotControlledError:
         return db_version_control(0)
 
 
-def db_version_control(version=None):
-    repo_path = _find_migrate_repo()
+def db_version_control(version=None, repo_path=None):
+    if repo_path is None:
+        repo_path = find_migrate_repo()
     versioning_api.version_control(CONF.sql.connection, repo_path, version)
     return version
 
 
-def _find_migrate_repo():
+def find_migrate_repo(package=None):
     """Get the path for the migrate repository."""
-    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+    if package is None:
+        file = __file__
+    else:
+        file = package.__file__
+    path = os.path.join(os.path.abspath(os.path.dirname(file)),
                         'migrate_repo')
     assert os.path.exists(path)
     return path
diff --git a/keystone/contrib/example/__init__.py b/keystone/contrib/example/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone/contrib/example/__init__.py
diff --git a/keystone/contrib/example/migrate_repo/__init__.py b/keystone/contrib/example/migrate_repo/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/__init__.py
diff --git a/keystone/contrib/example/migrate_repo/migrate.cfg b/keystone/contrib/example/migrate_repo/migrate.cfg
new file mode 100644
index 00000000..5b1b1c0a
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/migrate.cfg
@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=example
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False
diff --git a/keystone/contrib/example/migrate_repo/versions/001_example_table.py b/keystone/contrib/example/migrate_repo/versions/001_example_table.py
new file mode 100644
index 00000000..bb2203d3
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/versions/001_example_table.py
@@ -0,0 +1,45 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind
+    # migrate_engine to your metadata
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    # catalog
+
+    service_table = sql.Table(
+        'example',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('type', sql.String(255)),
+        sql.Column('extra', sql.Text()))
+    service_table.create(migrate_engine, checkfirst=True)
+
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    tables = ['example']
+    for t in tables:
+        table = sql.Table(t, meta, autoload=True)
+        table.drop(migrate_engine, checkfirst=True)
diff --git a/keystone/contrib/example/migrate_repo/versions/__init__.py b/keystone/contrib/example/migrate_repo/versions/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/versions/__init__.py
diff --git a/tests/test_sql_migrate_extensions.py b/tests/test_sql_migrate_extensions.py
new file mode 100644
index 00000000..4a529559
--- /dev/null
+++ b/tests/test_sql_migrate_extensions.py
@@ -0,0 +1,47 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""
+To run these tests against a live database:
+1. Modify the file `tests/backend_sql.conf` to use the connection for your
+   live database
+2. Set up a blank, live database.
+3. run the tests using
+    ./run_tests.sh -N  test_sql_upgrade
+    WARNING::
+        Your database will be wiped.
+    Do not do this against a Database with valuable data as
+    all data will be lost.
+"""
+
+from keystone.contrib import example
+
+import test_sql_upgrade
+
+
+class SqlUpgradeExampleExtension(test_sql_upgrade.SqlMigrateBase):
+    def repo_package(self):
+        return example
+
+    def test_upgrade(self):
+        self.assertTableDoesNotExist('example')
+        self.upgrade(1, repository=self.repo_path)
+        self.assertTableColumns('example', ['id', 'type', 'extra'])
+
+    def test_downgrade(self):
+        self.upgrade(1, repository=self.repo_path)
+        self.assertTableColumns('example', ['id', 'type', 'extra'])
+        self.downgrade(0, repository=self.repo_path)
+        self.assertTableDoesNotExist('example')
diff --git a/tests/test_sql_upgrade.py b/tests/test_sql_upgrade.py
index ed00deae..1ac3506f 100644
--- a/tests/test_sql_upgrade.py
+++ b/tests/test_sql_upgrade.py
@@ -45,8 +45,7 @@ CONF = config.CONF
 DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
-class SqlUpgradeTests(test.TestCase):
-
+class SqlMigrateBase(test.TestCase):
     def initialize_sql(self):
         self.metadata = sqlalchemy.MetaData()
         self.metadata.bind = self.engine
@@ -55,12 +54,15 @@ class SqlUpgradeTests(test.TestCase):
                          test.testsdir('test_overrides.conf'),
                          test.testsdir('backend_sql.conf')]
 
-    #override this to sepcify the complete list of configuration files
+    #override this to specify the complete list of configuration files
     def config_files(self):
         return self._config_file_list
 
+    def repo_package(self):
+        return None
+
     def setUp(self):
-        super(SqlUpgradeTests, self).setUp()
+        super(SqlMigrateBase, self).setUp()
 
         self.config(self.config_files())
         self.base = sql.Base()
@@ -71,7 +73,7 @@ class SqlUpgradeTests(test.TestCase):
                                                   autocommit=False)
 
         self.initialize_sql()
-        self.repo_path = migration._find_migrate_repo()
+        self.repo_path = migration.find_migrate_repo(self.repo_package())
         self.schema = versioning_api.ControlledSchema.create(
             self.engine,
             self.repo_path, 0)
@@ -85,7 +87,64 @@ class SqlUpgradeTests(test.TestCase):
                                  autoload=True)
         self.downgrade(0)
         table.drop(self.engine, checkfirst=True)
-        super(SqlUpgradeTests, self).tearDown()
+        super(SqlMigrateBase, self).tearDown()
+
+    def select_table(self, name):
+        table = sqlalchemy.Table(name,
+                                 self.metadata,
+                                 autoload=True)
+        s = sqlalchemy.select([table])
+        return s
+
+    def assertTableExists(self, table_name):
+        try:
+            self.select_table(table_name)
+        except sqlalchemy.exc.NoSuchTableError:
+            raise AssertionError('Table "%s" does not exist' % table_name)
+
+    def assertTableDoesNotExist(self, table_name):
+        """Asserts that a given table exists cannot be selected by name."""
+        # Switch to a different metadata otherwise you might still
+        # detect renamed or dropped tables
+        try:
+            temp_metadata = sqlalchemy.MetaData()
+            temp_metadata.bind = self.engine
+            sqlalchemy.Table(table_name, temp_metadata, autoload=True)
+        except sqlalchemy.exc.NoSuchTableError:
+            pass
+        else:
+            raise AssertionError('Table "%s" already exists' % table_name)
+
+    def upgrade(self, *args, **kwargs):
+        self._migrate(*args, **kwargs)
+
+    def downgrade(self, *args, **kwargs):
+        self._migrate(*args, downgrade=True, **kwargs)
+
+    def _migrate(self, version, repository=None, downgrade=False,
+                 current_schema=None):
+        repository = repository or self.repo_path
+        err = ''
+        version = versioning_api._migrate_version(self.schema,
+                                                  version,
+                                                  not downgrade,
+                                                  err)
+        if not current_schema:
+            current_schema = self.schema
+        changeset = current_schema.changeset(version)
+        for ver, change in changeset:
+            self.schema.runchange(ver, change, changeset.step)
+        self.assertEqual(self.schema.version, version)
+
+    def assertTableColumns(self, table_name, expected_cols):
+        """Asserts that the table contains the expected set of columns."""
+        self.initialize_sql()
+        table = self.select_table(table_name)
+        actual_cols = [col.name for col in table.columns]
+        self.assertEqual(expected_cols, actual_cols, '%s table' % table_name)
+
+
+class SqlUpgradeTests(SqlMigrateBase):
 
     def test_blank_db_to_start(self):
         self.assertTableDoesNotExist('user')
@@ -108,13 +167,6 @@ class SqlUpgradeTests(test.TestCase):
             self.downgrade(x - 1)
             self.upgrade(x)
 
-    def assertTableColumns(self, table_name, expected_cols):
-        """Asserts that the table contains the expected set of columns."""
-        self.initialize_sql()
-        table = self.select_table(table_name)
-        actual_cols = [col.name for col in table.columns]
-        self.assertEqual(expected_cols, actual_cols, '%s table' % table_name)
-
     def test_upgrade_add_initial_tables(self):
         self.upgrade(1)
         self.assertTableColumns("user", ["id", "name", "extra"])
@@ -1260,50 +1312,6 @@ class SqlUpgradeTests(test.TestCase):
                          'extra': json.dumps(extra)})
             self.engine.execute(ins)
 
-    def select_table(self, name):
-        table = sqlalchemy.Table(name,
-                                 self.metadata,
-                                 autoload=True)
-        s = sqlalchemy.select([table])
-        return s
-
-    def assertTableExists(self, table_name):
-        try:
-            self.select_table(table_name)
-        except sqlalchemy.exc.NoSuchTableError:
-            raise AssertionError('Table "%s" does not exist' % table_name)
-
-    def assertTableDoesNotExist(self, table_name):
-        """Asserts that a given table exists cannot be selected by name."""
-        # Switch to a different metadata otherwise you might still
-        # detect renamed or dropped tables
-        try:
-            temp_metadata = sqlalchemy.MetaData()
-            temp_metadata.bind = self.engine
-            sqlalchemy.Table(table_name, temp_metadata, autoload=True)
-        except sqlalchemy.exc.NoSuchTableError:
-            pass
-        else:
-            raise AssertionError('Table "%s" already exists' % table_name)
-
-    def upgrade(self, *args, **kwargs):
-        self._migrate(*args, **kwargs)
-
-    def downgrade(self, *args, **kwargs):
-        self._migrate(*args, downgrade=True, **kwargs)
-
-    def _migrate(self, version, repository=None, downgrade=False):
-        repository = repository or self.repo_path
-        err = ''
-        version = versioning_api._migrate_version(self.schema,
-                                                  version,
-                                                  not downgrade,
-                                                  err)
-        changeset = self.schema.changeset(version)
-        for ver, change in changeset:
-            self.schema.runchange(ver, change, changeset.step)
-        self.assertEqual(self.schema.version, version)
-
     def _mysql_check_all_tables_innodb(self):
         database = self.engine.url.database