Merge "extension migrations"

10 files changed, 288 insertions, 77 deletions

diff --git a/doc/source/developing.rst b/doc/source/developing.rst
index c14ef7ab..2cf4b98e 100644
--- a/doc/source/developing.rst
+++ b/doc/source/developing.rst
@@ -71,6 +71,36 @@ place::
 
 .. _`python-keystoneclient`: https://github.com/openstack/python-keystoneclient
 
+Database Schema Migrations
+--------------------------
+
+Keystone uses SQLAlchemy-migrate
+_`SQLAlchemy-migrate`:http://code.google.com/p/sqlalchemy-migrate/ to migrate the SQL database
+between revisions.  For core components, the migrations are kept in a central
+repository under keystone/common/sql/migrate_repo.
+
+Extensions should be created as directories under `keystone/contrib`.  An
+extension that requires sql migrations should not change the common repository,
+but should instead have its own repository.  This repository must be in the
+extension's directory in `keystone/contrib/<extension>/migrate_repo.`  In
+addition it needs a subdirectory named `versions`.  For example, if the
+extension name is `my_extension` then the directory structure would be
+`keystone/contrib/my_extension/migrate_repo/versions/`.  For the migration
+o work, both the migrate_repo and versions subdirectories must have empty
+__init__.py files.  SQLAlchemy-migrate will look for a configuration file in
+the migrate_repo named migrate.cfg. This conforms to a Key/value ini file
+format.  A sample config file with the minimal set of values is::
+
+    [db_settings]
+    repository_id=my_extension
+    version_table=migrate_version
+    required_dbs=[]
+
+The directory `keystone/contrib/example` contains a sample extension migration.
+
+Migrations for extension must be explicitly run. To run a migration for a specific
+extension, run  `keystone-manage --extension <name> db_sync`.
+
 Initial Sample Data
 -------------------
 
diff --git a/keystone/cli.py b/keystone/cli.py
index 21d2ad40..18c095ce 100644
--- a/keystone/cli.py
+++ b/keystone/cli.py
@@ -20,12 +20,15 @@ import grp
 import os
 import pwd
 
+from migrate import exceptions
+
 from oslo.config import cfg
 import pbr.version
 
 from keystone.common import openssl
 from keystone.common.sql import migration
 from keystone import config
+from keystone import contrib
 from keystone.openstack.common import importutils
 from keystone.openstack.common import jsonutils
 from keystone import token
@@ -57,14 +60,35 @@ class DbSync(BaseApp):
                                   'version. If not provided, db_sync will '
                                   'migrate the database to the latest known '
                                   'version.'))
+        parser.add_argument('--extension', default=None,
+                            help=('Migrate the database for the specified '
+                                  'extension. If not provided, db_sync will '
+                                  'migrate the common repository.'))
+
         return parser
 
     @staticmethod
     def main():
-        for k in ['identity', 'catalog', 'policy', 'token', 'credential']:
-            driver = importutils.import_object(getattr(CONF, k).driver)
-            if hasattr(driver, 'db_sync'):
-                driver.db_sync(CONF.command.version)
+        version = CONF.command.version
+        extension = CONF.command.extension
+        if not extension:
+            migration.db_sync(version=version)
+        else:
+            package_name = "%s.%s.migrate_repo" % (contrib.__name__, extension)
+            try:
+                package = importutils.import_module(package_name)
+                repo_path = os.path.abspath(os.path.dirname(package.__file__))
+            except ImportError:
+                print _("This extension does not provide migrations.")
+                exit(0)
+            try:
+                # Register the repo with the version control API
+                # If it already knows about the repo, it will throw
+                # an exception that we can safely ignore
+                migration.db_version_control(version=None, repo_path=repo_path)
+            except exceptions.DatabaseAlreadyControlledError:
+                pass
+            migration.db_sync(version=None, repo_path=repo_path)
 
 
 class DbVersion(BaseApp):
@@ -72,9 +96,29 @@ class DbVersion(BaseApp):
 
     name = 'db_version'
 
+    @classmethod
+    def add_argument_parser(cls, subparsers):
+        parser = super(DbVersion, cls).add_argument_parser(subparsers)
+        parser.add_argument('--extension', default=None,
+                            help=('Migrate the database for the specified '
+                                  'extension. If not provided, db_sync will '
+                                  'migrate the common repository.'))
+
     @staticmethod
     def main():
-        print(migration.db_version())
+        extension = CONF.command.extension
+        if extension:
+            try:
+                package_name = ("%s.%s.migrate_repo" %
+                                (contrib.__name__, extension))
+                package = importutils.import_module(package_name)
+                repo_path = os.path.abspath(os.path.dirname(package.__file__))
+                print(migration.db_version(repo_path))
+            except ImportError:
+                print _("This extension does not provide migrations.")
+                exit(1)
+        else:
+            print(migration.db_version())
 
 
 class BaseCertificateSetup(BaseApp):
diff --git a/keystone/common/sql/migration.py b/keystone/common/sql/migration.py
index 86e0254c..3cb9cd63 100644
--- a/keystone/common/sql/migration.py
+++ b/keystone/common/sql/migration.py
@@ -39,39 +39,51 @@ except ImportError:
         sys.exit('python-migrate is not installed. Exiting.')
 
 
-def db_sync(version=None):
+def migrate_repository(version, current_version, repo_path):
+    if version is None or version > current_version:
+        result = versioning_api.upgrade(CONF.sql.connection,
+                                        repo_path, version)
+    else:
+        result = versioning_api.downgrade(
+            CONF.sql.connection, repo_path, version)
+    return result
+
+
+def db_sync(version=None, repo_path=None):
     if version is not None:
         try:
             version = int(version)
         except ValueError:
             raise Exception(_('version should be an integer'))
+    if repo_path is None:
+        repo_path = find_migrate_repo()
+    current_version = db_version(repo_path=repo_path)
+    return migrate_repository(version, current_version, repo_path)
 
-    current_version = db_version()
-    repo_path = _find_migrate_repo()
-    if version is None or version > current_version:
-        return versioning_api.upgrade(CONF.sql.connection, repo_path, version)
-    else:
-        return versioning_api.downgrade(
-            CONF.sql.connection, repo_path, version)
 
-
-def db_version():
-    repo_path = _find_migrate_repo()
+def db_version(repo_path=None):
+    if repo_path is None:
+        repo_path = find_migrate_repo()
     try:
         return versioning_api.db_version(CONF.sql.connection, repo_path)
     except versioning_exceptions.DatabaseNotControlledError:
         return db_version_control(0)
 
 
-def db_version_control(version=None):
-    repo_path = _find_migrate_repo()
+def db_version_control(version=None, repo_path=None):
+    if repo_path is None:
+        repo_path = find_migrate_repo()
     versioning_api.version_control(CONF.sql.connection, repo_path, version)
     return version
 
 
-def _find_migrate_repo():
+def find_migrate_repo(package=None):
     """Get the path for the migrate repository."""
-    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+    if package is None:
+        file = __file__
+    else:
+        file = package.__file__
+    path = os.path.join(os.path.abspath(os.path.dirname(file)),
                         'migrate_repo')
     assert os.path.exists(path)
     return path
diff --git a/keystone/contrib/example/__init__.py b/keystone/contrib/example/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone/contrib/example/__init__.py
diff --git a/keystone/contrib/example/migrate_repo/__init__.py b/keystone/contrib/example/migrate_repo/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/__init__.py
diff --git a/keystone/contrib/example/migrate_repo/migrate.cfg b/keystone/contrib/example/migrate_repo/migrate.cfg
new file mode 100644
index 00000000..5b1b1c0a
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/migrate.cfg
@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=example
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False
diff --git a/keystone/contrib/example/migrate_repo/versions/001_example_table.py b/keystone/contrib/example/migrate_repo/versions/001_example_table.py
new file mode 100644
index 00000000..bb2203d3
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/versions/001_example_table.py
@@ -0,0 +1,45 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy as sql
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine; bind
+    # migrate_engine to your metadata
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    # catalog
+
+    service_table = sql.Table(
+        'example',
+        meta,
+        sql.Column('id', sql.String(64), primary_key=True),
+        sql.Column('type', sql.String(255)),
+        sql.Column('extra', sql.Text()))
+    service_table.create(migrate_engine, checkfirst=True)
+
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    tables = ['example']
+    for t in tables:
+        table = sql.Table(t, meta, autoload=True)
+        table.drop(migrate_engine, checkfirst=True)
diff --git a/keystone/contrib/example/migrate_repo/versions/__init__.py b/keystone/contrib/example/migrate_repo/versions/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/keystone/contrib/example/migrate_repo/versions/__init__.py
diff --git a/tests/test_sql_migrate_extensions.py b/tests/test_sql_migrate_extensions.py
new file mode 100644
index 00000000..4a529559
--- /dev/null
+++ b/tests/test_sql_migrate_extensions.py
@@ -0,0 +1,47 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""
+To run these tests against a live database:
+1. Modify the file `tests/backend_sql.conf` to use the connection for your
+   live database
+2. Set up a blank, live database.
+3. run the tests using
+    ./run_tests.sh -N  test_sql_upgrade
+    WARNING::
+        Your database will be wiped.
+    Do not do this against a Database with valuable data as
+    all data will be lost.
+"""
+
+from keystone.contrib import example
+
+import test_sql_upgrade
+
+
+class SqlUpgradeExampleExtension(test_sql_upgrade.SqlMigrateBase):
+    def repo_package(self):
+        return example
+
+    def test_upgrade(self):
+        self.assertTableDoesNotExist('example')
+        self.upgrade(1, repository=self.repo_path)
+        self.assertTableColumns('example', ['id', 'type', 'extra'])
+
+    def test_downgrade(self):
+        self.upgrade(1, repository=self.repo_path)
+        self.assertTableColumns('example', ['id', 'type', 'extra'])
+        self.downgrade(0, repository=self.repo_path)
+        self.assertTableDoesNotExist('example')
diff --git a/tests/test_sql_upgrade.py b/tests/test_sql_upgrade.py
index cf82b814..5975fb9d 100644
--- a/tests/test_sql_upgrade.py
+++ b/tests/test_sql_upgrade.py
@@ -45,8 +45,7 @@ CONF = config.CONF
 DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id
 
 
-class SqlUpgradeTests(test.TestCase):
-
+class SqlMigrateBase(test.TestCase):
     def initialize_sql(self):
         self.metadata = sqlalchemy.MetaData()
         self.metadata.bind = self.engine
@@ -55,12 +54,15 @@ class SqlUpgradeTests(test.TestCase):
                          test.testsdir('test_overrides.conf'),
                          test.testsdir('backend_sql.conf')]
 
-    #override this to sepcify the complete list of configuration files
+    #override this to specify the complete list of configuration files
     def config_files(self):
         return self._config_file_list
 
+    def repo_package(self):
+        return None
+
     def setUp(self):
-        super(SqlUpgradeTests, self).setUp()
+        super(SqlMigrateBase, self).setUp()
 
         self.config(self.config_files())
         self.base = sql.Base()
@@ -71,7 +73,7 @@ class SqlUpgradeTests(test.TestCase):
                                                   autocommit=False)
 
         self.initialize_sql()
-        self.repo_path = migration._find_migrate_repo()
+        self.repo_path = migration.find_migrate_repo(self.repo_package())
         self.schema = versioning_api.ControlledSchema.create(
             self.engine,
             self.repo_path, 0)
@@ -85,7 +87,64 @@ class SqlUpgradeTests(test.TestCase):
                                  autoload=True)
         self.downgrade(0)
         table.drop(self.engine, checkfirst=True)
-        super(SqlUpgradeTests, self).tearDown()
+        super(SqlMigrateBase, self).tearDown()
+
+    def select_table(self, name):
+        table = sqlalchemy.Table(name,
+                                 self.metadata,
+                                 autoload=True)
+        s = sqlalchemy.select([table])
+        return s
+
+    def assertTableExists(self, table_name):
+        try:
+            self.select_table(table_name)
+        except sqlalchemy.exc.NoSuchTableError:
+            raise AssertionError('Table "%s" does not exist' % table_name)
+
+    def assertTableDoesNotExist(self, table_name):
+        """Asserts that a given table exists cannot be selected by name."""
+        # Switch to a different metadata otherwise you might still
+        # detect renamed or dropped tables
+        try:
+            temp_metadata = sqlalchemy.MetaData()
+            temp_metadata.bind = self.engine
+            sqlalchemy.Table(table_name, temp_metadata, autoload=True)
+        except sqlalchemy.exc.NoSuchTableError:
+            pass
+        else:
+            raise AssertionError('Table "%s" already exists' % table_name)
+
+    def upgrade(self, *args, **kwargs):
+        self._migrate(*args, **kwargs)
+
+    def downgrade(self, *args, **kwargs):
+        self._migrate(*args, downgrade=True, **kwargs)
+
+    def _migrate(self, version, repository=None, downgrade=False,
+                 current_schema=None):
+        repository = repository or self.repo_path
+        err = ''
+        version = versioning_api._migrate_version(self.schema,
+                                                  version,
+                                                  not downgrade,
+                                                  err)
+        if not current_schema:
+            current_schema = self.schema
+        changeset = current_schema.changeset(version)
+        for ver, change in changeset:
+            self.schema.runchange(ver, change, changeset.step)
+        self.assertEqual(self.schema.version, version)
+
+    def assertTableColumns(self, table_name, expected_cols):
+        """Asserts that the table contains the expected set of columns."""
+        self.initialize_sql()
+        table = self.select_table(table_name)
+        actual_cols = [col.name for col in table.columns]
+        self.assertEqual(expected_cols, actual_cols, '%s table' % table_name)
+
+
+class SqlUpgradeTests(SqlMigrateBase):
 
     def test_blank_db_to_start(self):
         self.assertTableDoesNotExist('user')
@@ -108,13 +167,6 @@ class SqlUpgradeTests(test.TestCase):
             self.downgrade(x - 1)
             self.upgrade(x)
 
-    def assertTableColumns(self, table_name, expected_cols):
-        """Asserts that the table contains the expected set of columns."""
-        self.initialize_sql()
-        table = self.select_table(table_name)
-        actual_cols = [col.name for col in table.columns]
-        self.assertEqual(expected_cols, actual_cols, '%s table' % table_name)
-
     def test_upgrade_add_initial_tables(self):
         self.upgrade(1)
         self.assertTableColumns("user", ["id", "name", "extra"])
@@ -1284,50 +1336,6 @@ class SqlUpgradeTests(test.TestCase):
                          'extra': json.dumps(extra)})
             self.engine.execute(ins)
 
-    def select_table(self, name):
-        table = sqlalchemy.Table(name,
-                                 self.metadata,
-                                 autoload=True)
-        s = sqlalchemy.select([table])
-        return s
-
-    def assertTableExists(self, table_name):
-        try:
-            self.select_table(table_name)
-        except sqlalchemy.exc.NoSuchTableError:
-            raise AssertionError('Table "%s" does not exist' % table_name)
-
-    def assertTableDoesNotExist(self, table_name):
-        """Asserts that a given table exists cannot be selected by name."""
-        # Switch to a different metadata otherwise you might still
-        # detect renamed or dropped tables
-        try:
-            temp_metadata = sqlalchemy.MetaData()
-            temp_metadata.bind = self.engine
-            sqlalchemy.Table(table_name, temp_metadata, autoload=True)
-        except sqlalchemy.exc.NoSuchTableError:
-            pass
-        else:
-            raise AssertionError('Table "%s" already exists' % table_name)
-
-    def upgrade(self, *args, **kwargs):
-        self._migrate(*args, **kwargs)
-
-    def downgrade(self, *args, **kwargs):
-        self._migrate(*args, downgrade=True, **kwargs)
-
-    def _migrate(self, version, repository=None, downgrade=False):
-        repository = repository or self.repo_path
-        err = ''
-        version = versioning_api._migrate_version(self.schema,
-                                                  version,
-                                                  not downgrade,
-                                                  err)
-        changeset = self.schema.changeset(version)
-        for ver, change in changeset:
-            self.schema.runchange(ver, change, changeset.step)
-        self.assertEqual(self.schema.version, version)
-
     def _mysql_check_all_tables_innodb(self):
         database = self.engine.url.database