diff options
author | Jenkins <jenkins@review.openstack.org> | 2013-08-07 19:36:58 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2013-08-07 19:36:58 +0000 |
commit | c547eb41097c86a41b6165088423df71b6ff7e01 (patch) | |
tree | 47b03728bdfc013ad870674100c527e14ae3c01e | |
parent | db9535cfb46db4a4f3a7fdb565ea9be4db2a9ef6 (diff) | |
parent | 10ef682f46e34e4e19d467c9b0e45f4f8838a134 (diff) | |
download | keystone-c547eb41097c86a41b6165088423df71b6ff7e01.tar.gz keystone-c547eb41097c86a41b6165088423df71b6ff7e01.tar.xz keystone-c547eb41097c86a41b6165088423df71b6ff7e01.zip |
Merge "extension migrations"
-rw-r--r-- | doc/source/developing.rst | 30 | ||||
-rw-r--r-- | keystone/cli.py | 54 | ||||
-rw-r--r-- | keystone/common/sql/migration.py | 42 | ||||
-rw-r--r-- | keystone/contrib/example/__init__.py | 0 | ||||
-rw-r--r-- | keystone/contrib/example/migrate_repo/__init__.py | 0 | ||||
-rw-r--r-- | keystone/contrib/example/migrate_repo/migrate.cfg | 25 | ||||
-rw-r--r-- | keystone/contrib/example/migrate_repo/versions/001_example_table.py | 45 | ||||
-rw-r--r-- | keystone/contrib/example/migrate_repo/versions/__init__.py | 0 | ||||
-rw-r--r-- | tests/test_sql_migrate_extensions.py | 47 | ||||
-rw-r--r-- | tests/test_sql_upgrade.py | 122 |
10 files changed, 288 insertions, 77 deletions
diff --git a/doc/source/developing.rst b/doc/source/developing.rst index c14ef7ab..2cf4b98e 100644 --- a/doc/source/developing.rst +++ b/doc/source/developing.rst @@ -71,6 +71,36 @@ place:: .. _`python-keystoneclient`: https://github.com/openstack/python-keystoneclient +Database Schema Migrations +-------------------------- + +Keystone uses SQLAlchemy-migrate +_`SQLAlchemy-migrate`:http://code.google.com/p/sqlalchemy-migrate/ to migrate the SQL database +between revisions. For core components, the migrations are kept in a central +repository under keystone/common/sql/migrate_repo. + +Extensions should be created as directories under `keystone/contrib`. An +extension that requires sql migrations should not change the common repository, +but should instead have its own repository. This repository must be in the +extension's directory in `keystone/contrib/<extension>/migrate_repo.` In +addition it needs a subdirectory named `versions`. For example, if the +extension name is `my_extension` then the directory structure would be +`keystone/contrib/my_extension/migrate_repo/versions/`. For the migration +o work, both the migrate_repo and versions subdirectories must have empty +__init__.py files. SQLAlchemy-migrate will look for a configuration file in +the migrate_repo named migrate.cfg. This conforms to a Key/value ini file +format. A sample config file with the minimal set of values is:: + + [db_settings] + repository_id=my_extension + version_table=migrate_version + required_dbs=[] + +The directory `keystone/contrib/example` contains a sample extension migration. + +Migrations for extension must be explicitly run. To run a migration for a specific +extension, run `keystone-manage --extension <name> db_sync`. + Initial Sample Data ------------------- diff --git a/keystone/cli.py b/keystone/cli.py index 21d2ad40..18c095ce 100644 --- a/keystone/cli.py +++ b/keystone/cli.py @@ -20,12 +20,15 @@ import grp import os import pwd +from migrate import exceptions + from oslo.config import cfg import pbr.version from keystone.common import openssl from keystone.common.sql import migration from keystone import config +from keystone import contrib from keystone.openstack.common import importutils from keystone.openstack.common import jsonutils from keystone import token @@ -57,14 +60,35 @@ class DbSync(BaseApp): 'version. If not provided, db_sync will ' 'migrate the database to the latest known ' 'version.')) + parser.add_argument('--extension', default=None, + help=('Migrate the database for the specified ' + 'extension. If not provided, db_sync will ' + 'migrate the common repository.')) + return parser @staticmethod def main(): - for k in ['identity', 'catalog', 'policy', 'token', 'credential']: - driver = importutils.import_object(getattr(CONF, k).driver) - if hasattr(driver, 'db_sync'): - driver.db_sync(CONF.command.version) + version = CONF.command.version + extension = CONF.command.extension + if not extension: + migration.db_sync(version=version) + else: + package_name = "%s.%s.migrate_repo" % (contrib.__name__, extension) + try: + package = importutils.import_module(package_name) + repo_path = os.path.abspath(os.path.dirname(package.__file__)) + except ImportError: + print _("This extension does not provide migrations.") + exit(0) + try: + # Register the repo with the version control API + # If it already knows about the repo, it will throw + # an exception that we can safely ignore + migration.db_version_control(version=None, repo_path=repo_path) + except exceptions.DatabaseAlreadyControlledError: + pass + migration.db_sync(version=None, repo_path=repo_path) class DbVersion(BaseApp): @@ -72,9 +96,29 @@ class DbVersion(BaseApp): name = 'db_version' + @classmethod + def add_argument_parser(cls, subparsers): + parser = super(DbVersion, cls).add_argument_parser(subparsers) + parser.add_argument('--extension', default=None, + help=('Migrate the database for the specified ' + 'extension. If not provided, db_sync will ' + 'migrate the common repository.')) + @staticmethod def main(): - print(migration.db_version()) + extension = CONF.command.extension + if extension: + try: + package_name = ("%s.%s.migrate_repo" % + (contrib.__name__, extension)) + package = importutils.import_module(package_name) + repo_path = os.path.abspath(os.path.dirname(package.__file__)) + print(migration.db_version(repo_path)) + except ImportError: + print _("This extension does not provide migrations.") + exit(1) + else: + print(migration.db_version()) class BaseCertificateSetup(BaseApp): diff --git a/keystone/common/sql/migration.py b/keystone/common/sql/migration.py index 86e0254c..3cb9cd63 100644 --- a/keystone/common/sql/migration.py +++ b/keystone/common/sql/migration.py @@ -39,39 +39,51 @@ except ImportError: sys.exit('python-migrate is not installed. Exiting.') -def db_sync(version=None): +def migrate_repository(version, current_version, repo_path): + if version is None or version > current_version: + result = versioning_api.upgrade(CONF.sql.connection, + repo_path, version) + else: + result = versioning_api.downgrade( + CONF.sql.connection, repo_path, version) + return result + + +def db_sync(version=None, repo_path=None): if version is not None: try: version = int(version) except ValueError: raise Exception(_('version should be an integer')) + if repo_path is None: + repo_path = find_migrate_repo() + current_version = db_version(repo_path=repo_path) + return migrate_repository(version, current_version, repo_path) - current_version = db_version() - repo_path = _find_migrate_repo() - if version is None or version > current_version: - return versioning_api.upgrade(CONF.sql.connection, repo_path, version) - else: - return versioning_api.downgrade( - CONF.sql.connection, repo_path, version) - -def db_version(): - repo_path = _find_migrate_repo() +def db_version(repo_path=None): + if repo_path is None: + repo_path = find_migrate_repo() try: return versioning_api.db_version(CONF.sql.connection, repo_path) except versioning_exceptions.DatabaseNotControlledError: return db_version_control(0) -def db_version_control(version=None): - repo_path = _find_migrate_repo() +def db_version_control(version=None, repo_path=None): + if repo_path is None: + repo_path = find_migrate_repo() versioning_api.version_control(CONF.sql.connection, repo_path, version) return version -def _find_migrate_repo(): +def find_migrate_repo(package=None): """Get the path for the migrate repository.""" - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + if package is None: + file = __file__ + else: + file = package.__file__ + path = os.path.join(os.path.abspath(os.path.dirname(file)), 'migrate_repo') assert os.path.exists(path) return path diff --git a/keystone/contrib/example/__init__.py b/keystone/contrib/example/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/keystone/contrib/example/__init__.py diff --git a/keystone/contrib/example/migrate_repo/__init__.py b/keystone/contrib/example/migrate_repo/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/keystone/contrib/example/migrate_repo/__init__.py diff --git a/keystone/contrib/example/migrate_repo/migrate.cfg b/keystone/contrib/example/migrate_repo/migrate.cfg new file mode 100644 index 00000000..5b1b1c0a --- /dev/null +++ b/keystone/contrib/example/migrate_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=example + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/keystone/contrib/example/migrate_repo/versions/001_example_table.py b/keystone/contrib/example/migrate_repo/versions/001_example_table.py new file mode 100644 index 00000000..bb2203d3 --- /dev/null +++ b/keystone/contrib/example/migrate_repo/versions/001_example_table.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + meta = sql.MetaData() + meta.bind = migrate_engine + + # catalog + + service_table = sql.Table( + 'example', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('type', sql.String(255)), + sql.Column('extra', sql.Text())) + service_table.create(migrate_engine, checkfirst=True) + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + meta = sql.MetaData() + meta.bind = migrate_engine + + tables = ['example'] + for t in tables: + table = sql.Table(t, meta, autoload=True) + table.drop(migrate_engine, checkfirst=True) diff --git a/keystone/contrib/example/migrate_repo/versions/__init__.py b/keystone/contrib/example/migrate_repo/versions/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/keystone/contrib/example/migrate_repo/versions/__init__.py diff --git a/tests/test_sql_migrate_extensions.py b/tests/test_sql_migrate_extensions.py new file mode 100644 index 00000000..4a529559 --- /dev/null +++ b/tests/test_sql_migrate_extensions.py @@ -0,0 +1,47 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +To run these tests against a live database: +1. Modify the file `tests/backend_sql.conf` to use the connection for your + live database +2. Set up a blank, live database. +3. run the tests using + ./run_tests.sh -N test_sql_upgrade + WARNING:: + Your database will be wiped. + Do not do this against a Database with valuable data as + all data will be lost. +""" + +from keystone.contrib import example + +import test_sql_upgrade + + +class SqlUpgradeExampleExtension(test_sql_upgrade.SqlMigrateBase): + def repo_package(self): + return example + + def test_upgrade(self): + self.assertTableDoesNotExist('example') + self.upgrade(1, repository=self.repo_path) + self.assertTableColumns('example', ['id', 'type', 'extra']) + + def test_downgrade(self): + self.upgrade(1, repository=self.repo_path) + self.assertTableColumns('example', ['id', 'type', 'extra']) + self.downgrade(0, repository=self.repo_path) + self.assertTableDoesNotExist('example') diff --git a/tests/test_sql_upgrade.py b/tests/test_sql_upgrade.py index cf82b814..5975fb9d 100644 --- a/tests/test_sql_upgrade.py +++ b/tests/test_sql_upgrade.py @@ -45,8 +45,7 @@ CONF = config.CONF DEFAULT_DOMAIN_ID = CONF.identity.default_domain_id -class SqlUpgradeTests(test.TestCase): - +class SqlMigrateBase(test.TestCase): def initialize_sql(self): self.metadata = sqlalchemy.MetaData() self.metadata.bind = self.engine @@ -55,12 +54,15 @@ class SqlUpgradeTests(test.TestCase): test.testsdir('test_overrides.conf'), test.testsdir('backend_sql.conf')] - #override this to sepcify the complete list of configuration files + #override this to specify the complete list of configuration files def config_files(self): return self._config_file_list + def repo_package(self): + return None + def setUp(self): - super(SqlUpgradeTests, self).setUp() + super(SqlMigrateBase, self).setUp() self.config(self.config_files()) self.base = sql.Base() @@ -71,7 +73,7 @@ class SqlUpgradeTests(test.TestCase): autocommit=False) self.initialize_sql() - self.repo_path = migration._find_migrate_repo() + self.repo_path = migration.find_migrate_repo(self.repo_package()) self.schema = versioning_api.ControlledSchema.create( self.engine, self.repo_path, 0) @@ -85,7 +87,64 @@ class SqlUpgradeTests(test.TestCase): autoload=True) self.downgrade(0) table.drop(self.engine, checkfirst=True) - super(SqlUpgradeTests, self).tearDown() + super(SqlMigrateBase, self).tearDown() + + def select_table(self, name): + table = sqlalchemy.Table(name, + self.metadata, + autoload=True) + s = sqlalchemy.select([table]) + return s + + def assertTableExists(self, table_name): + try: + self.select_table(table_name) + except sqlalchemy.exc.NoSuchTableError: + raise AssertionError('Table "%s" does not exist' % table_name) + + def assertTableDoesNotExist(self, table_name): + """Asserts that a given table exists cannot be selected by name.""" + # Switch to a different metadata otherwise you might still + # detect renamed or dropped tables + try: + temp_metadata = sqlalchemy.MetaData() + temp_metadata.bind = self.engine + sqlalchemy.Table(table_name, temp_metadata, autoload=True) + except sqlalchemy.exc.NoSuchTableError: + pass + else: + raise AssertionError('Table "%s" already exists' % table_name) + + def upgrade(self, *args, **kwargs): + self._migrate(*args, **kwargs) + + def downgrade(self, *args, **kwargs): + self._migrate(*args, downgrade=True, **kwargs) + + def _migrate(self, version, repository=None, downgrade=False, + current_schema=None): + repository = repository or self.repo_path + err = '' + version = versioning_api._migrate_version(self.schema, + version, + not downgrade, + err) + if not current_schema: + current_schema = self.schema + changeset = current_schema.changeset(version) + for ver, change in changeset: + self.schema.runchange(ver, change, changeset.step) + self.assertEqual(self.schema.version, version) + + def assertTableColumns(self, table_name, expected_cols): + """Asserts that the table contains the expected set of columns.""" + self.initialize_sql() + table = self.select_table(table_name) + actual_cols = [col.name for col in table.columns] + self.assertEqual(expected_cols, actual_cols, '%s table' % table_name) + + +class SqlUpgradeTests(SqlMigrateBase): def test_blank_db_to_start(self): self.assertTableDoesNotExist('user') @@ -108,13 +167,6 @@ class SqlUpgradeTests(test.TestCase): self.downgrade(x - 1) self.upgrade(x) - def assertTableColumns(self, table_name, expected_cols): - """Asserts that the table contains the expected set of columns.""" - self.initialize_sql() - table = self.select_table(table_name) - actual_cols = [col.name for col in table.columns] - self.assertEqual(expected_cols, actual_cols, '%s table' % table_name) - def test_upgrade_add_initial_tables(self): self.upgrade(1) self.assertTableColumns("user", ["id", "name", "extra"]) @@ -1284,50 +1336,6 @@ class SqlUpgradeTests(test.TestCase): 'extra': json.dumps(extra)}) self.engine.execute(ins) - def select_table(self, name): - table = sqlalchemy.Table(name, - self.metadata, - autoload=True) - s = sqlalchemy.select([table]) - return s - - def assertTableExists(self, table_name): - try: - self.select_table(table_name) - except sqlalchemy.exc.NoSuchTableError: - raise AssertionError('Table "%s" does not exist' % table_name) - - def assertTableDoesNotExist(self, table_name): - """Asserts that a given table exists cannot be selected by name.""" - # Switch to a different metadata otherwise you might still - # detect renamed or dropped tables - try: - temp_metadata = sqlalchemy.MetaData() - temp_metadata.bind = self.engine - sqlalchemy.Table(table_name, temp_metadata, autoload=True) - except sqlalchemy.exc.NoSuchTableError: - pass - else: - raise AssertionError('Table "%s" already exists' % table_name) - - def upgrade(self, *args, **kwargs): - self._migrate(*args, **kwargs) - - def downgrade(self, *args, **kwargs): - self._migrate(*args, downgrade=True, **kwargs) - - def _migrate(self, version, repository=None, downgrade=False): - repository = repository or self.repo_path - err = '' - version = versioning_api._migrate_version(self.schema, - version, - not downgrade, - err) - changeset = self.schema.changeset(version) - for ver, change in changeset: - self.schema.runchange(ver, change, changeset.step) - self.assertEqual(self.schema.version, version) - def _mysql_check_all_tables_innodb(self): database = self.engine.url.database |