From 119808d60247ebe59cf84f89e76a262732da2bd9 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 5 Jan 2012 15:17:22 -0800 Subject: still wip, got migration mostly working --- keystonelight/backends/sql.py | 347 -------------------- keystonelight/backends/sql/__init__.py | 1 + keystonelight/backends/sql/core.py | 351 +++++++++++++++++++++ keystonelight/backends/sql/migrate_repo/README | 4 + .../backends/sql/migrate_repo/__init__.py | 0 keystonelight/backends/sql/migrate_repo/manage.py | 5 + .../backends/sql/migrate_repo/migrate.cfg | 25 ++ .../versions/001_add_initial_tables.py | 14 + .../backends/sql/migrate_repo/versions/__init__.py | 0 keystonelight/backends/sql/migration.py | 74 +++++ tests/default.conf | 2 +- tests/test_backend_kvs.py | 4 +- tests/test_backend_sql.py | 154 +++++++++ 13 files changed, 631 insertions(+), 350 deletions(-) delete mode 100644 keystonelight/backends/sql.py create mode 100644 keystonelight/backends/sql/__init__.py create mode 100644 keystonelight/backends/sql/core.py create mode 100644 keystonelight/backends/sql/migrate_repo/README create mode 100644 keystonelight/backends/sql/migrate_repo/__init__.py create mode 100644 keystonelight/backends/sql/migrate_repo/manage.py create mode 100644 keystonelight/backends/sql/migrate_repo/migrate.cfg create mode 100644 keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py create mode 100644 keystonelight/backends/sql/migrate_repo/versions/__init__.py create mode 100644 keystonelight/backends/sql/migration.py create mode 100644 tests/test_backend_sql.py diff --git a/keystonelight/backends/sql.py b/keystonelight/backends/sql.py deleted file mode 100644 index 38dc57e4..00000000 --- a/keystonelight/backends/sql.py +++ /dev/null @@ -1,347 +0,0 @@ -"""SQL backends for the various services.""" - -import json - -import eventlet.db_pool -import sqlalchemy as sql -from sqlalchemy import types as sql_types -from sqlalchemy.ext import declarative -import sqlalchemy.orm -import sqlalchemy.pool -import sqlalchemy.engine.url - -from keystonelight import models - - -Base = declarative.declarative_base() - - -class JsonBlob(sql_types.TypeDecorator): - impl = sql.Text - - def process_bind_param(self, value, dialect): - return json.dumps(value) - - def process_result_value(self, value, dialect): - return json.loads(value) - - -class DictBase(object): - def __setitem__(self, key, value): - setattr(self, key, value) - - def __getitem__(self, key): - return getattr(self, key) - - def get(self, key, default=None): - return getattr(self, key, default) - - def __iter__(self): - self._i = iter(object_mapper(self).columns) - return self - - def next(self): - n = self._i.next().name - return n, getattr(self, n) - - def update(self, values): - """Make the model object behave like a dict.""" - for k, v in values.iteritems(): - setattr(self, k, v) - - def iteritems(self): - """Make the model object behave like a dict. - - Includes attributes from joins. - - """ - local = dict(self) - joined = dict([(k, v) for k, v in self.__dict__.iteritems() - if not k[0] == '_']) - local.update(joined) - return local.iteritems() - - -class User(Base, DictBase): - __tablename__ = 'user' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - password = sql.Column(sql.String(64)) - - -class Tenant(Base, DictBase): - __tablename__ = 'tenant' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), unique=True) - - -class Role(Base, DictBase): - __tablename__ = 'role' - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64)) - - -class Extras(Base, DictBase): - __tablename__ = 'extras' - __table_args__ = ( - sql.Index('idx_extras_usertenant', 'user', 'tenant'), - ) - - user = sql.Column(sql.String(64), primary_key=True) - tenant = sql.Column(sql.String(64), primary_key=True) - data = sql.Column(JsonBlob()) - - -class Token(Base, DictBase): - __tablename__ = 'token' - id = sql.Column(sql.String(64), primary_key=True) - user = sql.Column(sql.String(64)) - tenant = sql.Column(sql.String(64)) - data = sql.Column(JsonBlob()) - - - -class SqlBase(object): - _MAKER = None - _ENGINE = None - - def __init__(self, options): - self.options = options - - def get_session(self, autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy session.""" - if self._MAKER is None or self._ENGINE is None: - self._ENGINE = self.get_engine() - self._MAKER = self.get_maker(self._ENGINE, autocommit, expire_on_commit) - - session = self._MAKER() - # TODO(termie): we may want to do something similar - #session.query = nova.exception.wrap_db_error(session.query) - #session.flush = nova.exception.wrap_db_error(session.flush) - return session - - def get_engine(self): - """Return a SQLAlchemy engine.""" - connection_dict = sqlalchemy.engine.url.make_url( - self.options.get('sql_connection')) - - engine_args = { - "pool_recycle": self.options.get('sql_idle_timeout'), - "echo": False, - } - - if "sqlite" in connection_dict.drivername: - engine_args["poolclass"] = sqlalchemy.pool.NullPool - elif MySQLdb and "mysql" in connection_dict.drivername: - LOG.info(_("Using mysql/eventlet db_pool.")) - # MySQLdb won't accept 'None' in the password field - password = connection_dict.password or '' - pool_args = { - "db": connection_dict.database, - "passwd": password, - "host": connection_dict.host, - "user": connection_dict.username, - "min_size": self.options.get('sql_min_pool_size'), - "max_size": self.options.get('sql_max_pool_size'), - "max_idle": self.options.get('sql_idle_timeout'), - } - creator = eventlet.db_pool.ConnectionPool(MySQLdb, **pool_args) - engine_args["pool_size"] = self.options.get('sql_max_pool_size') - engine_args["pool_timeout"] = self.options('sql_pool_timeout') - engine_args["creator"] = creator.create - - return sql.create_engine(self.options.get('sql_connection'), - **engine_args) - - def get_maker(self, engine, autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy sessionmaker using the given engine.""" - return sqlalchemy.orm.sessionmaker(bind=engine, - autocommit=autocommit, - expire_on_commit=expire_on_commit) - - -class SqlIdentity(SqlBase): - def authenticate(self, user_id=None, tenant_id=None, password=None): - """Authenticate based on a user, tenant and password. - - Expects the user object to have a password field and the tenant to be - in the list of tenants on the user. - - """ - user_ref = self.get_user(user_id) - tenant_ref = None - extras_ref = None - if not user_ref or user_ref.get('password') != password: - raise AssertionError('Invalid user / password') - if tenant_id and tenant_id not in user_ref['tenants']: - raise AssertionError('Invalid tenant') - - tenant_ref = self.get_tenant(tenant_id) - if tenant_ref: - extras_ref = self.get_extras(user_id, tenant_id) - else: - extras_ref = {} - return (user_ref, tenant_ref, extras_ref) - - def get_tenant(self, tenant_id): - session = self.get_session() - tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() - return models.Tenant(**tenant_ref) - - def get_tenant_by_name(self, tenant_name): - tenant_ref = self.db.get('tenant_name-%s' % tenant_name) - return tenant_ref - - def get_user(self, user_id): - session = self.get_session() - user_ref = session.query(User).filter_by(id=user_id).first() - return models.User(**user_ref) - - def get_user_by_name(self, user_name): - user_ref = self.db.get('user_name-%s' % user_name) - return user_ref - - def get_extras(self, user_id, tenant_id): - return self.db.get('extras-%s-%s' % (tenant_id, user_id)) - - def get_role(self, role_id): - role_ref = self.db.get('role-%s' % role_id) - return role_ref - - def list_users(self): - return self.db.get('user_list', []) - - def list_roles(self): - return self.db.get('role_list', []) - - # These should probably be part of the high-level API - def add_user_to_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.add(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) - - def remove_user_from_tenant(self, tenant_id, user_id): - user_ref = self.get_user(user_id) - tenants = set(user_ref.get('tenants', [])) - tenants.remove(tenant_id) - user_ref['tenants'] = list(tenants) - self.update_user(user_id, user_ref) - - def get_tenants_for_user(self, user_id): - user_ref = self.get_user(user_id) - return user_ref.get('tenants', []) - - def get_roles_for_user_and_tenant(self, user_id, tenant_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - return extras_ref.get('roles', []) - - def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - roles = set(extras_ref.get('roles', [])) - roles.add(role_id) - extras_ref['roles'] = list(roles) - self.update_extras(user_id, tenant_id, extras_ref) - - def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): - extras_ref = self.get_extras(user_id, tenant_id) - if not extras_ref: - extras_ref = {} - roles = set(extras_ref.get('roles', [])) - roles.remove(role_id) - extras_ref['roles'] = list(roles) - self.update_extras(user_id, tenant_id, extras_ref) - - # CRUD - def create_user(self, id, user): - session = self.get_session() - session.add(User(**user)) - #self.db.set('user-%s' % id, user) - #self.db.set('user_name-%s' % user['name'], user) - #user_list = set(self.db.get('user_list', [])) - #user_list.add(id) - #self.db.set('user_list', list(user_list)) - return user - - def update_user(self, id, user): - # get the old name and delete it too - old_user = self.db.get('user-%s' % id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.set('user-%s' % id, user) - self.db.set('user_name-%s' % user['name'], user) - return user - - def delete_user(self, id): - old_user = self.db.get('user-%s' % id) - self.db.delete('user_name-%s' % old_user['name']) - self.db.delete('user-%s' % id) - user_list = set(self.db.get('user_list', [])) - user_list.remove(id) - self.db.set('user_list', list(user_list)) - return None - - def create_tenant(self, id, tenant): - session = self.get_session() - session.add(Tenant(**tenant)) - #session.commit() - #self.db.set('tenant-%s' % id, tenant) - #self.db.set('tenant_name-%s' % tenant['name'], tenant) - return models.Tenant(**tenant) - - def update_tenant(self, id, tenant): - # get the old name and delete it too - old_tenant = self.db.get('tenant-%s' % id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.set('tenant-%s' % id, tenant) - self.db.set('tenant_name-%s' % tenant['name'], tenant) - return tenant - - def delete_tenant(self, id): - old_tenant = self.db.get('tenant-%s' % id) - self.db.delete('tenant_name-%s' % old_tenant['name']) - self.db.delete('tenant-%s' % id) - return None - - def create_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) - return extras - - def update_extras(self, user_id, tenant_id, extras): - self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) - return extras - - def delete_extras(self, user_id, tenant_id): - self.db.delete('extras-%s-%s' % (tenant_id, user_id)) - return None - - def create_role(self, id, role): - self.db.set('role-%s' % id, role) - role_list = set(self.db.get('role_list', [])) - role_list.add(id) - self.db.set('role_list', list(role_list)) - return role - - def update_role(self, id, role): - self.db.set('role-%s' % id, role) - return role - - def delete_role(self, id): - self.db.delete('role-%s' % id) - role_list = set(self.db.get('role_list', [])) - role_list.remove(id) - self.db.set('role_list', list(role_list)) - return None - - - - -class SqlToken(SqlBase): - pass - -class SqlCatalog(SqlBase): - pass diff --git a/keystonelight/backends/sql/__init__.py b/keystonelight/backends/sql/__init__.py new file mode 100644 index 00000000..38ecafd5 --- /dev/null +++ b/keystonelight/backends/sql/__init__.py @@ -0,0 +1 @@ +from keystonelight.backends.sql.core import * diff --git a/keystonelight/backends/sql/core.py b/keystonelight/backends/sql/core.py new file mode 100644 index 00000000..2fee455f --- /dev/null +++ b/keystonelight/backends/sql/core.py @@ -0,0 +1,351 @@ +"""SQL backends for the various services.""" + +import json + +import eventlet.db_pool +import sqlalchemy as sql +from sqlalchemy import types as sql_types +from sqlalchemy.ext import declarative +import sqlalchemy.orm +import sqlalchemy.pool +import sqlalchemy.engine.url + +from keystonelight import models + + +Base = declarative.declarative_base() + + +class JsonBlob(sql_types.TypeDecorator): + impl = sql.Text + + def process_bind_param(self, value, dialect): + return json.dumps(value) + + def process_result_value(self, value, dialect): + return json.loads(value) + + +class DictBase(object): + def to_dict(self): + return dict(self.iteritems()) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __iter__(self): + self._i = iter(sqlalchemy.orm.object_mapper(self).columns) + return self + + def next(self): + n = self._i.next().name + return n, getattr(self, n) + + def update(self, values): + """Make the model object behave like a dict.""" + for k, v in values.iteritems(): + setattr(self, k, v) + + def iteritems(self): + """Make the model object behave like a dict. + + Includes attributes from joins. + + """ + local = dict(self) + joined = dict([(k, v) for k, v in self.__dict__.iteritems() + if not k[0] == '_']) + local.update(joined) + return local.iteritems() + + +class User(Base, DictBase): + __tablename__ = 'user' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + password = sql.Column(sql.String(64)) + + +class Tenant(Base, DictBase): + __tablename__ = 'tenant' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), unique=True) + + +class Role(Base, DictBase): + __tablename__ = 'role' + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64)) + + +class Extras(Base, DictBase): + __tablename__ = 'extras' + __table_args__ = ( + sql.Index('idx_extras_usertenant', 'user', 'tenant'), + ) + + user = sql.Column(sql.String(64), primary_key=True) + tenant = sql.Column(sql.String(64), primary_key=True) + data = sql.Column(JsonBlob()) + + +class Token(Base, DictBase): + __tablename__ = 'token' + id = sql.Column(sql.String(64), primary_key=True) + user = sql.Column(sql.String(64)) + tenant = sql.Column(sql.String(64)) + data = sql.Column(JsonBlob()) + + + +class SqlBase(object): + _MAKER = None + _ENGINE = None + + def __init__(self, options): + self.options = options + + def get_session(self, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy session.""" + if self._MAKER is None or self._ENGINE is None: + self._ENGINE = self.get_engine() + self._MAKER = self.get_maker(self._ENGINE, autocommit, expire_on_commit) + + session = self._MAKER() + # TODO(termie): we may want to do something similar + #session.query = nova.exception.wrap_db_error(session.query) + #session.flush = nova.exception.wrap_db_error(session.flush) + return session + + def get_engine(self): + """Return a SQLAlchemy engine.""" + connection_dict = sqlalchemy.engine.url.make_url( + self.options.get('sql_connection')) + + engine_args = { + "pool_recycle": self.options.get('sql_idle_timeout'), + "echo": False, + } + + if "sqlite" in connection_dict.drivername: + engine_args["poolclass"] = sqlalchemy.pool.NullPool + elif MySQLdb and "mysql" in connection_dict.drivername: + LOG.info(_("Using mysql/eventlet db_pool.")) + # MySQLdb won't accept 'None' in the password field + password = connection_dict.password or '' + pool_args = { + "db": connection_dict.database, + "passwd": password, + "host": connection_dict.host, + "user": connection_dict.username, + "min_size": self.options.get('sql_min_pool_size'), + "max_size": self.options.get('sql_max_pool_size'), + "max_idle": self.options.get('sql_idle_timeout'), + } + creator = eventlet.db_pool.ConnectionPool(MySQLdb, **pool_args) + engine_args["pool_size"] = self.options.get('sql_max_pool_size') + engine_args["pool_timeout"] = self.options('sql_pool_timeout') + engine_args["creator"] = creator.create + + return sql.create_engine(self.options.get('sql_connection'), + **engine_args) + + def get_maker(self, engine, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy sessionmaker using the given engine.""" + return sqlalchemy.orm.sessionmaker(bind=engine, + autocommit=autocommit, + expire_on_commit=expire_on_commit) + + +class SqlIdentity(SqlBase): + def authenticate(self, user_id=None, tenant_id=None, password=None): + """Authenticate based on a user, tenant and password. + + Expects the user object to have a password field and the tenant to be + in the list of tenants on the user. + + """ + user_ref = self.get_user(user_id) + tenant_ref = None + extras_ref = None + if not user_ref or user_ref.get('password') != password: + raise AssertionError('Invalid user / password') + if tenant_id and tenant_id not in user_ref['tenants']: + raise AssertionError('Invalid tenant') + + tenant_ref = self.get_tenant(tenant_id) + if tenant_ref: + extras_ref = self.get_extras(user_id, tenant_id) + else: + extras_ref = {} + return (user_ref, tenant_ref, extras_ref) + + def get_tenant(self, tenant_id): + session = self.get_session() + tenant_ref = session.query(Tenant).filter_by(id=tenant_id).first() + return tenant_ref + + def get_tenant_by_name(self, tenant_name): + tenant_ref = self.db.get('tenant_name-%s' % tenant_name) + return tenant_ref + + def get_user(self, user_id): + session = self.get_session() + user_ref = session.query(User).filter_by(id=user_id).first() + return user_ref + + def get_user_by_name(self, user_name): + user_ref = self.db.get('user_name-%s' % user_name) + return user_ref + + def get_extras(self, user_id, tenant_id): + return self.db.get('extras-%s-%s' % (tenant_id, user_id)) + + def get_role(self, role_id): + role_ref = self.db.get('role-%s' % role_id) + return role_ref + + def list_users(self): + return self.db.get('user_list', []) + + def list_roles(self): + return self.db.get('role_list', []) + + # These should probably be part of the high-level API + def add_user_to_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.add(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def remove_user_from_tenant(self, tenant_id, user_id): + user_ref = self.get_user(user_id) + tenants = set(user_ref.get('tenants', [])) + tenants.remove(tenant_id) + user_ref['tenants'] = list(tenants) + self.update_user(user_id, user_ref) + + def get_tenants_for_user(self, user_id): + user_ref = self.get_user(user_id) + return user_ref.get('tenants', []) + + def get_roles_for_user_and_tenant(self, user_id, tenant_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + return extras_ref.get('roles', []) + + def add_role_to_user_and_tenant(self, user_id, tenant_id, role_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + roles = set(extras_ref.get('roles', [])) + roles.add(role_id) + extras_ref['roles'] = list(roles) + self.update_extras(user_id, tenant_id, extras_ref) + + def remove_role_from_user_and_tenant(self, user_id, tenant_id, role_id): + extras_ref = self.get_extras(user_id, tenant_id) + if not extras_ref: + extras_ref = {} + roles = set(extras_ref.get('roles', [])) + roles.remove(role_id) + extras_ref['roles'] = list(roles) + self.update_extras(user_id, tenant_id, extras_ref) + + # CRUD + def create_user(self, id, user): + session = self.get_session() + session.add(User(**user)) + session.flush() + #self.db.set('user-%s' % id, user) + #self.db.set('user_name-%s' % user['name'], user) + #user_list = set(self.db.get('user_list', [])) + #user_list.add(id) + #self.db.set('user_list', list(user_list)) + return user + + def update_user(self, id, user): + # get the old name and delete it too + old_user = self.db.get('user-%s' % id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.set('user-%s' % id, user) + self.db.set('user_name-%s' % user['name'], user) + return user + + def delete_user(self, id): + old_user = self.db.get('user-%s' % id) + self.db.delete('user_name-%s' % old_user['name']) + self.db.delete('user-%s' % id) + user_list = set(self.db.get('user_list', [])) + user_list.remove(id) + self.db.set('user_list', list(user_list)) + return None + + def create_tenant(self, id, tenant): + session = self.get_session() + session.add(Tenant(**tenant)) + #session.commit() + #self.db.set('tenant-%s' % id, tenant) + #self.db.set('tenant_name-%s' % tenant['name'], tenant) + return models.Tenant(**tenant) + + def update_tenant(self, id, tenant): + # get the old name and delete it too + old_tenant = self.db.get('tenant-%s' % id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.set('tenant-%s' % id, tenant) + self.db.set('tenant_name-%s' % tenant['name'], tenant) + return tenant + + def delete_tenant(self, id): + old_tenant = self.db.get('tenant-%s' % id) + self.db.delete('tenant_name-%s' % old_tenant['name']) + self.db.delete('tenant-%s' % id) + return None + + def create_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + return extras + + def update_extras(self, user_id, tenant_id, extras): + self.db.set('extras-%s-%s' % (tenant_id, user_id), extras) + return extras + + def delete_extras(self, user_id, tenant_id): + self.db.delete('extras-%s-%s' % (tenant_id, user_id)) + return None + + def create_role(self, id, role): + self.db.set('role-%s' % id, role) + role_list = set(self.db.get('role_list', [])) + role_list.add(id) + self.db.set('role_list', list(role_list)) + return role + + def update_role(self, id, role): + self.db.set('role-%s' % id, role) + return role + + def delete_role(self, id): + self.db.delete('role-%s' % id) + role_list = set(self.db.get('role_list', [])) + role_list.remove(id) + self.db.set('role_list', list(role_list)) + return None + + + + +class SqlToken(SqlBase): + pass + +class SqlCatalog(SqlBase): + pass diff --git a/keystonelight/backends/sql/migrate_repo/README b/keystonelight/backends/sql/migrate_repo/README new file mode 100644 index 00000000..6218f8ca --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/keystonelight/backends/sql/migrate_repo/__init__.py b/keystonelight/backends/sql/migrate_repo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/keystonelight/backends/sql/migrate_repo/manage.py b/keystonelight/backends/sql/migrate_repo/manage.py new file mode 100644 index 00000000..39fa3892 --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(debug='False') diff --git a/keystonelight/backends/sql/migrate_repo/migrate.cfg b/keystonelight/backends/sql/migrate_repo/migrate.cfg new file mode 100644 index 00000000..a8be6089 --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=keystone + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py b/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py new file mode 100644 index 00000000..e5410694 --- /dev/null +++ b/keystonelight/backends/sql/migrate_repo/versions/001_add_initial_tables.py @@ -0,0 +1,14 @@ +from sqlalchemy import * +from migrate import * + +from keystonelight.backends import sql + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata + sql.Base.metadata.create_all(migrate_engine) + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + pass diff --git a/keystonelight/backends/sql/migrate_repo/versions/__init__.py b/keystonelight/backends/sql/migrate_repo/versions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/keystonelight/backends/sql/migration.py b/keystonelight/backends/sql/migration.py new file mode 100644 index 00000000..1a4794b7 --- /dev/null +++ b/keystonelight/backends/sql/migration.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import sqlalchemy +from migrate.versioning import api as versioning_api + +try: + from migrate.versioning import exceptions as versioning_exceptions +except ImportError: + try: + # python-migration changed location of exceptions after 1.6.3 + # See LP Bug #717467 + from migrate import exceptions as versioning_exceptions + except ImportError: + sys.exit("python-migrate is not installed. Exiting.") + + +def db_sync(options, version=None): + if version is not None: + try: + version = int(version) + except ValueError: + raise Exception("version should be an integer") + + current_version = db_version(options) + repo_path = _find_migrate_repo() + if version is None or version > current_version: + return versioning_api.upgrade( + options.get('sql_connection'), repo_path, version) + else: + return versioning_api.downgrade( + options.get('sql_connection'), repo_path, version) + + +def db_version(options): + repo_path = _find_migrate_repo() + try: + return versioning_api.db_version( + options.get('sql_connection'), repo_path) + except versioning_exceptions.DatabaseNotControlledError: + return db_version_control(options, 0) + + +def db_version_control(options, version=None): + repo_path = _find_migrate_repo() + versioning_api.version_control( + options.get('sql_connection'), repo_path, version) + return version + + +def _find_migrate_repo(): + """Get the path for the migrate repository.""" + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'migrate_repo') + assert os.path.exists(path) + return path diff --git a/tests/default.conf b/tests/default.conf index 766b569e..295fe6ff 100644 --- a/tests/default.conf +++ b/tests/default.conf @@ -22,7 +22,7 @@ catalog.RegionOne.compute.internalURL = http://localhost:$(compute_port)s/v1.1/$ catalog.RegionOne.compute.name = 'Compute Service' # for sql backends -sql_connection = sqlite:///:memory: +sql_connection = sqlite:///bla.db sql_idle_timeout = 200 sql_min_pool_size = 5 sql_max_pool_size = 10 diff --git a/tests/test_backend_kvs.py b/tests/test_backend_kvs.py index c88effdd..5c644d10 100644 --- a/tests/test_backend_kvs.py +++ b/tests/test_backend_kvs.py @@ -11,8 +11,8 @@ class KvsIdentity(test.TestCase): def setUp(self): super(KvsIdentity, self).setUp() self.options = self.appconfig('default') - self.identity_api = kvs.KvsIdentity(options=self.options, db={}) - self.load_fixtures(default_fixtures) + #self.identity_api = kvs.KvsIdentity(options=self.options, db={}) + #self.load_fixtures(default_fixtures) def test_authenticate_bad_user(self): self.assertRaises(AssertionError, diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py new file mode 100644 index 00000000..96f71eb6 --- /dev/null +++ b/tests/test_backend_sql.py @@ -0,0 +1,154 @@ +import os +import uuid + +from keystonelight import models +from keystonelight import test +from keystonelight.backends import sql +from keystonelight.backends.sql import migration + +import test_backend_kvs +import default_fixtures + + +class SqlIdentity(test_backend_kvs.KvsIdentity): + def setUp(self): + super(SqlIdentity, self).setUp() + self.options = self.appconfig('default') + os.unlink('bla.db') + migration.db_sync(self.options, 1) + self.identity_api = sql.SqlIdentity(options=self.options) + self.load_fixtures(default_fixtures) + + def test_authenticate_bad_user(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + + def test_authenticate_bad_password(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password'] + 'WRONG') + + def test_authenticate_invalid_tenant(self): + self.assertRaises(AssertionError, + self.identity_api.authenticate, + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG', + password=self.user_foo['password']) + + def test_authenticate_no_tenant(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assert_(tenant_ref is None) + self.assert_(not extras_ref) + + def test_authenticate(self): + user_ref, tenant_ref, extras_ref = self.identity_api.authenticate( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'], + password=self.user_foo['password']) + self.assertDictEquals(user_ref, self.user_foo) + self.assertDictEquals(tenant_ref, self.tenant_bar) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_tenant_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant(self): + tenant_ref = self.identity_api.get_tenant(tenant_id=self.tenant_bar['id']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_tenant_by_name_bad_tenant(self): + tenant_ref = self.identity_api.get_tenant( + tenant_id=self.tenant_bar['name'] + 'WRONG') + self.assert_(tenant_ref is None) + + def test_get_tenant_by_name(self): + tenant_ref = self.identity_api.get_tenant_by_name( + tenant_name=self.tenant_bar['name']) + self.assertDictEquals(tenant_ref, self.tenant_bar) + + def test_get_user_bad_user(self): + user_ref = self.identity_api.get_user( + user_id=self.user_foo['id'] + 'WRONG') + self.assert_(user_ref is None) + + def test_get_user(self): + user_ref = self.identity_api.get_user(user_id=self.user_foo['id']) + self.assertDictEquals(user_ref, self.user_foo) + + def test_get_extras_bad_user(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'] + 'WRONG', + tenant_id=self.tenant_bar['id']) + self.assert_(extras_ref is None) + + def test_get_extras_bad_tenant(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id'] + 'WRONG') + self.assert_(extras_ref is None) + + def test_get_extras(self): + extras_ref = self.identity_api.get_extras( + user_id=self.user_foo['id'], + tenant_id=self.tenant_bar['id']) + self.assertDictEquals(extras_ref, self.extras_foobar) + + def test_get_role(self): + role_ref = self.identity_api.get_role( + role_id=self.role_keystone_admin['id']) + self.assertDictEquals(role_ref, self.role_keystone_admin) + + +class SqlToken(test_backend_kvs.KvsToken): + def setUp(self): + super(SqlToken, self).setUp() + self.token_api = sql.SqlToken(options=options) + self.load_fixtures(default_fixtures) + + def test_token_crud(self): + token_id = uuid.uuid4().hex + data = {'id': token_id, + 'a': 'b'} + data_ref = self.token_api.create_token(token_id, data) + self.assertDictEquals(data_ref, data) + + new_data_ref = self.token_api.get_token(token_id) + self.assertEquals(new_data_ref, data) + + self.token_api.delete_token(token_id) + deleted_data_ref = self.token_api.get_token(token_id) + self.assert_(deleted_data_ref is None) + + +class SqlCatalog(test_backend_kvs.KvsCatalog): + def setUp(self): + super(SqlCatalog, self).setUp() + self.catalog_api = sql.SqlCatalog(options=options) + self._load_fixtures() + + def _load_fixtures(self): + self.catalog_foobar = self.catalog_api._create_catalog( + 'foo', 'bar', + {'RegionFoo': {'service_bar': {'foo': 'bar'}}}) + + def test_get_catalog_bad_user(self): + catalog_ref = self.catalog_api.get_catalog('foo' + 'WRONG', 'bar') + self.assert_(catalog_ref is None) + + def test_get_catalog_bad_tenant(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar' + 'WRONG') + self.assert_(catalog_ref is None) + + def test_get_catalog(self): + catalog_ref = self.catalog_api.get_catalog('foo', 'bar') + self.assertDictEquals(catalog_ref, self.catalog_foobar) -- cgit