From 501ff418df4c08e52a697772200aa08dd67a4a43 Mon Sep 17 00:00:00 2001 From: Gary Kotton Date: Thu, 9 May 2013 13:42:44 +0000 Subject: Nova instance group DB support DB support for blueprint instance-group-api-extension Change-Id: I615af9826ef61fd63d4cd8017908f943969bf177 --- nova/db/api.py | 90 ++++++ nova/db/sqlalchemy/api.py | 354 +++++++++++++++++++++ .../versions/187_add_instance_groups.py | 121 +++++++ nova/db/sqlalchemy/models.py | 69 ++++ 4 files changed, 634 insertions(+) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/187_add_instance_groups.py (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index 8a7c6dc48..7ecf6dfc0 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -708,6 +708,96 @@ def instance_remove_security_group(context, instance_id, security_group_id): security_group_id) +#################### + + +def instance_group_create(context, values, policies=None, metadata=None, + members=None): + """Create a new group with metadata. + + Each group will receive a unique uuid. This will be used for access to the + group. + """ + return IMPL.instance_group_create(context, values, policies, metadata, + members) + + +def instance_group_get(context, group_uuid): + """Get a specific group by id.""" + return IMPL.instance_group_get(context, group_uuid) + + +def instance_group_update(context, group_uuid, values): + """Update the attributes of an group.""" + return IMPL.instance_group_update(context, group_uuid, values) + + +def instance_group_delete(context, group_uuid): + """Delete an group.""" + return IMPL.instance_group_delete(context, group_uuid) + + +def instance_group_get_all(context): + """Get all groups.""" + return IMPL.instance_group_get_all(context) + + +def instance_group_get_all_by_project_id(context, project_id): + """Get all groups for a specific project_id.""" + return IMPL.instance_group_get_all_by_project_id(context, project_id) + + +def instance_group_metadata_add(context, group_uuid, metadata, + set_delete=False): + """Add metadata to the group.""" + return IMPL.instance_group_metadata_add(context, group_uuid, metadata, + set_delete) + + +def instance_group_metadata_delete(context, group_uuid, key): + """Delete metadata from the group.""" + return IMPL.instance_group_metadata_delete(context, group_uuid, key) + + +def instance_group_metadata_get(context, group_uuid): + """Get the metadata from the group.""" + return IMPL.instance_group_metadata_get(context, group_uuid) + + +def instance_group_members_add(context, group_uuid, members, + set_delete=False): + """Add members to the group.""" + return IMPL.instance_group_members_add(context, group_uuid, members, + set_delete=set_delete) + + +def instance_group_member_delete(context, group_uuid, instance_id): + """Delete a specific member from the group.""" + return IMPL.instance_group_member_delete(context, group_uuid, instance_id) + + +def instance_group_members_get(context, group_uuid): + """Get the members from the group.""" + return IMPL.instance_group_members_get(context, group_uuid) + + +def instance_group_policies_add(context, group_uuid, policies, + set_delete=False): + """Add policies to the group.""" + return IMPL.instance_group_policies_add(context, group_uuid, policies, + set_delete=set_delete) + + +def instance_group_policy_delete(context, group_uuid, policy): + """Delete a specific policy from the group.""" + return IMPL.instance_group_policy_delete(context, group_uuid, policy) + + +def instance_group_policies_get(context, group_uuid): + """Get the policies from the group.""" + return IMPL.instance_group_policies_get(context, group_uuid) + + ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index adacc6ead..d27f4e695 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -5187,3 +5187,357 @@ def archive_deleted_rows(context, max_rows=None): if rows_archived >= max_rows: break return rows_archived + + +#################### + + +def _instance_group_get_query(context, model_class, id_field=None, id=None, + session=None, read_deleted=None): + columns_to_join = {models.InstanceGroup: ['_policies', '_metadata', + '_members']} + query = model_query(context, model_class, session=session, + read_deleted=read_deleted) + + for c in columns_to_join.get(model_class, []): + query = query.options(joinedload(c)) + + if id and id_field: + query = query.filter(id_field == id) + + return query + + +def instance_group_create(context, values, policies=None, metadata=None, + members=None): + """Create a new group with metadata.""" + uuid = values.get('uuid', None) + if uuid is None: + uuid = uuidutils.generate_uuid() + values['uuid'] = uuid + session = get_session() + with session.begin(): + try: + group = models.InstanceGroup() + group.update(values) + group.save(session=session) + except db_exc.DBDuplicateEntry: + raise exception.InstanceGroupIdExists(group_uuid=uuid) + + # We don't want these to be lazy loaded later. We know there is + # nothing here since we just created this instance group. + group._policies = [] + group._metadata = [] + group._members = [] + if policies: + _instance_group_policies_add(context, group.id, policies, + session=session) + if metadata: + _instance_group_metadata_add(context, group.id, metadata, + session=session) + if members: + _instance_group_members_add(context, group.id, members, + session=session) + return instance_group_get(context, uuid) + + +def instance_group_get(context, group_uuid): + """Get a specific group by uuid.""" + group = _instance_group_get_query(context, + models.InstanceGroup, + models.InstanceGroup.uuid, + group_uuid).\ + first() + if not group: + raise exception.InstanceGroupNotFound(group_uuid=group_uuid) + return group + + +def instance_group_update(context, group_uuid, values): + """Update the attributes of an group. + + If values contains a metadata key, it updates the aggregate metadata + too. Similary for the policies and members. + """ + session = get_session() + with session.begin(): + group = model_query(context, + models.InstanceGroup, + session=session).\ + filter_by(uuid=group_uuid).\ + first() + if not group: + raise exception.InstanceGroupNotFound(group_uuid=group_uuid) + + policies = values.get('policies') + if policies is not None: + _instance_group_policies_add(context, + group.id, + values.pop('policies'), + set_delete=True, + session=session) + metadata = values.get('metadata') + if metadata is not None: + _instance_group_metadata_add(context, + group.id, + values.pop('metadata'), + set_delete=True, + session=session) + members = values.get('members') + if members is not None: + _instance_group_members_add(context, + group.id, + values.pop('members'), + set_delete=True, + session=session) + + group.update(values) + group.save(session=session) + if policies: + values['policies'] = policies + if metadata: + values['metadata'] = metadata + if members: + values['members'] = members + + +def instance_group_delete(context, group_uuid): + """Delete an group.""" + session = get_session() + with session.begin(): + count = _instance_group_get_query(context, + models.InstanceGroup, + models.InstanceGroup.uuid, + group_uuid, + session=session).soft_delete() + if count == 0: + raise exception.InstanceGroupNotFound(group_uuid=group_uuid) + + # Delete policies, metadata and members + instance_models = [models.InstanceGroupPolicy, + models.InstanceGroupMetadata, + models.InstanceGroupMember] + for model in instance_models: + model_query(context, model, session=session).\ + filter_by(group_id=group_uuid).\ + soft_delete() + + +def instance_group_get_all(context): + """Get all groups.""" + return _instance_group_get_query(context, models.InstanceGroup).all() + + +def instance_group_get_all_by_project_id(context, project_id): + """Get all groups.""" + return _instance_group_get_query(context, models.InstanceGroup).\ + filter_by(project_id=project_id).\ + all() + + +def _instance_group_model_get_query(context, model_class, group_uuid, + session=None, read_deleted='no'): + return model_query(context, + model_class, + read_deleted=read_deleted, + session=session).\ + filter_by(group_id=group_uuid) + + +def _instance_group_id(context, group_uuid): + """Returns the group database ID for the group UUID.""" + + result = model_query(context, + models.InstanceGroup.id, + base_model=models.InstanceGroup).\ + filter_by(uuid=group_uuid).\ + first() + if not result: + raise exception.InstanceGroupNotFound(group_uuid=group_uuid) + return result.id + + +def _instance_group_metadata_add(context, id, metadata, set_delete=False, + session=None): + if not session: + session = get_session() + + with session.begin(subtransactions=True): + all_keys = metadata.keys() + query = _instance_group_model_get_query(context, + models.InstanceGroupMetadata, + id, + session=session) + if set_delete: + query.filter(~models.InstanceGroupMetadata.key.in_(all_keys)).\ + soft_delete(synchronize_session=False) + + query = query.filter(models.InstanceGroupMetadata.key.in_(all_keys)) + already_existing_keys = set() + for meta_ref in query.all(): + key = meta_ref.key + meta_ref.update({'value': metadata[key]}) + already_existing_keys.add(key) + + for key, value in metadata.iteritems(): + if key in already_existing_keys: + continue + meta_ref = models.InstanceGroupMetadata() + meta_ref.update({'key': key, + 'value': value, + 'group_id': id}) + session.add(meta_ref) + + return metadata + + +def instance_group_metadata_add(context, group_uuid, metadata, + set_delete=False): + id = _instance_group_id(context, group_uuid) + return _instance_group_metadata_add(context, id, metadata, + set_delete=set_delete) + + +def instance_group_metadata_delete(context, group_uuid, key): + id = _instance_group_id(context, group_uuid) + count = _instance_group_get_query(context, + models.InstanceGroupMetadata, + models.InstanceGroupMetadata.group_id, + id).\ + filter_by(key=key).\ + soft_delete() + if count == 0: + raise exception.InstanceGroupMetadataNotFound(group_uuid=group_uuid, + metadata_key=key) + + +def instance_group_metadata_get(context, group_uuid): + id = _instance_group_id(context, group_uuid) + rows = model_query(context, + models.InstanceGroupMetadata.key, + models.InstanceGroupMetadata.value, + base_model=models.InstanceGroupMetadata).\ + filter_by(group_id=id).all() + return dict((r[0], r[1]) for r in rows) + + +def _instance_group_members_add(context, id, members, set_delete=False, + session=None): + if not session: + session = get_session() + + all_members = set(members) + with session.begin(subtransactions=True): + query = _instance_group_model_get_query(context, + models.InstanceGroupMember, + id, + session=session) + if set_delete: + query.filter(~models.InstanceGroupMember.instance_id.in_( + all_members)).\ + soft_delete(synchronize_session=False) + + query = query.filter( + models.InstanceGroupMember.instance_id.in_(all_members)) + already_existing = set() + for member_ref in query.all(): + already_existing.add(member_ref.instance_id) + + for instance_id in members: + if instance_id in already_existing: + continue + member_ref = models.InstanceGroupMember() + member_ref.update({'instance_id': instance_id, + 'group_id': id}) + session.add(member_ref) + + return members + + +def instance_group_members_add(context, group_uuid, members, + set_delete=False): + id = _instance_group_id(context, group_uuid) + return _instance_group_members_add(context, id, members, + set_delete=set_delete) + + +def instance_group_member_delete(context, group_uuid, instance_id): + id = _instance_group_id(context, group_uuid) + count = _instance_group_get_query(context, + models.InstanceGroupMember, + models.InstanceGroupMember.group_id, + id).\ + filter_by(instance_id=instance_id).\ + soft_delete() + if count == 0: + raise exception.InstanceGroupMemberNotFound(group_uuid=group_uuid, + instance_id=instance_id) + + +def instance_group_members_get(context, group_uuid): + id = _instance_group_id(context, group_uuid) + instances = model_query(context, + models.InstanceGroupMember.instance_id, + base_model=models.InstanceGroupMember).\ + filter_by(group_id=id).all() + return [instance[0] for instance in instances] + + +def _instance_group_policies_add(context, id, policies, set_delete=False, + session=None): + if not session: + session = get_session() + + allpols = set(policies) + with session.begin(subtransactions=True): + query = _instance_group_model_get_query(context, + models.InstanceGroupPolicy, + id, + session=session) + if set_delete: + query.filter(~models.InstanceGroupPolicy.policy.in_(allpols)).\ + soft_delete(synchronize_session=False) + + query = query.filter(models.InstanceGroupPolicy.policy.in_(allpols)) + already_existing = set() + for policy_ref in query.all(): + already_existing.add(policy_ref.policy) + + for policy in policies: + if policy in already_existing: + continue + policy_ref = models.InstanceGroupPolicy() + policy_ref.update({'policy': policy, + 'group_id': id}) + session.add(policy_ref) + + return policies + + +def instance_group_policies_add(context, group_uuid, policies, + set_delete=False): + id = _instance_group_id(context, group_uuid) + return _instance_group_policies_add(context, id, policies, + set_delete=set_delete) + + +def instance_group_policy_delete(context, group_uuid, policy): + id = _instance_group_id(context, group_uuid) + count = _instance_group_get_query(context, + models.InstanceGroupPolicy, + models.InstanceGroupPolicy.group_id, + id).\ + filter_by(policy=policy).\ + soft_delete() + if count == 0: + raise exception.InstanceGroupPolicyNotFound(group_uuid=group_uuid, + policy=policy) + + +def instance_group_policies_get(context, group_uuid): + id = _instance_group_id(context, group_uuid) + policies = model_query(context, + models.InstanceGroupPolicy.policy, + base_model=models.InstanceGroupPolicy).\ + filter_by(group_id=id).all() + return [policy[0] for policy in policies] diff --git a/nova/db/sqlalchemy/migrate_repo/versions/187_add_instance_groups.py b/nova/db/sqlalchemy/migrate_repo/versions/187_add_instance_groups.py new file mode 100644 index 000000000..227188e90 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/187_add_instance_groups.py @@ -0,0 +1,121 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate.changeset import UniqueConstraint +from sqlalchemy import Column +from sqlalchemy import DateTime +from sqlalchemy import ForeignKey +from sqlalchemy import Index +from sqlalchemy import Integer +from sqlalchemy import MetaData +from sqlalchemy import String +from sqlalchemy import Table + +from nova.db.sqlalchemy import api as db +from nova.db.sqlalchemy import utils + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + groups = Table('instance_groups', meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('deleted', Integer), + Column('id', Integer, primary_key=True, nullable=False), + Column('user_id', String(length=255)), + Column('project_id', String(length=255)), + Column('uuid', String(length=36), nullable=False), + Column('name', String(length=255)), + UniqueConstraint('uuid', 'deleted', + name='uniq_instance_groups0uuid0deleted'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + group_metadata = Table('instance_group_metadata', meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('deleted', Integer), + Column('id', Integer, primary_key=True, nullable=False), + Column('key', String(length=255)), + Column('value', String(length=255)), + Column('group_id', Integer, ForeignKey('instance_groups.id'), + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + group_policy = Table('instance_group_policy', meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('deleted', Integer), + Column('id', Integer, primary_key=True, nullable=False), + Column('policy', String(length=255)), + Column('group_id', Integer, ForeignKey('instance_groups.id'), + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + group_member = Table('instance_group_member', meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('deleted', Integer), + Column('id', Integer, primary_key=True, nullable=False), + Column('instance_id', String(length=255)), + Column('group_id', Integer, ForeignKey('instance_groups.id'), + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + tables = [groups, group_metadata, group_policy, group_member] + + # create all of the tables + for table in tables: + table.create() + utils.create_shadow_table(migrate_engine, table=table) + + indexes = [ + Index('instance_group_metadata_key_idx', group_metadata.c.key), + Index('instance_group_member_instance_idx', + group_member.c.instance_id), + Index('instance_group_policy_policy_idx', group_policy.c.policy) + ] + + # Common indexes + if migrate_engine.name == 'mysql' or migrate_engine.name == 'postgresql': + for index in indexes: + index.create(migrate_engine) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + table_names = ['instance_group_member', 'instance_group_policy', + 'instance_group_metadata', 'instance_groups'] + for name in table_names: + table = Table(name, meta, autoload=True) + table.drop() + table = Table(db._SHADOW_TABLE_PREFIX + name, meta, autoload=True) + table.drop() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 386fcbdad..28fe36a0d 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -1039,3 +1039,72 @@ class TaskLog(BASE, NovaBase): message = Column(String(255), nullable=False) task_items = Column(Integer(), default=0) errors = Column(Integer(), default=0) + + +class InstanceGroupMember(BASE, NovaBase): + """Represents the members for an instance group.""" + __tablename__ = 'instance_group_member' + id = Column(Integer, primary_key=True, nullable=False) + instance_id = Column(String(255), nullable=False) + group_id = Column(Integer, ForeignKey('instance_groups.id'), + nullable=False) + + +class InstanceGroupPolicy(BASE, NovaBase): + """Represents the policy type for an instance group.""" + __tablename__ = 'instance_group_policy' + id = Column(Integer, primary_key=True, nullable=False) + policy = Column(String(255), nullable=False) + group_id = Column(Integer, ForeignKey('instance_groups.id'), + nullable=False) + + +class InstanceGroupMetadata(BASE, NovaBase): + """Represents a key/value pair for an instance group.""" + __tablename__ = 'instance_group_metadata' + id = Column(Integer, primary_key=True, nullable=False) + key = Column(String(255), nullable=False) + value = Column(String(255), nullable=False) + group_id = Column(Integer, ForeignKey('instance_groups.id'), + nullable=False) + + +class InstanceGroup(BASE, NovaBase): + """Represents an instance group. + + A group will maintain a collection of instances and the relationship + between them. + """ + + __tablename__ = 'instance_groups' + __table_args__ = (schema.UniqueConstraint("uuid", "deleted"), ) + + id = Column(Integer, primary_key=True, autoincrement=True) + user_id = Column(String(255)) + project_id = Column(String(255)) + uuid = Column(String(36), nullable=False) + name = Column(String(255)) + _policies = relationship(InstanceGroupPolicy, primaryjoin='and_(' + 'InstanceGroup.id == InstanceGroupPolicy.group_id,' + 'InstanceGroupPolicy.deleted == 0,' + 'InstanceGroup.deleted == 0)') + _metadata = relationship(InstanceGroupMetadata, primaryjoin='and_(' + 'InstanceGroup.id == InstanceGroupMetadata.group_id,' + 'InstanceGroupMetadata.deleted == 0,' + 'InstanceGroup.deleted == 0)') + _members = relationship(InstanceGroupMember, primaryjoin='and_(' + 'InstanceGroup.id == InstanceGroupMember.group_id,' + 'InstanceGroupMember.deleted == 0,' + 'InstanceGroup.deleted == 0)') + + @property + def policies(self): + return [p.policy for p in self._policies] + + @property + def metadetails(self): + return dict((m.key, m.value) for m in self._metadata) + + @property + def members(self): + return [m.instance_id for m in self._members] -- cgit