diff options
| author | Kevin L. Mitchell <kevin.mitchell@rackspace.com> | 2012-05-04 19:27:43 -0500 |
|---|---|---|
| committer | Kevin L. Mitchell <kevin.mitchell@rackspace.com> | 2012-05-16 08:58:53 -0500 |
| commit | 406ff304bb09f144a59448e0e9d2d01160c7d553 (patch) | |
| tree | fc43d88a568100c552a20cda9b2c6168024ad057 /nova/db | |
| parent | 823a114727e514f153b500a16c7cad98253300f5 (diff) | |
Rearchitect quota checking to partially fix bug 938317.
This is a rearchitecting/rewriting of quota handling to correct the
quota atomicity issues highlighted by bug 938317. Partially implements
blueprint quota-refactor as well.
This change is fairly substantial. To make it easier to review, it has been
broken up into 3 parts. This is the first part.
Change-Id: I805f5750c08de17487e59fe33fad0bed203188a6
Diffstat (limited to 'nova/db')
| -rw-r--r-- | nova/db/api.py | 112 | ||||
| -rw-r--r-- | nova/db/sqlalchemy/api.py | 397 | ||||
| -rw-r--r-- | nova/db/sqlalchemy/migrate_repo/versions/097_quota_usages_reservations.py | 106 | ||||
| -rw-r--r-- | nova/db/sqlalchemy/models.py | 41 |
4 files changed, 622 insertions, 34 deletions
diff --git a/nova/db/api.py b/nova/db/api.py index fed92072d..f2f74cc55 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -251,9 +251,10 @@ def floating_ip_create(context, values): return IMPL.floating_ip_create(context, values) -def floating_ip_count_by_project(context, project_id): +def floating_ip_count_by_project(context, project_id, session=None): """Count floating ips used by project.""" - return IMPL.floating_ip_count_by_project(context, project_id) + return IMPL.floating_ip_count_by_project(context, project_id, + session=session) def floating_ip_deallocate(context, address): @@ -520,9 +521,10 @@ def instance_create(context, values): return IMPL.instance_create(context, values) -def instance_data_get_for_project(context, project_id): +def instance_data_get_for_project(context, project_id, session=None): """Get (instance_count, total_cores, total_ram) for project.""" - return IMPL.instance_data_get_for_project(context, project_id) + return IMPL.instance_data_get_for_project(context, project_id, + session=session) def instance_destroy(context, instance_id): @@ -900,11 +902,6 @@ def quota_destroy(context, project_id, resource): return IMPL.quota_destroy(context, project_id, resource) -def quota_destroy_all_by_project(context, project_id): - """Destroy all quotas associated with a given project.""" - return IMPL.quota_get_all_by_project(context, project_id) - - ################### @@ -941,6 +938,93 @@ def quota_class_destroy_all_by_name(context, class_name): ################### +def quota_usage_create(context, project_id, resource, in_use, reserved, + until_refresh): + """Create a quota usage for the given project and resource.""" + return IMPL.quota_usage_create(context, project_id, resource, + in_use, reserved, until_refresh) + + +def quota_usage_get(context, project_id, resource): + """Retrieve a quota usage or raise if it does not exist.""" + return IMPL.quota_usage_get(context, project_id, resource) + + +def quota_usage_get_all_by_project(context, project_id): + """Retrieve all usage associated with a given resource.""" + return IMPL.quota_usage_get_all_by_project(context, project_id) + + +def quota_usage_update(context, class_name, resource, in_use, reserved, + until_refresh): + """Update a quota usage or raise if it does not exist.""" + return IMPL.quota_usage_update(context, project_id, resource, + in_use, reserved, until_refresh) + + +def quota_usage_destroy(context, project_id, resource): + """Destroy the quota usage or raise if it does not exist.""" + return IMPL.quota_usage_destroy(context, project_id, resource) + + +################### + + +def reservation_create(context, uuid, usage, project_id, resource, delta, + expire): + """Create a reservation for the given project and resource.""" + return IMPL.reservation_create(context, uuid, usage, project_id, + resource, delta, expire) + + +def reservation_get(context, uuid): + """Retrieve a reservation or raise if it does not exist.""" + return IMPL.reservation_get(context, uuid) + + +def reservation_get_all_by_project(context, project_id): + """Retrieve all reservations associated with a given project.""" + return IMPL.reservation_get_all_by_project(context, project_id) + + +def reservation_destroy(context, uuid): + """Destroy the reservation or raise if it does not exist.""" + return IMPL.reservation_destroy(context, uuid) + + +################### + + +def quota_reserve(context, resources, quotas, deltas, expire, + until_refresh, max_age): + """Check quotas and create appropriate reservations.""" + return IMPL.quota_reserve(context, resources, quotas, deltas, expire, + until_refresh, max_age) + + +def reservation_commit(context, reservations): + """Commit quota reservations.""" + return IMPL.reservation_commit(context, reservations) + + +def reservation_rollback(context, reservations): + """Roll back quota reservations.""" + return IMPL.reservation_rollback(context, reservations) + + +def quota_destroy_all_by_project(context, project_id): + """Destroy all quotas associated with a given project.""" + return IMPL.quota_get_all_by_project(context, project_id) + + +def reservation_expire(context): + """Roll back any expired reservations.""" + return IMPL.reservation_expire(context) + + +################### + + def volume_allocate_iscsi_target(context, volume_id, host): """Atomically allocate a free iscsi_target from the pool.""" return IMPL.volume_allocate_iscsi_target(context, volume_id, host) @@ -956,9 +1040,10 @@ def volume_create(context, values): return IMPL.volume_create(context, values) -def volume_data_get_for_project(context, project_id): +def volume_data_get_for_project(context, project_id, session=None): """Get (volume_count, gigabytes) for project.""" - return IMPL.volume_data_get_for_project(context, project_id) + return IMPL.volume_data_get_for_project(context, project_id, + session=session) def volume_destroy(context, volume_id): @@ -1161,9 +1246,10 @@ def security_group_destroy(context, security_group_id): return IMPL.security_group_destroy(context, security_group_id) -def security_group_count_by_project(context, project_id): +def security_group_count_by_project(context, project_id, session=None): """Count number of security groups in a project.""" - return IMPL.security_group_count_by_project(context, project_id) + return IMPL.security_group_count_by_project(context, project_id, + session=session) #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 4574e8033..3a4e8ea1b 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -677,10 +677,11 @@ def floating_ip_create(context, values): @require_context -def floating_ip_count_by_project(context, project_id): +def floating_ip_count_by_project(context, project_id, session=None): authorize_project_context(context, project_id) # TODO(tr3buchet): why leave auto_assigned floating IPs out? - return model_query(context, models.FloatingIp, read_deleted="no").\ + return model_query(context, models.FloatingIp, read_deleted="no", + session=session).\ filter_by(project_id=project_id).\ filter_by(auto_assigned=False).\ count() @@ -1295,12 +1296,13 @@ def instance_create(context, values): @require_admin_context -def instance_data_get_for_project(context, project_id): +def instance_data_get_for_project(context, project_id, session=None): result = model_query(context, func.count(models.Instance.id), func.sum(models.Instance.vcpus), func.sum(models.Instance.memory_mb), - read_deleted="no").\ + read_deleted="no", + session=session).\ filter_by(project_id=project_id).\ first() # NOTE(vish): convert None to 0 @@ -2293,19 +2295,6 @@ def quota_destroy(context, project_id, resource): quota_ref.delete(session=session) -@require_admin_context -def quota_destroy_all_by_project(context, project_id): - session = get_session() - with session.begin(): - quotas = model_query(context, models.Quota, session=session, - read_deleted="no").\ - filter_by(project_id=project_id).\ - all() - - for quota_ref in quotas: - quota_ref.delete(session=session) - - ################### @@ -2383,6 +2372,370 @@ def quota_class_destroy_all_by_name(context, class_name): ################### +@require_context +def quota_usage_get(context, project_id, resource, session=None): + result = model_query(context, models.QuotaUsage, session=session, + read_deleted="no").\ + filter_by(project_id=project_id).\ + filter_by(resource=resource).\ + first() + + if not result: + raise exception.QuotaUsageNotFound(project_id=project_id) + + return result + + +@require_context +def quota_usage_get_all_by_project(context, project_id): + authorize_project_context(context, project_id) + + rows = model_query(context, models.QuotaUsage, read_deleted="no").\ + filter_by(project_id=project_id).\ + all() + + result = {'project_id': project_id} + for row in rows: + result[row.resource] = dict(in_use=row.in_use, reserved=row.reserved) + + return result + + +@require_admin_context +def quota_usage_create(context, project_id, resource, in_use, reserved, + until_refresh, session=None, save=True): + quota_usage_ref = models.QuotaUsage() + quota_usage_ref.project_id = project_id + quota_usage_ref.resource = resource + quota_usage_ref.in_use = in_use + quota_usage_ref.reserved = reserved + quota_usage_ref.until_refresh = until_refresh + + # Allow us to hold the save operation until later; keeps the + # transaction in quota_reserve() from breaking too early + if save: + quota_usage_ref.save(session=session) + + return quota_usage_ref + + +@require_admin_context +def quota_usage_update(context, project_id, resource, in_use, reserved, + until_refresh, session=None): + def do_update(session): + quota_usage_ref = quota_usage_get(context, project_id, resource, + session=session) + quota_usage_ref.in_use = in_use + quota_usage_ref.reserved = reserved + quota_usage_ref.until_refresh = until_refresh + quota_usage_ref.save(session=session) + + if session: + # Assume caller started a transaction + do_update(session) + else: + session = get_session() + with session.begin(): + do_update(session) + + +@require_admin_context +def quota_usage_destroy(context, project_id, resource): + session = get_session() + with session.begin(): + quota_usage_ref = quota_usage_get(context, project_id, resource, + session=session) + quota_usage_ref.delete(session=session) + + +################### + + +@require_context +def reservation_get(context, uuid, session=None): + result = model_query(context, models.Reservation, session=session, + read_deleted="no").\ + filter_by(uuid=uuid).\ + first() + + if not result: + raise exception.ReservationNotFound(uuid=uuid) + + return result + + +@require_context +def reservation_get_all_by_project(context, project_id): + authorize_project_context(context, project_id) + + rows = model_query(context, models.QuotaUsage, read_deleted="no").\ + filter_by(project_id=project_id).\ + all() + + result = {'project_id': project_id} + for row in rows: + result.setdefault(row.resource, {}) + result[row.resource][row.uuid] = row.delta + + return result + + +@require_admin_context +def reservation_create(context, uuid, usage, project_id, resource, delta, + expire, session=None): + reservation_ref = models.Reservation() + reservation_ref.uuid = uuid + reservation_ref.usage = usage + reservation_ref.project_id = project_id + reservation_ref.resource = resource + reservation_ref.delta = delta + reservation_ref.expire = expire + reservation_ref.save(session=session) + return reservation_ref + + +@require_admin_context +def reservation_destroy(context, uuid): + session = get_session() + with session.begin(): + reservation_ref = reservation_get(context, uuid, session=session) + reservation_ref.delete(session=session) + + +################### + + +def _get_quota_usages(context, session, keys): + # Broken out for testability + rows = model_query(context, models.QuotaUsage, + read_deleted="no", + session=session).\ + filter_by(project_id=context.project_id).\ + filter(models.QuotaUsage.resource.in_(keys)).\ + with_lockmode('update').\ + all() + return dict((row.resource, row) for row in rows) + + +@require_context +def quota_reserve(context, resources, quotas, deltas, expire, + until_refresh, max_age): + elevated = context.elevated() + session = get_session() + with session.begin(): + # Get the current usages + usages = _get_quota_usages(context, session, deltas.keys()) + + # Handle usage refresh + work = set(deltas.keys()) + while work: + resource = work.pop() + + # Do we need to refresh the usage? + refresh = False + if resource not in usages: + # Note we're inhibiting save... + usages[resource] = quota_usage_create(elevated, + context.project_id, + resource, + 0, 0, + until_refresh or None, + session=session, + save=False) + refresh = True + elif usages[resource].until_refresh is not None: + usages[resource].until_refresh -= 1 + if usages[resource].until_refresh <= 0: + refresh = True + elif max_age and (usages[resource].updated_at - + utils.utcnow()).seconds >= max_age: + refresh = True + + # OK, refresh the usage + if refresh: + # Grab the sync routine + sync = resources[resource].sync + + updates = sync(elevated, context.project_id, session) + for res, in_use in updates.items(): + # Make sure we have a destination for the usage! + if res not in usages: + # Note we're inhibiting save... + usages[res] = quota_usage_create(elevated, + context.project_id, + res, + 0, 0, + until_refresh or None, + session=session, + save=False) + + # Update the usage + usages[res].in_use = in_use + usages[res].until_refresh = until_refresh or None + + # Because more than one resource may be refreshed + # by the call to the sync routine, and we don't + # want to double-sync, we make sure all refreshed + # resources are dropped from the work set. + work.discard(res) + + # NOTE(Vek): We make the assumption that the sync + # routine actually refreshes the + # resources that it is the sync routine + # for. We don't check, because this is + # a best-effort mechanism. + + # Check for deltas that would go negative + unders = [resource for resource, delta in deltas.items() + if delta < 0 and + delta + usages[resource].in_use < 0] + + # Now, let's check the quotas + # NOTE(Vek): We're only concerned about positive increments. + # If a project has gone over quota, we want them to + # be able to reduce their usage without any + # problems. + overs = [resource for resource, delta in deltas.items() + if quotas[resource] >= 0 and delta >= 0 and + quotas[resource] < delta + usages[resource].total] + + # NOTE(Vek): The quota check needs to be in the transaction, + # but the transaction doesn't fail just because + # we're over quota, so the OverQuota raise is + # outside the transaction. If we did the raise + # here, our usage updates would be discarded, but + # they're not invalidated by being over-quota. + + # Create the reservations + if not unders and not overs: + reservations = [] + for resource, delta in deltas.items(): + reservation = reservation_create(elevated, + str(utils.gen_uuid()), + usages[resource], + context.project_id, + resource, delta, expire, + session=session) + reservations.append(reservation.uuid) + + # Also update the reserved quantity + # NOTE(Vek): Again, we are only concerned here about + # positive increments. Here, though, we're + # worried about the following scenario: + # + # 1) User initiates resize down. + # 2) User allocates a new instance. + # 3) Resize down fails or is reverted. + # 4) User is now over quota. + # + # To prevent this, we only update the + # reserved value if the delta is positive. + if delta > 0: + usages[resource].reserved += delta + + # Apply updates to the usages table + for usage_ref in usages.values(): + usage_ref.save(session=session) + + if unders: + raise exception.InvalidQuotaValue(unders=sorted(unders)) + if overs: + usages = dict((k, dict(in_use=v['in_use'], reserved=v['reserved'])) + for k, v in usages.items()) + raise exception.OverQuota(overs=sorted(overs), quotas=quotas, + usages=usages) + + return reservations + + +def _quota_reservations(session, context, reservations): + """Return the relevant reservations.""" + + # Get the listed reservations + return model_query(context, models.Reservation, + read_deleted="no", + session=session).\ + options(joinedload('usage')).\ + filter(models.Reservation.uuid.in_(reservations)).\ + with_lockmode('update').\ + all() + + +@require_context +def reservation_commit(context, reservations): + session = get_session() + with session.begin(): + for reservation in _quota_reservations(session, context, reservations): + if reservation.delta >= 0: + reservation.usage.reserved -= reservation.delta + reservation.usage.in_use += reservation.delta + + reservation.usage.save(session=session) + reservation.delete(session=session) + + +@require_context +def reservation_rollback(context, reservations): + session = get_session() + with session.begin(): + for reservation in _quota_reservations(session, context, reservations): + if reservation.delta >= 0: + reservation.usage.reserved -= reservation.delta + reservation.usage.save(session=session) + + reservation.delete(session=session) + + +@require_admin_context +def quota_destroy_all_by_project(context, project_id): + session = get_session() + with session.begin(): + quotas = model_query(context, models.Quota, session=session, + read_deleted="no").\ + filter_by(project_id=project_id).\ + all() + + for quota_ref in quotas: + quota_ref.delete(session=session) + + quota_usages = model_query(context, models.QuotaUsage, + session=session, read_deleted="no").\ + filter_by(project_id=project_id).\ + all() + + for quota_usage_ref in quota_usages: + quota_usage_ref.delete(session=session) + + reservations = model_query(context, models.Reservation, + session=session, read_deleted="no").\ + filter_by(project_id=project_id).\ + all() + + for reservation_ref in reservations: + reservation_ref.delete(session=session) + + +@require_admin_context +def reservation_expire(context): + session = get_session() + with session.begin(): + results = model_query(context, models.Reservation, session=session, + read_deleted="no").\ + filter(models.Reservation.expire < utils.utcnow()).\ + all() + + if results: + for reservation in results: + if reservation.delta >= 0: + reservation.usage.reserved -= reservation.delta + reservation.usage.save(session=session) + + reservation.delete(session=session) + + +################### + + @require_admin_context def volume_allocate_iscsi_target(context, volume_id, host): session = get_session() @@ -2438,11 +2791,12 @@ def volume_create(context, values): @require_admin_context -def volume_data_get_for_project(context, project_id): +def volume_data_get_for_project(context, project_id, session=None): result = model_query(context, func.count(models.Volume.id), func.sum(models.Volume.size), - read_deleted="no").\ + read_deleted="no", + session=session).\ filter_by(project_id=project_id).\ first() @@ -3010,9 +3364,10 @@ def security_group_destroy(context, security_group_id): @require_context -def security_group_count_by_project(context, project_id): +def security_group_count_by_project(context, project_id, session=None): authorize_project_context(context, project_id) - return model_query(context, models.SecurityGroup, read_deleted="no").\ + return model_query(context, models.SecurityGroup, read_deleted="no", + session=session).\ filter_by(project_id=project_id).\ count() diff --git a/nova/db/sqlalchemy/migrate_repo/versions/097_quota_usages_reservations.py b/nova/db/sqlalchemy/migrate_repo/versions/097_quota_usages_reservations.py new file mode 100644 index 000000000..f56cc71b9 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/097_quota_usages_reservations.py @@ -0,0 +1,106 @@ +# Copyright 2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Boolean, Column, DateTime +from sqlalchemy import MetaData, Integer, String, Table, ForeignKey + +from nova import log as logging + +LOG = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + # New tables + quota_usages = Table('quota_usages', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True), + Column('project_id', + String(length=255, convert_unicode=True, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + index=True), + Column('resource', + String(length=255, convert_unicode=True, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False)), + Column('in_use', Integer(), nullable=False), + Column('reserved', Integer(), nullable=False), + Column('until_refresh', Integer(), nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + try: + quota_usages.create() + except Exception: + LOG.error(_("Table |%s| not created!"), repr(quota_usages)) + raise + + reservations = Table('reservations', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True), + Column('uuid', + String(length=36, convert_unicode=True, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), nullable=False), + Column('usage_id', Integer(), ForeignKey('quota_usages.id'), + nullable=False), + Column('project_id', + String(length=255, convert_unicode=True, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + index=True), + Column('resource', + String(length=255, convert_unicode=True, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False)), + Column('delta', Integer(), nullable=False), + Column('expire', DateTime(timezone=False)), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + try: + reservations.create() + except Exception: + LOG.error(_("Table |%s| not created!"), repr(reservations)) + raise + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + quota_usages = Table('quota_usages', meta, autoload=True) + try: + quota_usages.drop() + except Exception: + LOG.error(_("quota_usages table not dropped")) + raise + + reservations = Table('reservations', meta, autoload=True) + try: + reservations.drop() + except Exception: + LOG.error(_("reservations table not dropped")) + raise diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 056ecd411..8a446091b 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -439,6 +439,47 @@ class QuotaClass(BASE, NovaBase): hard_limit = Column(Integer, nullable=True) +class QuotaUsage(BASE, NovaBase): + """Represents the current usage for a given resource.""" + + __tablename__ = 'quota_usages' + id = Column(Integer, primary_key=True) + + project_id = Column(String(255), index=True) + resource = Column(String(255)) + + in_use = Column(Integer) + reserved = Column(Integer) + + @property + def total(self): + return self.in_use + self.reserved + + until_refresh = Column(Integer, nullable=True) + + +class Reservation(BASE, NovaBase): + """Represents a resource reservation for quotas.""" + + __tablename__ = 'reservations' + id = Column(Integer, primary_key=True) + uuid = Column(String(36), nullable=False) + + usage_id = Column(Integer, ForeignKey('quota_usages.id'), nullable=False) + usage = relationship(QuotaUsage, + backref=backref('reservations'), + foreign_keys=usage_id, + primaryjoin='and_(' + 'Reservation.usage_id == QuotaUsage.id,' + 'Reservation.deleted == False)') + + project_id = Column(String(255), index=True) + resource = Column(String(255)) + + delta = Column(Integer) + expire = Column(DateTime, nullable=False) + + class Snapshot(BASE, NovaBase): """Represents a block storage device that can be attached to a vm.""" __tablename__ = 'snapshots' |
