Migrate quota schema from hardcoded columns to a key-value approach. The hope is that this change would make it easier to change the quota system without future schema changes. It also adds the concept of quotas that are unlimited.

7 files changed, 374 insertions, 53 deletions

diff --git a/bin/nova-manage b/bin/nova-manage
index a36ec86d0..c95b216ce 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -397,11 +397,10 @@ class ProjectCommands(object):
         arguments: project_id [key] [value]"""
         ctxt = context.get_admin_context()
         if key:
-            quo = {'project_id': project_id, key: value}
             try:
-                db.quota_update(ctxt, project_id, quo)
+                db.quota_update(ctxt, project_id, key, value)
             except exception.NotFound:
-                db.quota_create(ctxt, quo)
+                db.quota_create(ctxt, project_id, key, value)
         project_quota = quota.get_quota(ctxt, project_id)
         for key, value in project_quota.iteritems():
             print '%s: %s' % (key, value)
diff --git a/nova/db/api.py b/nova/db/api.py
index f9a4b5b4b..ef8aa1143 100644
--- a/nova/db/api.py
+++ b/nova/db/api.py
@@ -756,24 +756,34 @@ def auth_token_create(context, token):
 ###################
 
 
-def quota_create(context, values):
-    """Create a quota from the values dictionary."""
-    return IMPL.quota_create(context, values)
+def quota_create(context, project_id, resource, limit):
+    """Create a quota for the given project and resource."""
+    return IMPL.quota_create(context, project_id, resource, limit)
 
 
-def quota_get(context, project_id):
+def quota_get(context, project_id, resource):
     """Retrieve a quota or raise if it does not exist."""
-    return IMPL.quota_get(context, project_id)
+    return IMPL.quota_get(context, project_id, resource)
 
 
-def quota_update(context, project_id, values):
-    """Update a quota from the values dictionary."""
-    return IMPL.quota_update(context, project_id, values)
+def quota_get_all_by_project(context, project_id):
+    """Retrieve all quotas associated with a given project."""
+    return IMPL.quota_get_all_by_project(context, project_id)
 
 
-def quota_destroy(context, project_id):
+def quota_update(context, project_id, resource, limit):
+    """Update a quota or raise if it does not exist."""
+    return IMPL.quota_update(context, project_id, resource, limit)
+
+
+def quota_destroy(context, project_id, resource):
     """Destroy the quota or raise if it does not exist."""
-    return IMPL.quota_destroy(context, project_id)
+    return IMPL.quota_destroy(context, project_id, resource)
+
+
+def quota_destroy_all_by_project(context, project_id):
+    """Destroy all quotas associated with a given project."""
+    return IMPL.quota_get_all_by_project(context, project_id)
 
 
 ###################
diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py
index 2949eec49..3681f30db 100644
--- a/nova/db/sqlalchemy/api.py
+++ b/nova/db/sqlalchemy/api.py
@@ -1498,45 +1498,71 @@ def auth_token_create(_context, token):
 
 
 @require_admin_context
-def quota_get(context, project_id, session=None):
+def quota_get(context, project_id, resource, session=None):
     if not session:
         session = get_session()
-
     result = session.query(models.Quota).\
                      filter_by(project_id=project_id).\
-                     filter_by(deleted=can_read_deleted(context)).\
+                     filter_by(resource=resource).\
+                     filter_by(deleted=False).\
                      first()
     if not result:
         raise exception.ProjectQuotaNotFound(project_id=project_id)
+    return result
+
 
+@require_admin_context
+def quota_get_all_by_project(context, project_id):
+    session = get_session()
+    result = {'project_id': project_id}
+    rows = session.query(models.Quota).\
+                   filter_by(project_id=project_id).\
+                   filter_by(deleted=False).\
+                   all()
+    for row in rows:
+        result[row.resource] = row.hard_limit
     return result
 
 
 @require_admin_context
-def quota_create(context, values):
+def quota_create(context, project_id, resource, limit):
     quota_ref = models.Quota()
-    quota_ref.update(values)
+    quota_ref.project_id = project_id
+    quota_ref.resource = resource
+    quota_ref.hard_limit = limit
     quota_ref.save()
     return quota_ref
 
 
 @require_admin_context
-def quota_update(context, project_id, values):
+def quota_update(context, project_id, resource, limit):
     session = get_session()
     with session.begin():
-        quota_ref = quota_get(context, project_id, session=session)
-        quota_ref.update(values)
+        quota_ref = quota_get(context, project_id, resource, session=session)
+        quota_ref.hard_limit = limit
         quota_ref.save(session=session)
 
 
 @require_admin_context
-def quota_destroy(context, project_id):
+def quota_destroy(context, project_id, resource):
     session = get_session()
     with session.begin():
-        quota_ref = quota_get(context, project_id, session=session)
+        quota_ref = quota_get(context, project_id, resource, session=session)
         quota_ref.delete(session=session)
 
 
+@require_admin_context
+def quota_destroy_all_by_project(context, project_id):
+    session = get_session()
+    with session.begin():
+        quotas = session.query(models.Quota).\
+                         filter_by(project_id=project_id).\
+                         filter_by(deleted=False).\
+                         all()
+        for quota_ref in quotas:
+            quota_ref.delete(session=session)
+
+
 ###################
 
 
diff --git a/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py b/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py
new file mode 100644
index 000000000..a2d8192ca
--- /dev/null
+++ b/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py
@@ -0,0 +1,203 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import Boolean, Column, DateTime, Integer
+from sqlalchemy import MetaData, String, Table
+
+import datetime
+
+meta = MetaData()
+
+resources = [
+    'instances',
+    'cores',
+    'volumes',
+    'gigabytes',
+    'floating_ips',
+    'metadata_items',
+]
+
+
+def old_style_quotas_table(name):
+    return Table(name, meta,
+                 Column('id', Integer(), primary_key=True),
+                 Column('created_at', DateTime(),
+                        default=datetime.datetime.utcnow),
+                 Column('updated_at', DateTime(),
+                        onupdate=datetime.datetime.utcnow),
+                 Column('deleted_at', DateTime()),
+                 Column('deleted', Boolean(), default=False),
+                 Column('project_id',
+                        String(length=255, convert_unicode=False,
+                               assert_unicode=None, unicode_error=None,
+                               _warn_on_bytestring=False)),
+                 Column('instances', Integer()),
+                 Column('cores', Integer()),
+                 Column('volumes', Integer()),
+                 Column('gigabytes', Integer()),
+                 Column('floating_ips', Integer()),
+                 Column('metadata_items', Integer()),
+                )
+
+
+def new_style_quotas_table(name):
+    return Table(name, meta,
+                 Column('id', Integer(), primary_key=True),
+                 Column('created_at', DateTime(),
+                        default=datetime.datetime.utcnow),
+                 Column('updated_at', DateTime(),
+                        onupdate=datetime.datetime.utcnow),
+                 Column('deleted_at', DateTime()),
+                 Column('deleted', Boolean(), default=False),
+                 Column('project_id',
+                        String(length=255, convert_unicode=False,
+                               assert_unicode=None, unicode_error=None,
+                               _warn_on_bytestring=False)),
+                 Column('resource',
+                        String(length=255, convert_unicode=False,
+                               assert_unicode=None, unicode_error=None,
+                               _warn_on_bytestring=False),
+                        nullable=False),
+                 Column('hard_limit', Integer(), nullable=True),
+                )
+
+
+def existing_quotas_table(migrate_engine):
+    return Table('quotas', meta, autoload=True, autoload_with=migrate_engine)
+
+
+def _assert_no_duplicate_project_ids(quotas):
+    project_ids = set()
+    message = ('There are multiple active quotas for project "%s" '
+               '(among others, possibly). '
+               'Please resolve all ambiguous quotas before '
+               'reattempting the migration.')
+    for quota in quotas:
+        assert quota.project_id not in project_ids, message % quota.project_id
+        project_ids.add(quota.project_id)
+
+
+def assert_old_quotas_have_no_active_duplicates(migrate_engine, quotas):
+    """Ensure that there are no duplicate non-deleted quota entries."""
+    select = quotas.select().where(quotas.c.deleted == False)
+    results = migrate_engine.execute(select)
+    _assert_no_duplicate_project_ids(list(results))
+
+
+def assert_new_quotas_have_no_active_duplicates(migrate_engine, quotas):
+    """Ensure that there are no duplicate non-deleted quota entries."""
+    for resource in resources:
+        select = quotas.select().\
+                where(quotas.c.deleted == False).\
+                where(quotas.c.resource == resource)
+        results = migrate_engine.execute(select)
+        _assert_no_duplicate_project_ids(list(results))
+
+
+def convert_forward(migrate_engine, old_quotas, new_quotas):
+    quotas = list(migrate_engine.execute(old_quotas.select()))
+    for quota in quotas:
+        for resource in resources:
+            hard_limit = getattr(quota, resource)
+            if hard_limit is None:
+                continue
+            insert = new_quotas.insert().values(
+                created_at=quota.created_at,
+                updated_at=quota.updated_at,
+                deleted_at=quota.deleted_at,
+                deleted=quota.deleted,
+                project_id=quota.project_id,
+                resource=resource,
+                hard_limit=hard_limit)
+            migrate_engine.execute(insert)
+
+
+def earliest(date1, date2):
+    if date1 is None and date2 is None:
+        return None
+    if date1 is None:
+        return date2
+    if date2 is None:
+        return date1
+    if date1 < date2:
+        return date1
+    return date2
+
+
+def latest(date1, date2):
+    if date1 is None and date2 is None:
+        return None
+    if date1 is None:
+        return date2
+    if date2 is None:
+        return date1
+    if date1 > date2:
+        return date1
+    return date2
+
+
+def convert_backward(migrate_engine, old_quotas, new_quotas):
+    quotas = {}
+    for quota in migrate_engine.execute(new_quotas.select()):
+        if (quota.resource not in resources
+            or quota.hard_limit is None or quota.deleted):
+            continue
+        if not quota.project_id in quotas:
+            quotas[quota.project_id] = {
+                'project_id': quota.project_id,
+                'created_at': quota.created_at,
+                'updated_at': quota.updated_at,
+                quota.resource: quota.hard_limit
+            }
+        else:
+            quotas[quota.project_id]['created_at'] = earliest(
+                quota.created_at, quotas[quota.project_id]['created_at'])
+            quotas[quota.project_id]['updated_at'] = latest(
+                quota.updated_at, quotas[quota.project_id]['updated_at'])
+            quotas[quota.project_id][quota.resource] = quota.hard_limit
+
+    for quota in quotas.itervalues():
+        insert = old_quotas.insert().values(**quota)
+        migrate_engine.execute(insert)
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine;
+    # bind migrate_engine to your metadata
+    meta.bind = migrate_engine
+
+    old_quotas = existing_quotas_table(migrate_engine)
+    assert_old_quotas_have_no_active_duplicates(migrate_engine, old_quotas)
+
+    new_quotas = new_style_quotas_table('quotas_new')
+    new_quotas.create()
+    convert_forward(migrate_engine, old_quotas, new_quotas)
+    old_quotas.drop()
+    new_quotas.rename('quotas')
+
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    meta.bind = migrate_engine
+
+    new_quotas = existing_quotas_table(migrate_engine)
+    assert_new_quotas_have_no_active_duplicates(migrate_engine, new_quotas)
+
+    old_quotas = old_style_quotas_table('quotas_old')
+    old_quotas.create()
+    convert_backward(migrate_engine, old_quotas, new_quotas)
+    new_quotas.drop()
+    old_quotas.rename('quotas')
diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py
index 36a084a1d..0b46d5a05 100644
--- a/nova/db/sqlalchemy/models.py
+++ b/nova/db/sqlalchemy/models.py
@@ -313,18 +313,20 @@ class Volume(BASE, NovaBase):
 
 
 class Quota(BASE, NovaBase):
-    """Represents quota overrides for a project."""
+    """Represents a single quota override for a project.
+
+    If there is no row for a given project id and resource, then
+    the default for the deployment is used. If the row is present
+    but the hard limit is Null, then the resource is unlimited.
+    """
+
     __tablename__ = 'quotas'
     id = Column(Integer, primary_key=True)
 
-    project_id = Column(String(255))
+    project_id = Column(String(255), index=True)
 
-    instances = Column(Integer)
-    cores = Column(Integer)
-    volumes = Column(Integer)
-    gigabytes = Column(Integer)
-    floating_ips = Column(Integer)
-    metadata_items = Column(Integer)
+    resource = Column(String(255))
+    hard_limit = Column(Integer, nullable=True)
 
 
 class ExportDevice(BASE, NovaBase):
diff --git a/nova/quota.py b/nova/quota.py
index d8b5d9a93..a93cd0766 100644
--- a/nova/quota.py
+++ b/nova/quota.py
@@ -52,26 +52,31 @@ def get_quota(context, project_id):
             'floating_ips': FLAGS.quota_floating_ips,
             'metadata_items': FLAGS.quota_metadata_items}
 
-    try:
-        quota = db.quota_get(context, project_id)
-        for key in rval.keys():
-            if quota[key] is not None:
-                rval[key] = quota[key]
-    except exception.NotFound:
-        pass
+    quota = db.quota_get_all_by_project(context, project_id)
+    for key in rval.keys():
+        if key in quota:
+            rval[key] = quota[key]
     return rval
 
 
+def _get_request_allotment(requested, used, quota):
+    if quota is None:
+        return requested
+    return quota - used
+
+
 def allowed_instances(context, num_instances, instance_type):
     """Check quota and return min(num_instances, allowed_instances)."""
     project_id = context.project_id
     context = context.elevated()
+    num_cores = num_instances * instance_type['vcpus']
     used_instances, used_cores = db.instance_data_get_for_project(context,
                                                                   project_id)
     quota = get_quota(context, project_id)
-    allowed_instances = quota['instances'] - used_instances
-    allowed_cores = quota['cores'] - used_cores
-    num_cores = num_instances * instance_type['vcpus']
+    allowed_instances = _get_request_allotment(num_instances, used_instances,
+                                               quota['instances'])
+    allowed_cores = _get_request_allotment(num_cores, used_cores,
+                                           quota['cores'])
     allowed_instances = min(allowed_instances,
                             int(allowed_cores // instance_type['vcpus']))
     return min(num_instances, allowed_instances)
@@ -81,13 +86,15 @@ def allowed_volumes(context, num_volumes, size):
     """Check quota and return min(num_volumes, allowed_volumes)."""
     project_id = context.project_id
     context = context.elevated()
+    size = int(size)
+    num_gigabytes = num_volumes * size
     used_volumes, used_gigabytes = db.volume_data_get_for_project(context,
                                                                   project_id)
     quota = get_quota(context, project_id)
-    allowed_volumes = quota['volumes'] - used_volumes
-    allowed_gigabytes = quota['gigabytes'] - used_gigabytes
-    size = int(size)
-    num_gigabytes = num_volumes * size
+    allowed_volumes = _get_request_allotment(num_volumes, used_volumes,
+                                             quota['volumes'])
+    allowed_gigabytes = _get_request_allotment(num_gigabytes, used_gigabytes,
+                                               quota['gigabytes'])
     allowed_volumes = min(allowed_volumes,
                           int(allowed_gigabytes // size))
     return min(num_volumes, allowed_volumes)
@@ -99,7 +106,9 @@ def allowed_floating_ips(context, num_floating_ips):
     context = context.elevated()
     used_floating_ips = db.floating_ip_count_by_project(context, project_id)
     quota = get_quota(context, project_id)
-    allowed_floating_ips = quota['floating_ips'] - used_floating_ips
+    allowed_floating_ips = _get_request_allotment(num_floating_ips,
+                                                  used_floating_ips,
+                                                  quota['floating_ips'])
     return min(num_floating_ips, allowed_floating_ips)
 
 
@@ -108,8 +117,9 @@ def allowed_metadata_items(context, num_metadata_items):
     project_id = context.project_id
     context = context.elevated()
     quota = get_quota(context, project_id)
-    num_allowed_metadata_items = quota['metadata_items']
-    return min(num_metadata_items, num_allowed_metadata_items)
+    allowed_metadata_items = _get_request_allotment(num_metadata_items, 0,
+                                                    quota['metadata_items'])
+    return min(num_metadata_items, allowed_metadata_items)
 
 
 def allowed_injected_files(context):
diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py
index 39a123158..7ace2ad7d 100644
--- a/nova/tests/test_quota.py
+++ b/nova/tests/test_quota.py
@@ -96,12 +96,11 @@ class QuotaTestCase(test.TestCase):
         num_instances = quota.allowed_instances(self.context, 100,
             self._get_instance_type('m1.small'))
         self.assertEqual(num_instances, 2)
-        db.quota_create(self.context, {'project_id': self.project.id,
-                                       'instances': 10})
+        db.quota_create(self.context, self.project.id, 'instances', 10)
         num_instances = quota.allowed_instances(self.context, 100,
             self._get_instance_type('m1.small'))
         self.assertEqual(num_instances, 4)
-        db.quota_update(self.context, self.project.id, {'cores': 100})
+        db.quota_create(self.context, self.project.id, 'cores', 100)
         num_instances = quota.allowed_instances(self.context, 100,
             self._get_instance_type('m1.small'))
         self.assertEqual(num_instances, 10)
@@ -111,13 +110,85 @@ class QuotaTestCase(test.TestCase):
         num_metadata_items = quota.allowed_metadata_items(self.context,
                                                           too_many_items)
         self.assertEqual(num_metadata_items, FLAGS.quota_metadata_items)
-        db.quota_update(self.context, self.project.id, {'metadata_items': 5})
+        db.quota_create(self.context, self.project.id, 'metadata_items', 5)
         num_metadata_items = quota.allowed_metadata_items(self.context,
                                                           too_many_items)
         self.assertEqual(num_metadata_items, 5)
 
         # Cleanup
-        db.quota_destroy(self.context, self.project.id)
+        db.quota_destroy_all_by_project(self.context, self.project.id)
+
+    def test_unlimited_instances(self):
+        FLAGS.quota_instances = 2
+        FLAGS.quota_cores = 1000
+        instance_type = self._get_instance_type('m1.small')
+        num_instances = quota.allowed_instances(self.context, 100,
+                                                instance_type)
+        self.assertEqual(num_instances, 2)
+        db.quota_create(self.context, self.project.id, 'instances', None)
+        num_instances = quota.allowed_instances(self.context, 100,
+                                                instance_type)
+        self.assertEqual(num_instances, 100)
+        num_instances = quota.allowed_instances(self.context, 101,
+                                                instance_type)
+        self.assertEqual(num_instances, 101)
+
+    def test_unlimited_cores(self):
+        FLAGS.quota_instances = 1000
+        FLAGS.quota_cores = 2
+        instance_type = self._get_instance_type('m1.small')
+        num_instances = quota.allowed_instances(self.context, 100,
+                                                instance_type)
+        self.assertEqual(num_instances, 2)
+        db.quota_create(self.context, self.project.id, 'cores', None)
+        num_instances = quota.allowed_instances(self.context, 100,
+                                                instance_type)
+        self.assertEqual(num_instances, 100)
+        num_instances = quota.allowed_instances(self.context, 101,
+                                                instance_type)
+        self.assertEqual(num_instances, 101)
+
+    def test_unlimited_volumes(self):
+        FLAGS.quota_volumes = 10
+        FLAGS.quota_gigabytes = 1000
+        volumes = quota.allowed_volumes(self.context, 100, 1)
+        self.assertEqual(volumes, 10)
+        db.quota_create(self.context, self.project.id, 'volumes', None)
+        volumes = quota.allowed_volumes(self.context, 100, 1)
+        self.assertEqual(volumes, 100)
+        volumes = quota.allowed_volumes(self.context, 101, 1)
+        self.assertEqual(volumes, 101)
+
+    def test_unlimited_gigabytes(self):
+        FLAGS.quota_volumes = 1000
+        FLAGS.quota_gigabytes = 10
+        volumes = quota.allowed_volumes(self.context, 100, 1)
+        self.assertEqual(volumes, 10)
+        db.quota_create(self.context, self.project.id, 'gigabytes', None)
+        volumes = quota.allowed_volumes(self.context, 100, 1)
+        self.assertEqual(volumes, 100)
+        volumes = quota.allowed_volumes(self.context, 101, 1)
+        self.assertEqual(volumes, 101)
+
+    def test_unlimited_floating_ips(self):
+        FLAGS.quota_floating_ips = 10
+        floating_ips = quota.allowed_floating_ips(self.context, 100)
+        self.assertEqual(floating_ips, 10)
+        db.quota_create(self.context, self.project.id, 'floating_ips', None)
+        floating_ips = quota.allowed_floating_ips(self.context, 100)
+        self.assertEqual(floating_ips, 100)
+        floating_ips = quota.allowed_floating_ips(self.context, 101)
+        self.assertEqual(floating_ips, 101)
+
+    def test_unlimited_metadata_items(self):
+        FLAGS.quota_metadata_items = 10
+        items = quota.allowed_metadata_items(self.context, 100)
+        self.assertEqual(items, 10)
+        db.quota_create(self.context, self.project.id, 'metadata_items', None)
+        items = quota.allowed_metadata_items(self.context, 100)
+        self.assertEqual(items, 100)
+        items = quota.allowed_metadata_items(self.context, 101)
+        self.assertEqual(items, 101)
 
     def test_too_many_instances(self):
         instance_ids = []