Simple usage extension for nova. Uses db to calculate tenant_usage for specified time periods.

Methods:
    index: return a list of tenant_usages, with option of incuding detailed server_usage
    show: returns a specific tenant_usage object

tenant_usage object:
    tenant_usage.total_memory_mb_usage: sum of memory_mb * hours for all instances in tenant for this period
    tenant_usage.total_local_gb_usage: sum of local_gb * hours for all instances in tenant for this period
    tenant_usage.total_vcpus_usage: sum of vcpus * hours for all instances in tenant for this period
    tenant_usage.total_hours: sum of all instance hours for this period
    tenant_usage.server_usages: A detailed list of server_usages, which describe the usage of a specific server

For larger instances db tables, indexes on instance.launched_at and instance.terminated_at should significantly help performance.

2 files changed, 457 insertions, 0 deletions

diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py
new file mode 100644
index 000000000..d578b2b67
--- /dev/null
+++ b/nova/api/openstack/contrib/simple_tenant_usage.py
@@ -0,0 +1,268 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import urlparse
+import webob
+
+from datetime import datetime
+from nova import db
+from nova import exception
+from nova import flags
+from nova.compute import instance_types
+from nova.api.openstack import extensions
+from nova.api.openstack import views
+from nova.db.sqlalchemy.session import get_session
+from webob import exc
+
+
+FLAGS = flags.FLAGS
+
+INSTANCE_FIELDS = ['id',
+                   'image_ref',
+                   'project_id',
+                   'user_id',
+                   'display_name',
+                   'state_description',
+                   'instance_type_id',
+                   'launched_at',
+                   'terminated_at']
+
+
+class SimpleTenantUsageController(object):
+
+    def _get_instances_for_time_period(self, period_start, period_stop,
+                                       tenant_id):
+        tenant_clause = ''
+        if tenant_id:
+            tenant_clause = " and project_id='%s'" % tenant_id
+
+        conn = get_session().connection()
+        rows = conn.execute("select %s from instances where \
+                            (terminated_at is NULL or terminated_at > '%s') \
+                            and (launched_at < '%s') %s" %\
+                            (','.join(INSTANCE_FIELDS),
+                            period_start.isoformat(' '),\
+                            period_stop.isoformat(' '),
+                            tenant_clause)).fetchall()
+
+        return rows
+
+    def _hours_for(self, instance, period_start, period_stop):
+        launched_at = instance['launched_at']
+        terminated_at = instance['terminated_at']
+        if terminated_at is not None:
+            if not isinstance(terminated_at, datetime):
+                terminated_at = datetime.strptime(terminated_at,
+                                                  "%Y-%m-%d %H:%M:%S.%f")
+
+        if launched_at is not None:
+            if not isinstance(launched_at, datetime):
+                launched_at = datetime.strptime(launched_at,
+                                                "%Y-%m-%d %H:%M:%S.%f")
+
+        if terminated_at and terminated_at < period_start:
+            return 0
+        # nothing if it started after the usage report ended
+        if launched_at and launched_at > period_stop:
+            return 0
+        if launched_at:
+            # if instance launched after period_started, don't charge for first
+            start = max(launched_at, period_start)
+            if terminated_at:
+                # if instance stopped before period_stop, don't charge after
+                stop = min(period_stop, terminated_at)
+            else:
+                # instance is still running, so charge them up to current time
+                stop = period_stop
+            dt = stop - start
+            seconds = dt.days * 3600 * 24 + dt.seconds\
+                      + dt.microseconds / 100000.0
+
+            return seconds / 3600.0
+        else:
+            # instance hasn't launched, so no charge
+            return 0
+
+    def _usage_for_period(self, context, period_start,
+                          period_stop, tenant_id=None, detailed=True):
+
+        rows = self._get_instances_for_time_period(period_start,
+                                                   period_stop,
+                                                   tenant_id)
+        rval = {}
+        flavors = {}
+
+        for row in rows:
+            info = {}
+            for i in range(len(INSTANCE_FIELDS)):
+                info[INSTANCE_FIELDS[i]] = row[i]
+            info['hours'] = self._hours_for(info, period_start, period_stop)
+            flavor_type = info['instance_type_id']
+
+            if not flavors.get(flavor_type):
+                try:
+                    flavors[flavor_type] = db.instance_type_get(context,
+                                                 info['instance_type_id'])
+                except exception.InstanceTypeNotFound:
+                    # can't bill if there is no instance type
+                    continue
+
+            flavor = flavors[flavor_type]
+
+            info['name'] = info['display_name']
+            del(info['display_name'])
+
+            info['memory_mb'] = flavor['memory_mb']
+            info['local_gb'] = flavor['local_gb']
+            info['vcpus'] = flavor['vcpus']
+
+            info['tenant_id'] = info['project_id']
+            del(info['project_id'])
+
+            info['flavor'] = flavor['name']
+            del(info['instance_type_id'])
+
+            info['started_at'] = info['launched_at']
+            del(info['launched_at'])
+
+            info['ended_at'] = info['terminated_at']
+            del(info['terminated_at'])
+
+            if info['ended_at']:
+                info['state'] = 'terminated'
+            else:
+                info['state'] = info['state_description']
+
+            del(info['state_description'])
+
+            now = datetime.utcnow()
+
+            if info['state'] == 'terminated':
+                delta = self._parse_datetime(info['ended_at'])\
+                             - self._parse_datetime(info['started_at'])
+            else:
+                delta = now - self._parse_datetime(info['started_at'])
+
+            info['uptime'] = delta.days * 24 * 60 + delta.seconds
+
+            if not info['tenant_id'] in rval:
+                summary = {}
+                summary['tenant_id'] = info['tenant_id']
+                if detailed:
+                    summary['server_usages'] = []
+                summary['total_local_gb_usage'] = 0
+                summary['total_vcpus_usage'] = 0
+                summary['total_memory_mb_usage'] = 0
+                summary['total_hours'] = 0
+                summary['start'] = period_start
+                summary['stop'] = period_stop
+                rval[info['tenant_id']] = summary
+
+            summary = rval[info['tenant_id']]
+            summary['total_local_gb_usage'] += info['local_gb'] * info['hours']
+            summary['total_vcpus_usage'] += info['vcpus'] * info['hours']
+            summary['total_memory_mb_usage'] += info['memory_mb']\
+                                                * info['hours']
+
+            summary['total_hours'] += info['hours']
+            if detailed:
+                summary['server_usages'].append(info)
+
+        return rval.values()
+
+    def _parse_datetime(self, dtstr):
+        if isinstance(dtstr, datetime):
+            return dtstr
+        try:
+            return datetime.strptime(dtstr, "%Y-%m-%dT%H:%M:%S")
+        except:
+            try:
+                return datetime.strptime(dtstr, "%Y-%m-%dT%H:%M:%S.%f")
+            except:
+                return datetime.strptime(dtstr, "%Y-%m-%d %H:%M:%S.%f")
+
+    def _get_datetime_range(self, req):
+        qs = req.environ.get('QUERY_STRING', '')
+        env = urlparse.parse_qs(qs)
+        period_start = self._parse_datetime(env.get('start',
+                                    [datetime.utcnow().isoformat()])[0])
+        period_stop = self._parse_datetime(env.get('end',
+                                    [datetime.utcnow().isoformat()])[0])
+
+        detailed = bool(env.get('detailed', False))
+        return (period_start, period_stop, detailed)
+
+    def index(self, req):
+        """Retrive tenant_usage for all tenants"""
+        (period_start, period_stop, detailed) = self._get_datetime_range(req)
+        context = req.environ['nova.context']
+
+        if not context.is_admin and FLAGS.allow_admin_api:
+            return webob.Response(status_int=403)
+
+        usages = self._usage_for_period(context,
+                                        period_start,
+                                        period_stop,
+                                        detailed=detailed)
+        return {'tenant_usages': usages}
+
+    def show(self, req, id):
+        """Retrive tenant_usage for a specified tenant"""
+        (period_start, period_stop, ignore) = self._get_datetime_range(req)
+        context = req.environ['nova.context']
+
+        if not context.is_admin and FLAGS.allow_admin_api:
+            if id != context.project_id:
+                return webob.Response(status_int=403)
+
+        usage = self._usage_for_period(context,
+                                       period_start,
+                                       period_stop,
+                                       id,
+                                       detailed=True)
+        if len(usage):
+            usage = usage[0]
+        else:
+            usage = {}
+        return {'tenant_usage': usage}
+
+
+class Simple_tenant_usage(extensions.ExtensionDescriptor):
+
+    def get_name(self):
+        return "Simple_tenant_usage"
+
+    def get_alias(self):
+        return "os-simple-tenant-usage"
+
+    def get_description(self):
+        return "Simple tenant usage extension"
+
+    def get_namespace(self):
+        return "http://docs.openstack.org/ext/os-simple-tenant-usage/api/v1.1"
+
+    def get_updated(self):
+        return "2011-08-19T00:00:00+00:00"
+
+    def get_resources(self):
+        resources = []
+
+        res = extensions.ResourceExtension('os-simple-tenant-usage',
+                                            SimpleTenantUsageController())
+        resources.append(res)
+
+        return resources
diff --git a/nova/tests/api/openstack/contrib/test_simple_tenant_usage.py b/nova/tests/api/openstack/contrib/test_simple_tenant_usage.py
new file mode 100644
index 000000000..d20e36aaf
--- /dev/null
+++ b/nova/tests/api/openstack/contrib/test_simple_tenant_usage.py
@@ -0,0 +1,189 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+import json
+import webob
+
+from nova import context
+from nova import db
+from nova import flags
+from nova import test
+from nova.compute import instance_types
+from nova.db.sqlalchemy import models
+from nova.db.sqlalchemy import session
+from nova.tests.api.openstack import fakes
+from webob import exc
+
+from nova.api.openstack.contrib import simple_tenant_usage
+
+
+FLAGS = flags.FLAGS
+
+SERVERS = 5
+TENANTS = 2
+HOURS = 24
+LOCAL_GB = 10
+MEMORY_MB = 1024
+VCPUS = 2
+STOP = datetime.datetime.utcnow()
+START = STOP - datetime.timedelta(hours=HOURS)
+
+
+def fake_get_session():
+    class FakeFetcher(object):
+        def fetchall(fetcher_self):
+            # return 10 rows, 2 tenants, 5 servers each, each run for 1 day
+            return [get_fake_db_row(START,
+                                    STOP,
+                                    x,
+                                    "faketenant_%s" % (x / SERVERS))
+                                    for x in xrange(TENANTS * SERVERS)]
+
+    class FakeConn(object):
+        def execute(self, query):
+            return FakeFetcher()
+
+    class FakeSession(object):
+        def connection(self):
+            return FakeConn()
+
+    return FakeSession()
+
+
+def fake_instance_type_get(context, instance_type_id):
+    return {'id': 1,
+            'vcpus': VCPUS,
+            'local_gb': LOCAL_GB,
+            'memory_mb': MEMORY_MB,
+            'name':
+            'fakeflavor'}
+
+
+def get_fake_db_row(start, end, instance_id, tenant_id):
+    return  [instance_id,
+            '1',
+            tenant_id,
+            'fakeuser',
+            'name',
+            'state',
+            1,
+            start,
+            None]
+
+
+class SimpleTenantUsageTest(test.TestCase):
+    def setUp(self):
+        super(SimpleTenantUsageTest, self).setUp()
+        self.stubs.Set(session, "get_session",
+                       fake_get_session)
+        self.stubs.Set(db, "instance_type_get",
+                       fake_instance_type_get)
+        self.admin_context = context.RequestContext('fakeadmin_0',
+                                                    'faketenant_0',
+                                                    is_admin=True)
+        self.user_context = context.RequestContext('fakeadmin_0',
+                                                   'faketenant_0',
+                                                    is_admin=False)
+        self.alt_user_context = context.RequestContext('fakeadmin_0',
+                                                      'faketenant_1',
+                                                       is_admin=False)
+        FLAGS.allow_admin_api = True
+
+    def test_verify_db_fields_exist_in_instance_model(self):
+        for field in simple_tenant_usage.INSTANCE_FIELDS:
+            self.assertTrue(field in models.Instance.__table__.columns)
+
+    def test_verify_index(self):
+        req = webob.Request.blank(
+                    '/v1.1/os-simple-tenant-usage?start=%s&end=%s' %
+                    (START.isoformat(), STOP.isoformat()))
+        req.method = "GET"
+        req.headers["content-type"] = "application/json"
+
+        res = req.get_response(fakes.wsgi_app(
+                               fake_auth_context=self.admin_context))
+
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+        usages = res_dict['tenant_usages']
+        for i in xrange(TENANTS):
+            self.assertEqual(int(usages[i]['total_hours']),
+                             SERVERS * HOURS)
+            self.assertEqual(int(usages[i]['total_local_gb_usage']),
+                             SERVERS * LOCAL_GB * HOURS)
+            self.assertEqual(int(usages[i]['total_memory_mb_usage']),
+                             SERVERS * MEMORY_MB * HOURS)
+            self.assertEqual(int(usages[i]['total_vcpus_usage']),
+                             SERVERS * VCPUS * HOURS)
+            self.assertFalse(usages[i].get('server_usages'))
+
+    def test_verify_detailed_index(self):
+        req = webob.Request.blank(
+                    '/v1.1/os-simple-tenant-usage?detailed=1&start=%s&end=%s' %
+                    (START.isoformat(), STOP.isoformat()))
+        req.method = "GET"
+        req.headers["content-type"] = "application/json"
+
+        res = req.get_response(fakes.wsgi_app(
+                               fake_auth_context=self.admin_context))
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+        usages = res_dict['tenant_usages']
+        for i in xrange(TENANTS):
+            servers = usages[i]['server_usages']
+            for j in xrange(SERVERS):
+                self.assertEqual(int(servers[j]['hours']), HOURS)
+
+    def test_verify_index_fails_for_nonadmin(self):
+        req = webob.Request.blank(
+                    '/v1.1/os-simple-tenant-usage?detailed=1&start=%s&end=%s' %
+                    (START.isoformat(), STOP.isoformat()))
+        req.method = "GET"
+        req.headers["content-type"] = "application/json"
+
+        res = req.get_response(fakes.wsgi_app())
+        self.assertEqual(res.status_int, 403)
+
+    def test_verify_show(self):
+        req = webob.Request.blank(
+                  '/v1.1/os-simple-tenant-usage/faketenant_0?start=%s&end=%s' %
+                  (START.isoformat(), STOP.isoformat()))
+        req.method = "GET"
+        req.headers["content-type"] = "application/json"
+
+        res = req.get_response(fakes.wsgi_app(
+                               fake_auth_context=self.user_context))
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+
+        usage = res_dict['tenant_usage']
+        servers = usage['server_usages']
+        self.assertEqual(len(usage['server_usages']), SERVERS)
+        for j in xrange(SERVERS):
+            self.assertEqual(int(servers[j]['hours']), HOURS)
+
+    def test_verify_show_cant_view_other_tenant(self):
+        req = webob.Request.blank(
+                  '/v1.1/os-simple-tenant-usage/faketenant_0?start=%s&end=%s' %
+                  (START.isoformat(), STOP.isoformat()))
+        req.method = "GET"
+        req.headers["content-type"] = "application/json"
+
+        res = req.get_response(fakes.wsgi_app(
+                               fake_auth_context=self.alt_user_context))
+        self.assertEqual(res.status_int, 403)