From 7726b3d763a136347f2324e630f0a3cdc60a045b Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 22 Aug 2011 14:08:03 -0700 Subject: Simple usage extension for nova. Uses db to calculate tenant_usage for specified time periods. Methods: index: return a list of tenant_usages, with option of incuding detailed server_usage show: returns a specific tenant_usage object tenant_usage object: tenant_usage.total_memory_mb_usage: sum of memory_mb * hours for all instances in tenant for this period tenant_usage.total_local_gb_usage: sum of local_gb * hours for all instances in tenant for this period tenant_usage.total_vcpus_usage: sum of vcpus * hours for all instances in tenant for this period tenant_usage.total_hours: sum of all instance hours for this period tenant_usage.server_usages: A detailed list of server_usages, which describe the usage of a specific server For larger instances db tables, indexes on instance.launched_at and instance.terminated_at should significantly help performance. --- nova/api/openstack/contrib/simple_tenant_usage.py | 268 ++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 nova/api/openstack/contrib/simple_tenant_usage.py (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py new file mode 100644 index 000000000..d578b2b67 --- /dev/null +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -0,0 +1,268 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import urlparse +import webob + +from datetime import datetime +from nova import db +from nova import exception +from nova import flags +from nova.compute import instance_types +from nova.api.openstack import extensions +from nova.api.openstack import views +from nova.db.sqlalchemy.session import get_session +from webob import exc + + +FLAGS = flags.FLAGS + +INSTANCE_FIELDS = ['id', + 'image_ref', + 'project_id', + 'user_id', + 'display_name', + 'state_description', + 'instance_type_id', + 'launched_at', + 'terminated_at'] + + +class SimpleTenantUsageController(object): + + def _get_instances_for_time_period(self, period_start, period_stop, + tenant_id): + tenant_clause = '' + if tenant_id: + tenant_clause = " and project_id='%s'" % tenant_id + + conn = get_session().connection() + rows = conn.execute("select %s from instances where \ + (terminated_at is NULL or terminated_at > '%s') \ + and (launched_at < '%s') %s" %\ + (','.join(INSTANCE_FIELDS), + period_start.isoformat(' '),\ + period_stop.isoformat(' '), + tenant_clause)).fetchall() + + return rows + + def _hours_for(self, instance, period_start, period_stop): + launched_at = instance['launched_at'] + terminated_at = instance['terminated_at'] + if terminated_at is not None: + if not isinstance(terminated_at, datetime): + terminated_at = datetime.strptime(terminated_at, + "%Y-%m-%d %H:%M:%S.%f") + + if launched_at is not None: + if not isinstance(launched_at, datetime): + launched_at = datetime.strptime(launched_at, + "%Y-%m-%d %H:%M:%S.%f") + + if terminated_at and terminated_at < period_start: + return 0 + # nothing if it started after the usage report ended + if launched_at and launched_at > period_stop: + return 0 + if launched_at: + # if instance launched after period_started, don't charge for first + start = max(launched_at, period_start) + if terminated_at: + # if instance stopped before period_stop, don't charge after + stop = min(period_stop, terminated_at) + else: + # instance is still running, so charge them up to current time + stop = period_stop + dt = stop - start + seconds = dt.days * 3600 * 24 + dt.seconds\ + + dt.microseconds / 100000.0 + + return seconds / 3600.0 + else: + # instance hasn't launched, so no charge + return 0 + + def _usage_for_period(self, context, period_start, + period_stop, tenant_id=None, detailed=True): + + rows = self._get_instances_for_time_period(period_start, + period_stop, + tenant_id) + rval = {} + flavors = {} + + for row in rows: + info = {} + for i in range(len(INSTANCE_FIELDS)): + info[INSTANCE_FIELDS[i]] = row[i] + info['hours'] = self._hours_for(info, period_start, period_stop) + flavor_type = info['instance_type_id'] + + if not flavors.get(flavor_type): + try: + flavors[flavor_type] = db.instance_type_get(context, + info['instance_type_id']) + except exception.InstanceTypeNotFound: + # can't bill if there is no instance type + continue + + flavor = flavors[flavor_type] + + info['name'] = info['display_name'] + del(info['display_name']) + + info['memory_mb'] = flavor['memory_mb'] + info['local_gb'] = flavor['local_gb'] + info['vcpus'] = flavor['vcpus'] + + info['tenant_id'] = info['project_id'] + del(info['project_id']) + + info['flavor'] = flavor['name'] + del(info['instance_type_id']) + + info['started_at'] = info['launched_at'] + del(info['launched_at']) + + info['ended_at'] = info['terminated_at'] + del(info['terminated_at']) + + if info['ended_at']: + info['state'] = 'terminated' + else: + info['state'] = info['state_description'] + + del(info['state_description']) + + now = datetime.utcnow() + + if info['state'] == 'terminated': + delta = self._parse_datetime(info['ended_at'])\ + - self._parse_datetime(info['started_at']) + else: + delta = now - self._parse_datetime(info['started_at']) + + info['uptime'] = delta.days * 24 * 60 + delta.seconds + + if not info['tenant_id'] in rval: + summary = {} + summary['tenant_id'] = info['tenant_id'] + if detailed: + summary['server_usages'] = [] + summary['total_local_gb_usage'] = 0 + summary['total_vcpus_usage'] = 0 + summary['total_memory_mb_usage'] = 0 + summary['total_hours'] = 0 + summary['start'] = period_start + summary['stop'] = period_stop + rval[info['tenant_id']] = summary + + summary = rval[info['tenant_id']] + summary['total_local_gb_usage'] += info['local_gb'] * info['hours'] + summary['total_vcpus_usage'] += info['vcpus'] * info['hours'] + summary['total_memory_mb_usage'] += info['memory_mb']\ + * info['hours'] + + summary['total_hours'] += info['hours'] + if detailed: + summary['server_usages'].append(info) + + return rval.values() + + def _parse_datetime(self, dtstr): + if isinstance(dtstr, datetime): + return dtstr + try: + return datetime.strptime(dtstr, "%Y-%m-%dT%H:%M:%S") + except: + try: + return datetime.strptime(dtstr, "%Y-%m-%dT%H:%M:%S.%f") + except: + return datetime.strptime(dtstr, "%Y-%m-%d %H:%M:%S.%f") + + def _get_datetime_range(self, req): + qs = req.environ.get('QUERY_STRING', '') + env = urlparse.parse_qs(qs) + period_start = self._parse_datetime(env.get('start', + [datetime.utcnow().isoformat()])[0]) + period_stop = self._parse_datetime(env.get('end', + [datetime.utcnow().isoformat()])[0]) + + detailed = bool(env.get('detailed', False)) + return (period_start, period_stop, detailed) + + def index(self, req): + """Retrive tenant_usage for all tenants""" + (period_start, period_stop, detailed) = self._get_datetime_range(req) + context = req.environ['nova.context'] + + if not context.is_admin and FLAGS.allow_admin_api: + return webob.Response(status_int=403) + + usages = self._usage_for_period(context, + period_start, + period_stop, + detailed=detailed) + return {'tenant_usages': usages} + + def show(self, req, id): + """Retrive tenant_usage for a specified tenant""" + (period_start, period_stop, ignore) = self._get_datetime_range(req) + context = req.environ['nova.context'] + + if not context.is_admin and FLAGS.allow_admin_api: + if id != context.project_id: + return webob.Response(status_int=403) + + usage = self._usage_for_period(context, + period_start, + period_stop, + id, + detailed=True) + if len(usage): + usage = usage[0] + else: + usage = {} + return {'tenant_usage': usage} + + +class Simple_tenant_usage(extensions.ExtensionDescriptor): + + def get_name(self): + return "Simple_tenant_usage" + + def get_alias(self): + return "os-simple-tenant-usage" + + def get_description(self): + return "Simple tenant usage extension" + + def get_namespace(self): + return "http://docs.openstack.org/ext/os-simple-tenant-usage/api/v1.1" + + def get_updated(self): + return "2011-08-19T00:00:00+00:00" + + def get_resources(self): + resources = [] + + res = extensions.ResourceExtension('os-simple-tenant-usage', + SimpleTenantUsageController()) + resources.append(res) + + return resources -- cgit From 4b0944731a25d3cfcd30358619376dedf2251701 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 22 Aug 2011 14:31:26 -0700 Subject: some readability fixes per ja feedback --- nova/api/openstack/contrib/simple_tenant_usage.py | 29 ++++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py index d578b2b67..80d819365 100644 --- a/nova/api/openstack/contrib/simple_tenant_usage.py +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -97,8 +97,8 @@ class SimpleTenantUsageController(object): # instance hasn't launched, so no charge return 0 - def _usage_for_period(self, context, period_start, - period_stop, tenant_id=None, detailed=True): + def _tenant_usages_for_period(self, context, period_start, + period_stop, tenant_id=None, detailed=True): rows = self._get_instances_for_time_period(period_start, period_stop, @@ -208,32 +208,33 @@ class SimpleTenantUsageController(object): def index(self, req): """Retrive tenant_usage for all tenants""" - (period_start, period_stop, detailed) = self._get_datetime_range(req) context = req.environ['nova.context'] if not context.is_admin and FLAGS.allow_admin_api: return webob.Response(status_int=403) - usages = self._usage_for_period(context, - period_start, - period_stop, - detailed=detailed) + (period_start, period_stop, detailed) = self._get_datetime_range(req) + usages = self._tenant_usages_for_period(context, + period_start, + period_stop, + detailed=detailed) return {'tenant_usages': usages} def show(self, req, id): """Retrive tenant_usage for a specified tenant""" - (period_start, period_stop, ignore) = self._get_datetime_range(req) + tenant_id = id context = req.environ['nova.context'] if not context.is_admin and FLAGS.allow_admin_api: - if id != context.project_id: + if tenant_id != context.project_id: return webob.Response(status_int=403) - usage = self._usage_for_period(context, - period_start, - period_stop, - id, - detailed=True) + (period_start, period_stop, ignore) = self._get_datetime_range(req) + usage = self._tenant_usages_for_period(context, + period_start, + period_stop, + tenant_id=tenant_id, + detailed=True) if len(usage): usage = usage[0] else: -- cgit From de0a17310e7228aa96263243851a89fb016f9730 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 22 Aug 2011 15:21:31 -0700 Subject: remove extra spaces --- nova/api/openstack/contrib/simple_tenant_usage.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py index 80d819365..5f4218237 100644 --- a/nova/api/openstack/contrib/simple_tenant_usage.py +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -43,7 +43,6 @@ INSTANCE_FIELDS = ['id', class SimpleTenantUsageController(object): - def _get_instances_for_time_period(self, period_start, period_stop, tenant_id): tenant_clause = '' @@ -243,7 +242,6 @@ class SimpleTenantUsageController(object): class Simple_tenant_usage(extensions.ExtensionDescriptor): - def get_name(self): return "Simple_tenant_usage" -- cgit From 400427ab786779109d49b27eda2fe9e246503dd6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 26 Aug 2011 16:17:40 -0700 Subject: use db layer for aggregation --- nova/api/openstack/contrib/simple_tenant_usage.py | 72 +++++++---------------- 1 file changed, 20 insertions(+), 52 deletions(-) (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py index 5f4218237..16e712815 100644 --- a/nova/api/openstack/contrib/simple_tenant_usage.py +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -31,35 +31,7 @@ from webob import exc FLAGS = flags.FLAGS -INSTANCE_FIELDS = ['id', - 'image_ref', - 'project_id', - 'user_id', - 'display_name', - 'state_description', - 'instance_type_id', - 'launched_at', - 'terminated_at'] - - class SimpleTenantUsageController(object): - def _get_instances_for_time_period(self, period_start, period_stop, - tenant_id): - tenant_clause = '' - if tenant_id: - tenant_clause = " and project_id='%s'" % tenant_id - - conn = get_session().connection() - rows = conn.execute("select %s from instances where \ - (terminated_at is NULL or terminated_at > '%s') \ - and (launched_at < '%s') %s" %\ - (','.join(INSTANCE_FIELDS), - period_start.isoformat(' '),\ - period_stop.isoformat(' '), - tenant_clause)).fetchall() - - return rows - def _hours_for(self, instance, period_start, period_stop): launched_at = instance['launched_at'] terminated_at = instance['terminated_at'] @@ -99,62 +71,58 @@ class SimpleTenantUsageController(object): def _tenant_usages_for_period(self, context, period_start, period_stop, tenant_id=None, detailed=True): - rows = self._get_instances_for_time_period(period_start, - period_stop, - tenant_id) + instances = db.instance_get_active_by_window(context, + period_start, + period_stop, + tenant_id, + fast=True) + from nova import log as logging + logging.info(instances) rval = {} flavors = {} - for row in rows: + for instance in instances: info = {} - for i in range(len(INSTANCE_FIELDS)): - info[INSTANCE_FIELDS[i]] = row[i] - info['hours'] = self._hours_for(info, period_start, period_stop) - flavor_type = info['instance_type_id'] + info['hours'] = self._hours_for(instance, + period_start, + period_stop) + flavor_type = instance['instance_type_id'] if not flavors.get(flavor_type): try: flavors[flavor_type] = db.instance_type_get(context, - info['instance_type_id']) + flavor_type) except exception.InstanceTypeNotFound: # can't bill if there is no instance type continue flavor = flavors[flavor_type] - info['name'] = info['display_name'] - del(info['display_name']) + info['name'] = instance['display_name'] info['memory_mb'] = flavor['memory_mb'] info['local_gb'] = flavor['local_gb'] info['vcpus'] = flavor['vcpus'] - info['tenant_id'] = info['project_id'] - del(info['project_id']) + info['tenant_id'] = instance['project_id'] info['flavor'] = flavor['name'] - del(info['instance_type_id']) - info['started_at'] = info['launched_at'] - del(info['launched_at']) + info['started_at'] = instance['launched_at'] - info['ended_at'] = info['terminated_at'] - del(info['terminated_at']) + info['ended_at'] = instance['terminated_at'] if info['ended_at']: info['state'] = 'terminated' else: - info['state'] = info['state_description'] - - del(info['state_description']) + info['state'] = instance['state_description'] now = datetime.utcnow() if info['state'] == 'terminated': - delta = self._parse_datetime(info['ended_at'])\ - - self._parse_datetime(info['started_at']) + delta = info['ended_at'] - info['started_at'] else: - delta = now - self._parse_datetime(info['started_at']) + delta = now - info['started_at'] info['uptime'] = delta.days * 24 * 60 + delta.seconds -- cgit From dcf5970dd9bed27201c593d7d053970a632e5eee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 30 Aug 2011 12:01:18 -0700 Subject: make two functions instead of fast flag and add compute api commands instead of hitting db directly --- nova/api/openstack/contrib/simple_tenant_usage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py index 16e712815..363ac1451 100644 --- a/nova/api/openstack/contrib/simple_tenant_usage.py +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -19,10 +19,9 @@ import urlparse import webob from datetime import datetime -from nova import db from nova import exception from nova import flags -from nova.compute import instance_types +from nova.compute import api from nova.api.openstack import extensions from nova.api.openstack import views from nova.db.sqlalchemy.session import get_session @@ -71,11 +70,11 @@ class SimpleTenantUsageController(object): def _tenant_usages_for_period(self, context, period_start, period_stop, tenant_id=None, detailed=True): - instances = db.instance_get_active_by_window(context, + compute_api = api.API() + instances = compute_api.get_active_by_window(context, period_start, period_stop, - tenant_id, - fast=True) + tenant_id) from nova import log as logging logging.info(instances) rval = {} @@ -90,8 +89,9 @@ class SimpleTenantUsageController(object): if not flavors.get(flavor_type): try: - flavors[flavor_type] = db.instance_type_get(context, - flavor_type) + it_ref = compute_api.get_instance_type(context, + flavor_type) + flavors[flavor_type] = it_ref except exception.InstanceTypeNotFound: # can't bill if there is no instance type continue -- cgit From 2fcc6da8ba528c5169f7394d57f90ccd2754a23c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 30 Aug 2011 12:14:25 -0700 Subject: pep8, fix fakes --- nova/api/openstack/contrib/simple_tenant_usage.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py index 363ac1451..e81aef66e 100644 --- a/nova/api/openstack/contrib/simple_tenant_usage.py +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -30,6 +30,7 @@ from webob import exc FLAGS = flags.FLAGS + class SimpleTenantUsageController(object): def _hours_for(self, instance, period_start, period_stop): launched_at = instance['launched_at'] -- cgit From 5cf27b5a338f7821f82c91df5889159b56fa0bb6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 30 Aug 2011 12:41:30 -0700 Subject: fix remaining tests --- nova/api/openstack/contrib/simple_tenant_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/api') diff --git a/nova/api/openstack/contrib/simple_tenant_usage.py b/nova/api/openstack/contrib/simple_tenant_usage.py index e81aef66e..69b38e229 100644 --- a/nova/api/openstack/contrib/simple_tenant_usage.py +++ b/nova/api/openstack/contrib/simple_tenant_usage.py @@ -212,7 +212,7 @@ class SimpleTenantUsageController(object): class Simple_tenant_usage(extensions.ExtensionDescriptor): def get_name(self): - return "Simple_tenant_usage" + return "SimpleTenantUsage" def get_alias(self): return "os-simple-tenant-usage" -- cgit