From aad7743f42e559aefde37bb44622986df4e2ae98 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Fri, 27 Jul 2012 19:51:32 +0000 Subject: Correct host count in instance_usage_audit_log extension. This fixes bug 1030106. Basically we were not counting disabled hosts in the total host count. Also I have refactored the get_audit_task_logs method, it's really part of the extension (only the extension uses it), so I've moved it out of nova.compute.utils into the extension itself. (the tests that exercise it already lived in the extension's tests anyway.) Change-Id: Iaf42d887b824ba0cbf6ab0ed143a4c01b37b0cb1 --- .../compute/contrib/instance_usage_audit_log.py | 69 +++++++++++++++++++++- nova/compute/utils.py | 58 ------------------ .../contrib/test_instance_usage_audit_log.py | 15 +++-- 3 files changed, 75 insertions(+), 67 deletions(-) diff --git a/nova/api/openstack/compute/contrib/instance_usage_audit_log.py b/nova/api/openstack/compute/contrib/instance_usage_audit_log.py index 4a427d2a7..3c3588860 100644 --- a/nova/api/openstack/compute/contrib/instance_usage_audit_log.py +++ b/nova/api/openstack/compute/contrib/instance_usage_audit_log.py @@ -20,10 +20,11 @@ import datetime from webob import exc from nova.api.openstack import extensions -from nova.compute import utils as compute_utils from nova import context as nova_context +from nova import db from nova import exception from nova import flags +from nova import utils FLAGS = flags.FLAGS @@ -37,7 +38,7 @@ class InstanceUsageAuditLogController(object): def index(self, req): context = req.environ['nova.context'] authorize(context) - task_log = compute_utils.get_audit_task_logs(context) + task_log = self._get_audit_task_logs(context) return {'instance_usage_audit_logs': task_log} def show(self, req, id): @@ -53,10 +54,72 @@ class InstanceUsageAuditLogController(object): except ValueError: msg = _("Invalid timestamp for date %s") % id raise webob.exc.HTTPBadRequest(explanation=msg) - task_log = compute_utils.get_audit_task_logs(context, + task_log = self._get_audit_task_logs(context, before=before_date) return {'instance_usage_audit_log': task_log} + def _get_audit_task_logs(self, context, begin=None, end=None, + before=None): + """Returns a full log for all instance usage audit tasks on all + computes. + + :param begin: datetime beginning of audit period to get logs for, + Defaults to the beginning of the most recently completed + audit period prior to the 'before' date. + :param end: datetime ending of audit period to get logs for, + Defaults to the ending of the most recently completed + audit period prior to the 'before' date. + :param before: By default we look for the audit period most recently + completed before this datetime. Has no effect if both begin and end + are specified. + """ + defbegin, defend = utils.last_completed_audit_period(before=before) + if begin is None: + begin = defbegin + if end is None: + end = defend + task_logs = db.task_log_get_all(context, "instance_usage_audit", + begin, end) + # We do this this way to include disabled compute services, + # which can have instances on them. (mdragon) + services = [svc for svc in db.service_get_all(context) + if svc['topic'] == 'compute'] + hosts = set(serv['host'] for serv in services) + seen_hosts = set() + done_hosts = set() + running_hosts = set() + total_errors = 0 + total_items = 0 + for tlog in task_logs: + seen_hosts.add(tlog['host']) + if tlog['state'] == "DONE": + done_hosts.add(tlog['host']) + if tlog['state'] == "RUNNING": + running_hosts.add(tlog['host']) + total_errors += tlog['errors'] + total_items += tlog['task_items'] + log = dict((tl['host'], dict(state=tl['state'], + instances=tl['task_items'], + errors=tl['errors'], + message=tl['message'])) + for tl in task_logs) + missing_hosts = hosts - seen_hosts + overall_status = "%s hosts done. %s errors." % ( + 'ALL' if len(done_hosts) == len(hosts) + else "%s of %s" % (len(done_hosts), len(hosts)), + total_errors) + return dict(period_beginning=str(begin), + period_ending=str(end), + num_hosts=len(hosts), + num_hosts_done=len(done_hosts), + num_hosts_running=len(running_hosts), + num_hosts_not_run=len(missing_hosts), + hosts_not_run=list(missing_hosts), + total_instances=total_items, + total_errors=total_errors, + overall_status=overall_status, + log=log) + class Instance_usage_audit_log(extensions.ExtensionDescriptor): """Admin-only Task Log Monitoring""" diff --git a/nova/compute/utils.py b/nova/compute/utils.py index 0370988ea..d3beea68c 100644 --- a/nova/compute/utils.py +++ b/nova/compute/utils.py @@ -128,61 +128,3 @@ def start_instance_usage_audit(context, begin, end, host, num_instances): def finish_instance_usage_audit(context, begin, end, host, errors, message): db.task_log_end_task(context, "instance_usage_audit", begin, end, host, errors, message) - - -def get_audit_task_logs(context, begin=None, end=None, before=None): - """Returns a full log for all instance usage audit tasks on all computes. - - :param begin: datetime beginning of audit period to get logs for, - Defaults to the beginning of the most recently completed - audit period prior to the 'before' date. - :param end: datetime ending of audit period to get logs for, - Defaults to the ending of the most recently completed - audit period prior to the 'before' date. - :param before: By default we look for the audit period most recently - completed before this datetime. Has no effect if both begin and end - are specified. - """ - defbegin, defend = utils.last_completed_audit_period(before=before) - if begin is None: - begin = defbegin - if end is None: - end = defend - task_logs = db.task_log_get_all(context, "instance_usage_audit", - begin, end) - services = db.service_get_all_by_topic(context, "compute") - hosts = set(serv['host'] for serv in services) - seen_hosts = set() - done_hosts = set() - running_hosts = set() - total_errors = 0 - total_items = 0 - for tlog in task_logs: - seen_hosts.add(tlog['host']) - if tlog['state'] == "DONE": - done_hosts.add(tlog['host']) - if tlog['state'] == "RUNNING": - running_hosts.add(tlog['host']) - total_errors += tlog['errors'] - total_items += tlog['task_items'] - log = dict((tl['host'], dict(state=tl['state'], - instances=tl['task_items'], - errors=tl['errors'], - message=tl['message'])) - for tl in task_logs) - missing_hosts = hosts - seen_hosts - overall_status = "%s hosts done. %s errors." % ( - 'ALL' if len(done_hosts) == len(hosts) - else "%s of %s" % (len(done_hosts), len(hosts)), - total_errors) - return dict(period_beginning=str(begin), - period_ending=str(end), - num_hosts=len(hosts), - num_hosts_done=len(done_hosts), - num_hosts_running=len(running_hosts), - num_hosts_not_run=len(missing_hosts), - hosts_not_run=list(missing_hosts), - total_instances=total_items, - total_errors=total_errors, - overall_status=overall_status, - log=log) diff --git a/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py b/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py index b81052ddc..2e06897c7 100644 --- a/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py +++ b/nova/tests/api/openstack/compute/contrib/test_instance_usage_audit_log.py @@ -27,8 +27,12 @@ from nova.tests.api.openstack import fakes from nova import utils -TEST_COMPUTE_SERVICES = [dict(host=name) for name in - "foo bar baz plonk".split()] +TEST_COMPUTE_SERVICES = [dict(host='foo', topic='compute'), + dict(host='bar', topic='compute'), + dict(host='baz', topic='compute'), + dict(host='plonk', topic='compute'), + dict(host='wibble', topic='bogus'), + ] begin1 = datetime.datetime(2012, 7, 4, 6, 0, 0) @@ -79,8 +83,7 @@ TEST_LOGS3 = [ ] -def fake_service_get_all_by_topic(context, topic): - assert topic == "compute" +def fake_service_get_all(context): return TEST_COMPUTE_SERVICES @@ -117,8 +120,8 @@ class InstanceUsageAuditLogTest(test.TestCase): self.stubs.Set(utils, 'last_completed_audit_period', fake_last_completed_audit_period) - self.stubs.Set(db, 'service_get_all_by_topic', - fake_service_get_all_by_topic) + self.stubs.Set(db, 'service_get_all', + fake_service_get_all) self.stubs.Set(db, 'task_log_get_all', fake_task_log_get_all) -- cgit