From ad54ed53cf6a475ad0f8042f8b95454a8c0b35a4 Mon Sep 17 00:00:00 2001 From: Leander Bessa Beernaert Date: Fri, 22 Jun 2012 12:08:08 +0100 Subject: Adds diagnostics command for the libvirt driver. This code provides an implementation of the get_diagnostics API for the libvirt driver. This API is invoked by the 'nova diagnostic' command. There is no existing formal specification for what data items to expose from the 'get_diagnostics' API, so the XenAPI driver is used as an informal guide. The Xen implementation currently produces the following output: +----------------+-----------------+ | Property | Value | +----------------+-----------------+ | cpu0 | 4.3627 | | memory | 1171088064.0000 | | memory_target | 1171088064.0000 | | vbd_xvda_read | 0.0 | | vbd_xvda_write | 0.0 | | vif_0_rx | 3223.6870 | | vif_0_tx | 0.0 | | vif_1_rx | 104.4955 | | vif_1_tx | 0.0 | +----------------+-----------------+ The new libvirt implementation will return a similar set of stats for guest CPUs, memory, disks and network interfaces, resulting in the following output: +------------------+------------+ | Property | Value | +------------------+------------+ | cpu0_time | 2870000000 | | memory | 524288 | | vda_errors | -1 | | vda_read | 262144 | | vda_read_req | 112 | | vda_write | 5606400 | | vda_write_req | 376 | | vnet0_rx | 63343 | | vnet0_rx_drop | 0 | | vnet0_rx_errors | 0 | | vnet0_rx_packets | 431 | | vnet0_tx | 4905 | | vnet0_tx_drop | 0 | | vnet0_tx_errors | 0 | | vnet0_tx_packets | 45 | +------------------+------------+ The implementation has only been tested with the libvirt KVM driver. In a future a formal specification for the required data format should be written and all virt drivers made to comply. Finally, the fakelibvirt driver has been updated in order to support the methods required by the get_diagnostics method. For those interested, below is the formula to extract the CPU usage based on the CPU time. I've added it here for reference since it's not used at the moment and it took quite a while to locate it in the libvirt documentation. usage = 100 * (current_cpu_time - last_cpu_time) / (measurement_interval * (10 ** 9)) Fixes bug 986200. Change-Id: I92da392f43d0840ed31cfd88f54570ea5830bb4a --- nova/tests/fakelibvirt.py | 13 ++ nova/tests/test_libvirt.py | 390 ++++++++++++++++++++++++++++++++++++++++ nova/tests/test_virt_drivers.py | 2 +- nova/virt/libvirt/driver.py | 75 ++++++++ 4 files changed, 479 insertions(+), 1 deletion(-) diff --git a/nova/tests/fakelibvirt.py b/nova/tests/fakelibvirt.py index ab5d4073c..4e337aa51 100644 --- a/nova/tests/fakelibvirt.py +++ b/nova/tests/fakelibvirt.py @@ -424,6 +424,19 @@ class Domain(object): self._snapshots[name] = snapshot return snapshot + def vcpus(self): + vcpus = ([], []) + for i in range(0, self._def['vcpu']): + vcpus[0].append((i, 1, 120405L, i)) + vcpus[1].append((True, True, True, True)) + return vcpus + + def memoryStats(self): + return {} + + def maxMemory(self): + return self._def['memory'] + class DomainSnapshot(object): def __init__(self, name, domain): diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 86128626a..a506b0ad8 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -1,6 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright 2010 OpenStack LLC +# Copyright 2012 University Of Minho # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -1919,6 +1920,395 @@ class LibvirtConnTestCase(test.TestCase): got = jsonutils.loads(conn.get_cpu_info()) self.assertEqual(want, got) + def test_diagnostic_vcpus_exception(self): + xml = """ + + + + + + + + + + + + + + + + + + """ + + class DiagFakeDomain(FakeVirtDomain): + + def __init__(self): + super(DiagFakeDomain, self).__init__(fake_xml=xml) + + def vcpus(self): + raise libvirt.libvirtError('vcpus missing') + + def blockStats(self, path): + return (169L, 688640L, 0L, 0L, -1L) + + def interfaceStats(self, path): + return (4408L, 82L, 0L, 0L, 0L, 0L, 0L, 0L) + + def memoryStats(self): + return {'actual': 220160L, 'rss': 200164L} + + def maxMemory(self): + return 280160L + + def fake_lookup_name(name): + return DiagFakeDomain() + + self.mox.StubOutWithMock(libvirt_driver.LibvirtDriver, '_conn') + libvirt_driver.LibvirtDriver._conn.lookupByName = fake_lookup_name + + conn = libvirt_driver.LibvirtDriver(False) + actual = conn.get_diagnostics({"name": "testvirt"}) + expect = {'vda_read': 688640L, + 'vda_read_req': 169L, + 'vda_write': 0L, + 'vda_write_req': 0L, + 'vda_errors': -1L, + 'vdb_read': 688640L, + 'vdb_read_req': 169L, + 'vdb_write': 0L, + 'vdb_write_req': 0L, + 'vdb_errors': -1L, + 'memory': 280160L, + 'memory-actual': 220160L, + 'memory-rss': 200164L, + 'vnet0_rx': 4408L, + 'vnet0_rx_drop': 0L, + 'vnet0_rx_errors': 0L, + 'vnet0_rx_packets': 82L, + 'vnet0_tx': 0L, + 'vnet0_tx_drop': 0L, + 'vnet0_tx_errors': 0L, + 'vnet0_tx_packets': 0L, + } + self.assertEqual(actual, expect) + + def test_diagnostic_blockstats_exception(self): + xml = """ + + + + + + + + + + + + + + + + + + """ + + class DiagFakeDomain(FakeVirtDomain): + + def __init__(self): + super(DiagFakeDomain, self).__init__(fake_xml=xml) + + def vcpus(self): + return ([(0, 1, 15340000000L, 0), + (1, 1, 1640000000L, 0), + (2, 1, 3040000000L, 0), + (3, 1, 1420000000L, 0)], + [(True, False), + (True, False), + (True, False), + (True, False)]) + + def blockStats(self, path): + raise libvirt.libvirtError('blockStats missing') + + def interfaceStats(self, path): + return (4408L, 82L, 0L, 0L, 0L, 0L, 0L, 0L) + + def memoryStats(self): + return {'actual': 220160L, 'rss': 200164L} + + def maxMemory(self): + return 280160L + + def fake_lookup_name(name): + return DiagFakeDomain() + + self.mox.StubOutWithMock(libvirt_driver.LibvirtDriver, '_conn') + libvirt_driver.LibvirtDriver._conn.lookupByName = fake_lookup_name + + conn = libvirt_driver.LibvirtDriver(False) + actual = conn.get_diagnostics({"name": "testvirt"}) + expect = {'cpu0_time': 15340000000L, + 'cpu1_time': 1640000000L, + 'cpu2_time': 3040000000L, + 'cpu3_time': 1420000000L, + 'memory': 280160L, + 'memory-actual': 220160L, + 'memory-rss': 200164L, + 'vnet0_rx': 4408L, + 'vnet0_rx_drop': 0L, + 'vnet0_rx_errors': 0L, + 'vnet0_rx_packets': 82L, + 'vnet0_tx': 0L, + 'vnet0_tx_drop': 0L, + 'vnet0_tx_errors': 0L, + 'vnet0_tx_packets': 0L, + } + self.assertEqual(actual, expect) + + def test_diagnostic_interfacestats_exception(self): + xml = """ + + + + + + + + + + + + + + + + + + """ + + class DiagFakeDomain(FakeVirtDomain): + + def __init__(self): + super(DiagFakeDomain, self).__init__(fake_xml=xml) + + def vcpus(self): + return ([(0, 1, 15340000000L, 0), + (1, 1, 1640000000L, 0), + (2, 1, 3040000000L, 0), + (3, 1, 1420000000L, 0)], + [(True, False), + (True, False), + (True, False), + (True, False)]) + + def blockStats(self, path): + return (169L, 688640L, 0L, 0L, -1L) + + def interfaceStats(self, path): + raise libvirt.libvirtError('interfaceStat missing') + + def memoryStats(self): + return {'actual': 220160L, 'rss': 200164L} + + def maxMemory(self): + return 280160L + + def fake_lookup_name(name): + return DiagFakeDomain() + + self.mox.StubOutWithMock(libvirt_driver.LibvirtDriver, '_conn') + libvirt_driver.LibvirtDriver._conn.lookupByName = fake_lookup_name + + conn = libvirt_driver.LibvirtDriver(False) + actual = conn.get_diagnostics({"name": "testvirt"}) + expect = {'cpu0_time': 15340000000L, + 'cpu1_time': 1640000000L, + 'cpu2_time': 3040000000L, + 'cpu3_time': 1420000000L, + 'vda_read': 688640L, + 'vda_read_req': 169L, + 'vda_write': 0L, + 'vda_write_req': 0L, + 'vda_errors': -1L, + 'vdb_read': 688640L, + 'vdb_read_req': 169L, + 'vdb_write': 0L, + 'vdb_write_req': 0L, + 'vdb_errors': -1L, + 'memory': 280160L, + 'memory-actual': 220160L, + 'memory-rss': 200164L, + } + self.assertEqual(actual, expect) + + def test_diagnostic_memorystats_exception(self): + xml = """ + + + + + + + + + + + + + + + + + + """ + + class DiagFakeDomain(FakeVirtDomain): + + def __init__(self): + super(DiagFakeDomain, self).__init__(fake_xml=xml) + + def vcpus(self): + return ([(0, 1, 15340000000L, 0), + (1, 1, 1640000000L, 0), + (2, 1, 3040000000L, 0), + (3, 1, 1420000000L, 0)], + [(True, False), + (True, False), + (True, False), + (True, False)]) + + def blockStats(self, path): + return (169L, 688640L, 0L, 0L, -1L) + + def interfaceStats(self, path): + return (4408L, 82L, 0L, 0L, 0L, 0L, 0L, 0L) + + def memoryStats(self): + raise libvirt.libvirtError('memoryStats missing') + + def maxMemory(self): + return 280160L + + def fake_lookup_name(name): + return DiagFakeDomain() + + self.mox.StubOutWithMock(libvirt_driver.LibvirtDriver, '_conn') + libvirt_driver.LibvirtDriver._conn.lookupByName = fake_lookup_name + + conn = libvirt_driver.LibvirtDriver(False) + actual = conn.get_diagnostics({"name": "testvirt"}) + expect = {'cpu0_time': 15340000000L, + 'cpu1_time': 1640000000L, + 'cpu2_time': 3040000000L, + 'cpu3_time': 1420000000L, + 'vda_read': 688640L, + 'vda_read_req': 169L, + 'vda_write': 0L, + 'vda_write_req': 0L, + 'vda_errors': -1L, + 'vdb_read': 688640L, + 'vdb_read_req': 169L, + 'vdb_write': 0L, + 'vdb_write_req': 0L, + 'vdb_errors': -1L, + 'memory': 280160L, + 'vnet0_rx': 4408L, + 'vnet0_rx_drop': 0L, + 'vnet0_rx_errors': 0L, + 'vnet0_rx_packets': 82L, + 'vnet0_tx': 0L, + 'vnet0_tx_drop': 0L, + 'vnet0_tx_errors': 0L, + 'vnet0_tx_packets': 0L, + } + self.assertEqual(actual, expect) + + def test_diagnostic_full(self): + xml = """ + + + + + + + + + + + + + + + + + + """ + + class DiagFakeDomain(FakeVirtDomain): + + def __init__(self): + super(DiagFakeDomain, self).__init__(fake_xml=xml) + + def vcpus(self): + return ([(0, 1, 15340000000L, 0), + (1, 1, 1640000000L, 0), + (2, 1, 3040000000L, 0), + (3, 1, 1420000000L, 0)], + [(True, False), + (True, False), + (True, False), + (True, False)]) + + def blockStats(self, path): + return (169L, 688640L, 0L, 0L, -1L) + + def interfaceStats(self, path): + return (4408L, 82L, 0L, 0L, 0L, 0L, 0L, 0L) + + def memoryStats(self): + return {'actual': 220160L, 'rss': 200164L} + + def maxMemory(self): + return 280160L + + def fake_lookup_name(name): + return DiagFakeDomain() + + self.mox.StubOutWithMock(libvirt_driver.LibvirtDriver, '_conn') + libvirt_driver.LibvirtDriver._conn.lookupByName = fake_lookup_name + + conn = libvirt_driver.LibvirtDriver(False) + actual = conn.get_diagnostics({"name": "testvirt"}) + expect = {'cpu0_time': 15340000000L, + 'cpu1_time': 1640000000L, + 'cpu2_time': 3040000000L, + 'cpu3_time': 1420000000L, + 'vda_read': 688640L, + 'vda_read_req': 169L, + 'vda_write': 0L, + 'vda_write_req': 0L, + 'vda_errors': -1L, + 'vdb_read': 688640L, + 'vdb_read_req': 169L, + 'vdb_write': 0L, + 'vdb_write_req': 0L, + 'vdb_errors': -1L, + 'memory': 280160L, + 'memory-actual': 220160L, + 'memory-rss': 200164L, + 'vnet0_rx': 4408L, + 'vnet0_rx_drop': 0L, + 'vnet0_rx_errors': 0L, + 'vnet0_rx_packets': 82L, + 'vnet0_tx': 0L, + 'vnet0_tx_drop': 0L, + 'vnet0_tx_errors': 0L, + 'vnet0_tx_packets': 0L, + } + self.assertEqual(actual, expect) + class HostStateTestCase(test.TestCase): diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py index e95d15ae5..bc89a24e8 100644 --- a/nova/tests/test_virt_drivers.py +++ b/nova/tests/test_virt_drivers.py @@ -402,7 +402,7 @@ class _VirtDriverTestCase(_FakeDriverBackendTestCase): @catch_notimplementederror def test_get_diagnostics(self): instance_ref, network_info = self._get_running_instance() - self.connection.get_diagnostics(instance_ref['name']) + self.connection.get_diagnostics(instance_ref) @catch_notimplementederror def test_block_stats(self): diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 14f5620c9..96bcecdf8 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -5,6 +5,7 @@ # All Rights Reserved. # Copyright (c) 2010 Citrix Systems, Inc. # Copyright (c) 2011 Piston Cloud Computing, Inc +# Copyright (c) 2012 University Of Minho # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -2715,6 +2716,80 @@ class LibvirtDriver(driver.ComputeDriver): """Confirms a resize, destroying the source VM""" self._cleanup_resize(instance) + def get_diagnostics(self, instance): + def get_io_devices(xml_doc): + """ get the list of io devices from the + xml document.""" + result = {"volumes": [], "ifaces": []} + try: + doc = etree.fromstring(xml_doc) + except Exception: + return result + blocks = [('./devices/disk', 'volumes'), + ('./devices/interface', 'ifaces')] + for block, key in blocks: + section = doc.findall(block) + for node in section: + for child in node.getchildren(): + if child.tag == 'target' and child.get('dev'): + result[key].append(child.get('dev')) + return result + + domain = self._lookup_by_name(instance['name']) + output = {} + # get cpu time, might launch an exception if the method + # is not supported by the underlying hypervisor being + # used by libvirt + try: + cputime = domain.vcpus()[0] + for i in range(len(cputime)): + output["cpu" + str(i) + "_time"] = cputime[i][2] + except libvirt.libvirtError: + pass + # get io status + xml = domain.XMLDesc(0) + dom_io = get_io_devices(xml) + for disk in dom_io["volumes"]: + try: + # blockStats might launch an exception if the method + # is not supported by the underlying hypervisor being + # used by libvirt + stats = domain.blockStats(disk) + output[disk + "_read_req"] = stats[0] + output[disk + "_read"] = stats[1] + output[disk + "_write_req"] = stats[2] + output[disk + "_write"] = stats[3] + output[disk + "_errors"] = stats[4] + except libvirt.libvirtError: + pass + for interface in dom_io["ifaces"]: + try: + # interfaceStats might launch an exception if the method + # is not supported by the underlying hypervisor being + # used by libvirt + stats = domain.interfaceStats(interface) + output[interface + "_rx"] = stats[0] + output[interface + "_rx_packets"] = stats[1] + output[interface + "_rx_errors"] = stats[2] + output[interface + "_rx_drop"] = stats[3] + output[interface + "_tx"] = stats[4] + output[interface + "_tx_packets"] = stats[5] + output[interface + "_tx_errors"] = stats[6] + output[interface + "_tx_drop"] = stats[7] + except libvirt.libvirtError: + pass + output["memory"] = domain.maxMemory() + # memoryStats might launch an exception if the method + # is not supported by the underlying hypervisor being + # used by libvirt + try: + mem = domain.memoryStats() + for key in mem.keys(): + output["memory-" + key] = mem[key] + except libvirt.libvirtError: + pass + return output + class HostState(object): """Manages information about the compute node through libvirt""" -- cgit