From 37b4ae369f3a086990d3c1499bf8b741c7226107 Mon Sep 17 00:00:00 2001 From: Wangpan Date: Mon, 8 Apr 2013 17:57:10 +0800 Subject: Add cpuset attr to vcpu conf in libvirt xml Currently the instances can use all of the pcpu of compute node, the host may become slow when vcpus of instances are busy, so we need to pin vcpus to the specific pcpus of host instead of all pcpus. Also added a cache of total vcpus num in libvirt driver here. DocImpact: please refer to the bp page. Implements blueprint: instance-vcpu-pin Change-Id: I6b6e805c15b249cc5e7517e26619d79bb0183de8 --- nova/tests/test_libvirt.py | 99 +++++++++++++++++++++++++++++++++++++++ nova/tests/test_libvirt_config.py | 12 +++-- nova/virt/libvirt/config.py | 8 +++- nova/virt/libvirt/driver.py | 75 +++++++++++++++++++++++++++-- 4 files changed, 186 insertions(+), 8 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index a955d2f38..437c4c202 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -5097,6 +5097,105 @@ class LibvirtDriverTestCase(test.TestCase): self.libvirtconnection.get_instance_disk_info, instance_name) + def test_get_cpuset_ids(self): + # correct syntax + self.flags(vcpu_pin_set="1") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1], cpuset_ids) + + self.flags(vcpu_pin_set="1,2") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1, 2], cpuset_ids) + + self.flags(vcpu_pin_set=", , 1 , ,, 2, ,") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1, 2], cpuset_ids) + + self.flags(vcpu_pin_set="1-1") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1], cpuset_ids) + + self.flags(vcpu_pin_set=" 1 - 1, 1 - 2 , 1 -3") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1, 2, 3], cpuset_ids) + + self.flags(vcpu_pin_set="1,^2") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1], cpuset_ids) + + self.flags(vcpu_pin_set="1-2, ^1") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([2], cpuset_ids) + + self.flags(vcpu_pin_set="1-3,5,^2") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1, 3, 5], cpuset_ids) + + self.flags(vcpu_pin_set=" 1 - 3 , ^2, 5") + cpuset_ids = self.libvirtconnection._get_cpuset_ids() + self.assertEqual([1, 3, 5], cpuset_ids) + + # invalid syntax + self.flags(vcpu_pin_set=" -1-3,5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-3-,5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="-3,5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-,5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-3,5,^2^") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-3,5,^2-") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="--13,^^5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="a-3,5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-a,5,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-3,b,^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="1-3,5,^c") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="3 - 1, 5 , ^ 2 ") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set=" 1,1, ^1") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set=" 1,^1,^1,2, ^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + + self.flags(vcpu_pin_set="^2") + self.assertRaises(exception.Invalid, + self.libvirtconnection._get_cpuset_ids) + class LibvirtVolumeUsageTestCase(test.TestCase): """Test for nova.virt.libvirt.libvirt_driver.LibvirtDriver diff --git a/nova/tests/test_libvirt_config.py b/nova/tests/test_libvirt_config.py index f98e6dd51..8eed7136e 100644 --- a/nova/tests/test_libvirt_config.py +++ b/nova/tests/test_libvirt_config.py @@ -699,6 +699,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): obj.virt_type = "lxc" obj.memory = 1024 * 1024 * 100 obj.vcpus = 2 + obj.cpuset = "0-3,^2,4-5" obj.name = "demo" obj.uuid = "b38a3f43-4be2-4046-897f-b67c2f5e0147" obj.os_type = "exe" @@ -716,7 +717,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): b38a3f43-4be2-4046-897f-b67c2f5e0147 demo 104857600 - 2 + 2 exe /sbin/init @@ -734,6 +735,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): obj.virt_type = "xen" obj.memory = 1024 * 1024 * 100 obj.vcpus = 2 + obj.cpuset = "0-3,^2,4-5" obj.name = "demo" obj.uuid = "b38a3f43-4be2-4046-897f-b67c2f5e0147" obj.os_type = "linux" @@ -756,7 +758,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): b38a3f43-4be2-4046-897f-b67c2f5e0147 demo 104857600 - 2 + 2 linux /tmp/vmlinuz @@ -777,6 +779,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): obj.virt_type = "xen" obj.memory = 1024 * 1024 * 100 obj.vcpus = 2 + obj.cpuset = "0-3,^2,4-5" obj.name = "demo" obj.uuid = "b38a3f43-4be2-4046-897f-b67c2f5e0147" obj.os_type = "hvm" @@ -800,7 +803,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): b38a3f43-4be2-4046-897f-b67c2f5e0147 demo 104857600 - 2 + 2 hvm /usr/lib/xen/boot/hvmloader @@ -824,6 +827,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): obj.virt_type = "kvm" obj.memory = 1024 * 1024 * 100 obj.vcpus = 2 + obj.cpuset = "0-3,^2,4-5" obj.cpu_shares = 100 obj.cpu_quota = 50000 obj.cpu_period = 25000 @@ -853,7 +857,7 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): b38a3f43-4be2-4046-897f-b67c2f5e0147 demo 104857600 - 2 + 2 Acme diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 729d27ea4..08a0566dd 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -789,6 +789,7 @@ class LibvirtConfigGuest(LibvirtConfigObject): self.name = None self.memory = 1024 * 1024 * 500 self.vcpus = 1 + self.cpuset = None self.cpu = None self.cpu_shares = None self.cpu_quota = None @@ -812,7 +813,12 @@ class LibvirtConfigGuest(LibvirtConfigObject): root.append(self._text_node("uuid", self.uuid)) root.append(self._text_node("name", self.name)) root.append(self._text_node("memory", self.memory)) - root.append(self._text_node("vcpu", self.vcpus)) + if self.cpuset is not None: + vcpu = self._text_node("vcpu", self.vcpus) + vcpu.set("cpuset", self.cpuset) + root.append(vcpu) + else: + root.append(self._text_node("vcpu", self.vcpus)) def _format_os(self, root): os = etree.Element("os") diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index a0d2d0a7b..03a02c806 100755 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -211,6 +211,10 @@ libvirt_opts = [ default=[], help='Specific cachemodes to use for different disk types ' 'e.g: ["file=directsync","block=none"]'), + cfg.StrOpt('vcpu_pin_set', + default=None, + help='Which pcpus can be used by vcpus of instance ' + 'e.g: "4-12,^8,15"'), ] CONF = cfg.CONF @@ -305,6 +309,7 @@ class LibvirtDriver(driver.ComputeDriver): self._fc_wwpns = None self._wrapped_conn = None self._caps = None + self._vcpu_total = 0 self.read_only = read_only self.firewall_driver = firewall.load_driver( DEFAULT_FIREWALL_DRIVER, @@ -2145,6 +2150,7 @@ class LibvirtDriver(driver.ComputeDriver): guest.uuid = instance['uuid'] guest.memory = inst_type['memory_mb'] * 1024 guest.vcpus = inst_type['vcpus'] + guest.cpuset = CONF.vcpu_pin_set quota_items = ['cpu_shares', 'cpu_period', 'cpu_quota'] for key, value in inst_type['extra_specs'].iteritems(): @@ -2517,20 +2523,83 @@ class LibvirtDriver(driver.ComputeDriver): return interfaces + def _get_cpuset_ids(self): + """ + Parsing vcpu_pin_set config. + + Returns a list of pcpu ids can be used by instances. + """ + cpuset_ids = set() + cpuset_reject_ids = set() + for rule in CONF.vcpu_pin_set.split(','): + rule = rule.strip() + # Handle multi ',' + if len(rule) < 1: + continue + # Note the count limit in the .split() call + range_parts = rule.split('-', 1) + if len(range_parts) > 1: + # So, this was a range; start by converting the parts to ints + try: + start, end = [int(p.strip()) for p in range_parts] + except ValueError: + raise exception.Invalid(_("Invalid range expression %r") + % rule) + # Make sure it's a valid range + if start > end: + raise exception.Invalid(_("Invalid range expression %r") + % rule) + # Add available pcpu ids to set + cpuset_ids |= set(range(start, end + 1)) + elif rule[0] == '^': + # Not a range, the rule is an exclusion rule; convert to int + try: + cpuset_reject_ids.add(int(rule[1:].strip())) + except ValueError: + raise exception.Invalid(_("Invalid exclusion " + "expression %r") % rule) + else: + # OK, a single PCPU to include; convert to int + try: + cpuset_ids.add(int(rule)) + except ValueError: + raise exception.Invalid(_("Invalid inclusion " + "expression %r") % rule) + # Use sets to handle the exclusion rules for us + cpuset_ids -= cpuset_reject_ids + if not cpuset_ids: + raise exception.Invalid(_("No CPUs available after parsing %r") % + CONF.vcpu_pin_set) + # This will convert the set to a sorted list for us + return sorted(cpuset_ids) + def get_vcpu_total(self): - """Get vcpu number of physical computer. + """Get available vcpu number of physical computer. - :returns: the number of cpu core. + :returns: the number of cpu core instances can be used. """ + if self._vcpu_total != 0: + return self._vcpu_total try: - return self._conn.getInfo()[2] + total_pcpus = self._conn.getInfo()[2] except libvirt.libvirtError: LOG.warn(_("Cannot get the number of cpu, because this " "function is not implemented for this platform. ")) return 0 + if CONF.vcpu_pin_set is None: + self._vcpu_total = total_pcpus + return self._vcpu_total + + available_ids = self._get_cpuset_ids() + if available_ids[-1] >= total_pcpus: + raise exception.Invalid(_("Invalid vcpu_pin_set config, " + "out of hypervisor cpu range.")) + self._vcpu_total = len(available_ids) + return self._vcpu_total + def get_memory_mb_total(self): """Get the total memory size(MB) of physical computer. -- cgit