From c94ec9a5bab6c07b402b68e2f4ff081247a27cda Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 14:17:58 -0700 Subject: Initial implementation of refresh instance states --- nova/compute/driver.py | 38 ++++++++++++++++++++++++++++++ nova/compute/manager.py | 56 ++++++++++++++++++++++++++++++++++++++++++++- nova/compute/power_state.py | 16 +++++++++---- 3 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 nova/compute/driver.py (limited to 'nova/compute') diff --git a/nova/compute/driver.py b/nova/compute/driver.py new file mode 100644 index 000000000..bda82c60a --- /dev/null +++ b/nova/compute/driver.py @@ -0,0 +1,38 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Driver base-classes: + + (Beginning of) the contract that compute drivers must follow, and shared + types that support that contract +""" + +from nova.compute import power_state + + +class InstanceInfo(object): + def __init__(self, name, state): + self.name = name + assert state in power_state.valid_states() + self.state = state + + +class ComputeDriver(object): + def list_instances_detail(self): + """Return a list of InstanceInfo for all registered VMs""" + raise NotImplementedError() diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 0cab10fc3..057371d40 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -51,6 +52,7 @@ from nova import manager from nova import rpc from nova import utils from nova.compute import power_state +from nova.compute import driver FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', @@ -115,7 +117,9 @@ class ComputeManager(manager.Manager): # and redocument the module docstring if not compute_driver: compute_driver = FLAGS.compute_driver - self.driver = utils.import_object(compute_driver) + self.driver = utils.check_instance(utils.import_object( + compute_driver), + driver.ComputeDriver) self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(*args, **kwargs) @@ -974,3 +978,53 @@ class ComputeManager(manager.Manager): for volume in instance_ref['volumes']: self.db.volume_update(ctxt, volume['id'], {'status': 'in-use'}) + + def periodic_tasks(self, context=None): + """Tasks to be run at a periodic interval.""" + super(ComputeManager, self).periodic_tasks(context) + try: + self._poll_instance_states(context) + except Exception as ex: + LOG.warning(_("Error during instance poll: %s"), + unicode(ex)) + + def _poll_instance_states(self, context): + vm_instances = self.driver.list_instances_detail(context) + vm_instances = dict((vm.name, vm) for vm in vm_instances) + + # Keep a list of VMs not in the DB, cross them off as we find them + vms_not_found_in_db = [vm.name for vm in vm_instances] + + db_instances = self.db.instance_get_all_by_host(context, self.host) + for db_instance in db_instances: + name = db_instance['name'] + vm_instance = vm_instances.get(name) + if vm_instance is None: + LOG.info(_("Found instance '%(name)' in DB but no VM. " + "Shutting off.") % locals()) + vm_state = power_state.SHUTOFF + else: + vm_state = vm_instance.state + vms_not_found_in_db.remove(name) + + db_state = db_instance['state'] + if vm_state != db_state: + LOG.info(_("DB/VM state mismatch. Changing state from " + "%(db_state) to %(vm_state)") % locals()) + self.db.instance_set_state(context, + db_instance['id'], + vm_state) + + if vm_state == power_state.SHUTOFF: + # TODO(soren): This is what the compute manager does when you + # terminate an instance. At some point I figure we'll have a + # "terminated" state and some sort of cleanup job that runs + # occasionally, cleaning them out. + self.db.instance_destroy(context, db_instance['id']) + + # Are there VMs not in the DB? + for vm_not_found_in_db in vms_not_found_in_db: + name = vm_not_found_in_db + #TODO(justinsb): What to do here? Adopt it? Shut it down? + LOG.warning(_("Found VM not in DB: %(name). Ignoring") + % locals()) diff --git a/nova/compute/power_state.py b/nova/compute/power_state.py index adfc2dff0..145362f97 100644 --- a/nova/compute/power_state.py +++ b/nova/compute/power_state.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # Copyright (c) 2010 Citrix Systems, Inc. # @@ -19,6 +20,7 @@ """The various power states that a VM can be in.""" +#NOTE(justinsb): These are the virDomainState values from libvirt NOSTATE = 0x00 RUNNING = 0x01 BLOCKED = 0x02 @@ -29,9 +31,8 @@ CRASHED = 0x06 SUSPENDED = 0x07 FAILED = 0x08 - -def name(code): - d = { +#TODO(justinsb): Power state really needs to be a proper class... +_STATE_MAP = { NOSTATE: 'pending', RUNNING: 'running', BLOCKED: 'blocked', @@ -41,4 +42,11 @@ def name(code): CRASHED: 'crashed', SUSPENDED: 'suspended', FAILED: 'failed to spawn'} - return d[code] + + +def name(code): + return _STATE_MAP[code] + + +def valid_states(): + return _STATE_MAP.values() -- cgit From 738653b6b4ac744519a050fe50e7c795a7c63579 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 15:11:14 -0700 Subject: Added test and fixed up code so that it works --- nova/compute/manager.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 057371d40..a7727a239 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -981,26 +981,32 @@ class ComputeManager(manager.Manager): def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval.""" - super(ComputeManager, self).periodic_tasks(context) + error_list = super(ComputeManager, self).periodic_tasks(context) + if error_list is None: + error_list = [] + try: self._poll_instance_states(context) except Exception as ex: LOG.warning(_("Error during instance poll: %s"), unicode(ex)) + error_list.append(ex) + return error_list def _poll_instance_states(self, context): - vm_instances = self.driver.list_instances_detail(context) + vm_instances = self.driver.list_instances_detail() vm_instances = dict((vm.name, vm) for vm in vm_instances) # Keep a list of VMs not in the DB, cross them off as we find them vms_not_found_in_db = [vm.name for vm in vm_instances] db_instances = self.db.instance_get_all_by_host(context, self.host) + for db_instance in db_instances: name = db_instance['name'] vm_instance = vm_instances.get(name) if vm_instance is None: - LOG.info(_("Found instance '%(name)' in DB but no VM. " + LOG.info(_("Found instance '%(name)s' in DB but no VM. " "Shutting off.") % locals()) vm_state = power_state.SHUTOFF else: @@ -1010,7 +1016,7 @@ class ComputeManager(manager.Manager): db_state = db_instance['state'] if vm_state != db_state: LOG.info(_("DB/VM state mismatch. Changing state from " - "%(db_state) to %(vm_state)") % locals()) + "'%(db_state)s' to '%(vm_state)s'") % locals()) self.db.instance_set_state(context, db_instance['id'], vm_state) @@ -1026,5 +1032,5 @@ class ComputeManager(manager.Manager): for vm_not_found_in_db in vms_not_found_in_db: name = vm_not_found_in_db #TODO(justinsb): What to do here? Adopt it? Shut it down? - LOG.warning(_("Found VM not in DB: %(name). Ignoring") + LOG.warning(_("Found VM not in DB: '%(name)s'. Ignoring") % locals()) -- cgit From 9dce9ee5fe5a1df018b9a606a3ea35b2dbfc987e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 15:37:29 -0700 Subject: Clarified message when a VM is not running but still in DB --- nova/compute/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/compute') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index a7727a239..019bb3c89 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1007,7 +1007,7 @@ class ComputeManager(manager.Manager): vm_instance = vm_instances.get(name) if vm_instance is None: LOG.info(_("Found instance '%(name)s' in DB but no VM. " - "Shutting off.") % locals()) + "Setting state to shutoff.") % locals()) vm_state = power_state.SHUTOFF else: vm_state = vm_instance.state -- cgit From c8e8b44ef27e49b3986659ee0cb6bd77b38430d8 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Mar 2011 22:01:39 -0700 Subject: Forgot this in the rename of check_instance -> check_isinstance --- nova/compute/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/compute') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 019bb3c89..b21f0b836 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -117,7 +117,7 @@ class ComputeManager(manager.Manager): # and redocument the module docstring if not compute_driver: compute_driver = FLAGS.compute_driver - self.driver = utils.check_instance(utils.import_object( + self.driver = utils.check_isinstance(utils.import_object( compute_driver), driver.ComputeDriver) self.network_manager = utils.import_object(FLAGS.network_manager) -- cgit From 19da125805eedbfcfd202abac4a90c57e6c538c4 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Mar 2011 22:38:37 -0700 Subject: Filled out the base-driver contract, so it's not a false-promise --- nova/compute/driver.py | 38 -------------------------------------- nova/compute/manager.py | 3 ++- 2 files changed, 2 insertions(+), 39 deletions(-) delete mode 100644 nova/compute/driver.py (limited to 'nova/compute') diff --git a/nova/compute/driver.py b/nova/compute/driver.py deleted file mode 100644 index bda82c60a..000000000 --- a/nova/compute/driver.py +++ /dev/null @@ -1,38 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Driver base-classes: - - (Beginning of) the contract that compute drivers must follow, and shared - types that support that contract -""" - -from nova.compute import power_state - - -class InstanceInfo(object): - def __init__(self, name, state): - self.name = name - assert state in power_state.valid_states() - self.state = state - - -class ComputeDriver(object): - def list_instances_detail(self): - """Return a list of InstanceInfo for all registered VMs""" - raise NotImplementedError() diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b21f0b836..f37651ea6 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -52,7 +52,7 @@ from nova import manager from nova import rpc from nova import utils from nova.compute import power_state -from nova.compute import driver +from nova.virt import driver FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', @@ -441,6 +441,7 @@ class ComputeManager(manager.Manager): #TODO(mdietz): we may want to split these into separate methods. if migration_ref['source_compute'] == FLAGS.host: + #NOTE(justinsb): Naughty calling of internal method self.driver._start(instance_ref) self.db.migration_update(context, migration_id, {'status': 'reverted'}) -- cgit From a0432e417a13fd9579c40bdd54f0201b06470f45 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Mar 2011 22:42:12 -0700 Subject: Added note about the advantages of using a type vs using a set of global constants --- nova/compute/power_state.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'nova/compute') diff --git a/nova/compute/power_state.py b/nova/compute/power_state.py index 145362f97..d304285b2 100644 --- a/nova/compute/power_state.py +++ b/nova/compute/power_state.py @@ -31,7 +31,9 @@ CRASHED = 0x06 SUSPENDED = 0x07 FAILED = 0x08 -#TODO(justinsb): Power state really needs to be a proper class... +#TODO(justinsb): Power state really needs to be a proper class, +# so that we're not locked into the libvirt status codes and can put mapping +# logic here rather than spread throughout the code _STATE_MAP = { NOSTATE: 'pending', RUNNING: 'running', -- cgit From a4d78e44d7ca35a6cca4454667cab743409fd95a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Mar 2011 22:45:15 -0700 Subject: Added space in between # and TODO in #TODO --- nova/compute/manager.py | 2 +- nova/compute/power_state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index f37651ea6..cfd2b0ac4 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1032,6 +1032,6 @@ class ComputeManager(manager.Manager): # Are there VMs not in the DB? for vm_not_found_in_db in vms_not_found_in_db: name = vm_not_found_in_db - #TODO(justinsb): What to do here? Adopt it? Shut it down? + # TODO(justinsb): What to do here? Adopt it? Shut it down? LOG.warning(_("Found VM not in DB: '%(name)s'. Ignoring") % locals()) diff --git a/nova/compute/power_state.py b/nova/compute/power_state.py index d304285b2..ed50e492e 100644 --- a/nova/compute/power_state.py +++ b/nova/compute/power_state.py @@ -31,7 +31,7 @@ CRASHED = 0x06 SUSPENDED = 0x07 FAILED = 0x08 -#TODO(justinsb): Power state really needs to be a proper class, +# TODO(justinsb): Power state really needs to be a proper class, # so that we're not locked into the libvirt status codes and can put mapping # logic here rather than spread throughout the code _STATE_MAP = { -- cgit From 3c295817f91eb7c76a64d157ff4a938c85075a36 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 18:50:30 -0700 Subject: pep8 fixes, backported some important fixes that didn't make it over from my testing system :-( --- nova/compute/manager.py | 2 +- nova/compute/power_state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d85ead88b..b67b27dd0 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1001,7 +1001,7 @@ class ComputeManager(manager.Manager): vm_instances = dict((vm.name, vm) for vm in vm_instances) # Keep a list of VMs not in the DB, cross them off as we find them - vms_not_found_in_db = [vm.name for vm in vm_instances] + vms_not_found_in_db = list(vm_instances.keys()) db_instances = self.db.instance_get_all_by_host(context, self.host) diff --git a/nova/compute/power_state.py b/nova/compute/power_state.py index ed50e492e..ef013b2ef 100644 --- a/nova/compute/power_state.py +++ b/nova/compute/power_state.py @@ -51,4 +51,4 @@ def name(code): def valid_states(): - return _STATE_MAP.values() + return _STATE_MAP.keys() -- cgit