Fix issues with re-raising exceptions.

There is complication with re-raising exceptions and our usage of
eventlet.  If the code in the exception handler accesses the db or rpc
in the exception handler, it will no longer be able to re-raise the
exception.  Using excutils.save_and_reraise_exception() works aorund
this issue.

The most common error is calling LOG.error() or LOG.exception(), as it
is possible for these to go access rpc.  There is an option to turn on
notifications for these errors.

Fix bug 845866.

Change-Id: Icfca2af63805711229249aa7abe60a938edd1b35

9 files changed, 81 insertions, 65 deletions

diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 7e2fb1c0d..bc4171603 100755
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -192,11 +192,11 @@ def reverts_task_state(function):
         try:
             return function(self, context, *args, **kwargs)
         except exception.UnexpectedTaskStateError:
-            LOG.exception(_("Possibly task preempted."))
             # Note(maoy): unexpected task state means the current
             # task is preempted. Do not clear task state in this
             # case.
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_("Possibly task preempted."))
         except Exception:
             with excutils.save_and_reraise_exception():
                 try:
@@ -788,12 +788,13 @@ class ComputeManager(manager.SchedulerDependentManager):
                                            injected_files, admin_password)
             except exception.InstanceNotFound:
                 # the instance got deleted during the spawn
-                try:
-                    self._deallocate_network(context, instance)
-                except Exception:
-                    msg = _('Failed to dealloc network for deleted instance')
-                    LOG.exception(msg, instance=instance)
-                raise
+                with excutils.save_and_reraise_exception():
+                    try:
+                        self._deallocate_network(context, instance)
+                    except Exception:
+                        msg = _('Failed to dealloc network '
+                                'for deleted instance')
+                        LOG.exception(msg, instance=instance)
             except exception.UnexpectedTaskStateError as e:
                 actual_task_state = e.kwargs.get('actual', None)
                 if actual_task_state == 'deleting':
@@ -843,8 +844,8 @@ class ComputeManager(manager.SchedulerDependentManager):
             self._deallocate_network(context, instance)
         except Exception:
             # do not attempt retry if network de-allocation failed:
-            self._log_original_error(exc_info, instance_uuid)
-            raise
+            with excutils.save_and_reraise_exception():
+                self._log_original_error(exc_info, instance_uuid)
 
         try:
             method_args = (request_spec, admin_password, injected_files,
@@ -1032,9 +1033,9 @@ class ComputeManager(manager.SchedulerDependentManager):
                                 conductor_api=self.conductor_api,
                                 security_groups=security_groups)
         except Exception:
-            LOG.exception(_('Instance failed network setup'),
-                          instance=instance)
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_('Instance failed network setup'),
+                              instance=instance)
 
         LOG.debug(_('Instance network_info: |%s|'), network_info,
                   instance=instance)
@@ -1046,9 +1047,9 @@ class ComputeManager(manager.SchedulerDependentManager):
         try:
             return self._setup_block_device_mapping(context, instance, bdms)
         except Exception:
-            LOG.exception(_('Instance failed block device setup'),
-                          instance=instance)
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_('Instance failed block device setup'),
+                              instance=instance)
 
     def _spawn(self, context, instance, image_meta, network_info,
                block_device_info, injected_files, admin_password):
@@ -1063,8 +1064,8 @@ class ComputeManager(manager.SchedulerDependentManager):
                               self._legacy_nw_info(network_info),
                               block_device_info)
         except Exception:
-            LOG.exception(_('Instance failed to spawn'), instance=instance)
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_('Instance failed to spawn'), instance=instance)
 
         current_power_state = self._get_power_state(context, instance)
         return self._instance_update(context, instance['uuid'],
diff --git a/nova/console/xvp.py b/nova/console/xvp.py
index ba3daadba..1e9fbcb5d 100644
--- a/nova/console/xvp.py
+++ b/nova/console/xvp.py
@@ -26,6 +26,7 @@ from oslo.config import cfg
 from nova import context
 from nova import db
 from nova import exception
+from nova.openstack.common import excutils
 from nova.openstack.common import log as logging
 from nova import paths
 from nova import utils
@@ -119,8 +120,8 @@ class XVPConsoleProxy(object):
             with open(CONF.console_xvp_conf, 'w') as cfile:
                 cfile.write(config)
         except IOError:
-            LOG.exception(_("Failed to write configuration file"))
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_("Failed to write configuration file"))
 
     def _xvp_stop(self):
         LOG.debug(_('Stopping xvp'))
diff --git a/nova/crypto.py b/nova/crypto.py
index 4d72aa624..736e99961 100644
--- a/nova/crypto.py
+++ b/nova/crypto.py
@@ -38,6 +38,7 @@ from pyasn1.type import univ
 from nova import context
 from nova import db
 from nova import exception
+from nova.openstack.common import excutils
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import timeutils
@@ -387,8 +388,8 @@ def _sign_csr(csr_text, ca_folder):
             with open(inbound, 'w') as csrfile:
                 csrfile.write(csr_text)
         except IOError:
-            LOG.exception(_('Failed to write inbound.csr'))
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_('Failed to write inbound.csr'))
 
         LOG.debug(_('Flags path: %s'), ca_folder)
         start = os.getcwd()
diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py
index 900190b07..cb23e77b6 100644
--- a/nova/network/linux_net.py
+++ b/nova/network/linux_net.py
@@ -29,6 +29,7 @@ from oslo.config import cfg
 
 from nova import db
 from nova import exception
+from nova.openstack.common import excutils
 from nova.openstack.common import fileutils
 from nova.openstack.common import importutils
 from nova.openstack.common import jsonutils
@@ -1405,9 +1406,9 @@ class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver):
                 utils.execute('ip', 'link', 'delete', vlan_interface,
                               run_as_root=True, check_exit_code=[0, 2, 254])
             except exception.ProcessExecutionError:
-                LOG.error(_("Failed unplugging VLAN interface '%s'"),
-                          vlan_interface)
-                raise
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_("Failed unplugging VLAN interface '%s'"),
+                              vlan_interface)
             LOG.debug(_("Unplugged VLAN interface '%s'"), vlan_interface)
 
     @classmethod
@@ -1511,8 +1512,9 @@ class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver):
                 utils.execute('ip', 'link', 'delete', bridge, run_as_root=True,
                               check_exit_code=[0, 2, 254])
             except exception.ProcessExecutionError:
-                LOG.error(_("Failed unplugging bridge interface '%s'"), bridge)
-                raise
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_("Failed unplugging bridge interface '%s'"),
+                              bridge)
 
         LOG.debug(_("Unplugged bridge interface '%s'"), bridge)
 
@@ -1710,8 +1712,9 @@ class QuantumLinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver):
                 utils.execute('ip', 'link', 'delete', dev, run_as_root=True,
                               check_exit_code=[0, 2, 254])
             except exception.ProcessExecutionError:
-                LOG.error(_("Failed unplugging gateway interface '%s'"), dev)
-                raise
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_("Failed unplugging gateway interface '%s'"),
+                              dev)
             LOG.debug(_("Unplugged gateway interface '%s'"), dev)
             return dev
 
diff --git a/nova/network/security_group/quantum_driver.py b/nova/network/security_group/quantum_driver.py
index d8696afd2..05e019510 100644
--- a/nova/network/security_group/quantum_driver.py
+++ b/nova/network/security_group/quantum_driver.py
@@ -27,6 +27,7 @@ from nova import context
 from nova import exception
 from nova.network import quantumv2
 from nova.network.security_group import security_group_base
+from nova.openstack.common import excutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import uuidutils
 
@@ -332,8 +333,8 @@ class SecurityGroupAPI(security_group_base.SecurityGroupBase):
                           'port_id': port['id']})
                 quantum.update_port(port['id'], {'port': updated_port})
             except Exception:
-                LOG.exception(_("Quantum Error:"))
-                raise
+                with excutils.save_and_reraise_exception():
+                    LOG.exception(_("Quantum Error:"))
 
     @wrap_check_security_groups_policy
     def remove_from_instance(self, context, instance, security_group_name):
diff --git a/nova/notifications.py b/nova/notifications.py
index bed06acf6..16a49dcf6 100644
--- a/nova/notifications.py
+++ b/nova/notifications.py
@@ -26,6 +26,7 @@ from nova import db
 from nova.image import glance
 from nova import network
 from nova.network import model as network_model
+from nova.openstack.common import excutils
 from nova.openstack.common import log
 from nova.openstack.common.notifier import api as notifier_api
 from nova.openstack.common import timeutils
@@ -225,10 +226,14 @@ def bandwidth_usage(instance_ref, audit_start,
             nw_info = network.API().get_instance_nw_info(admin_context,
                     instance_ref)
         except Exception:
-            LOG.exception(_('Failed to get nw_info'), instance=instance_ref)
-            if ignore_missing_network_data:
-                return
-            raise
+            try:
+                with excutils.save_and_reraise_exception():
+                    LOG.exception(_('Failed to get nw_info'),
+                                  instance=instance_ref)
+            except Exception:
+                if ignore_missing_network_data:
+                    return
+                raise
 
     macs = [vif['address'] for vif in nw_info]
     uuids = [instance_ref["uuid"]]
diff --git a/nova/scheduler/scheduler_options.py b/nova/scheduler/scheduler_options.py
index 75b68e557..f7aa91155 100644
--- a/nova/scheduler/scheduler_options.py
+++ b/nova/scheduler/scheduler_options.py
@@ -28,6 +28,7 @@ import os
 
 from oslo.config import cfg
 
+from nova.openstack.common import excutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import timeutils
 
@@ -66,9 +67,9 @@ class SchedulerOptions(object):
         try:
             return os.path.getmtime(filename)
         except os.error, e:
-            LOG.exception(_("Could not stat scheduler options file "
-                            "%(filename)s: '%(e)s'"), locals())
-            raise
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_("Could not stat scheduler options file "
+                                "%(filename)s: '%(e)s'"), locals())
 
     def _load_file(self, handle):
         """Decode the JSON file. Broken out for testing."""
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 6b730cd35..9858e0af2 100755
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -714,10 +714,10 @@ class LibvirtDriver(driver.ComputeDriver):
                         is_okay = True
 
                 if not is_okay:
-                    LOG.error(_("Error from libvirt during destroy. "
-                                  "Code=%(errcode)s Error=%(e)s") %
-                                locals(), instance=instance)
-                    raise
+                    with excutils.save_and_reraise_exception():
+                        LOG.error(_("Error from libvirt during destroy. "
+                                      "Code=%(errcode)s Error=%(e)s") %
+                                    locals(), instance=instance)
 
         def _wait_for_destroy(expected_domid):
             """Called at an interval until the VM is gone."""
@@ -787,11 +787,11 @@ class LibvirtDriver(driver.ComputeDriver):
                         pass
                     virt_dom.undefine()
             except libvirt.libvirtError as e:
-                errcode = e.get_error_code()
-                LOG.error(_("Error from libvirt during undefine. "
-                              "Code=%(errcode)s Error=%(e)s") %
-                            locals(), instance=instance)
-                raise
+                with excutils.save_and_reraise_exception():
+                    errcode = e.get_error_code()
+                    LOG.error(_("Error from libvirt during undefine. "
+                                  "Code=%(errcode)s Error=%(e)s") %
+                                locals(), instance=instance)
 
     def _cleanup(self, instance, network_info, block_device_info,
                  destroy_disks):
@@ -1667,13 +1667,14 @@ class LibvirtDriver(driver.ComputeDriver):
                             "'%(ex)s'") % {'path': dirpath, 'ex': str(e)})
                 hasDirectIO = False
             else:
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_("Error on '%(path)s' while checking "
+                                "direct I/O: '%(ex)s'") %
+                                {'path': dirpath, 'ex': str(e)})
+        except Exception, e:
+            with excutils.save_and_reraise_exception():
                 LOG.error(_("Error on '%(path)s' while checking direct I/O: "
                             "'%(ex)s'") % {'path': dirpath, 'ex': str(e)})
-                raise
-        except Exception, e:
-            LOG.error(_("Error on '%(path)s' while checking direct I/O: "
-                        "'%(ex)s'") % {'path': dirpath, 'ex': str(e)})
-            raise
         finally:
             try:
                 os.unlink(testfile)
@@ -1850,9 +1851,10 @@ class LibvirtDriver(driver.ComputeDriver):
                 try:
                     cdb.make_drive(configdrive_path)
                 except exception.ProcessExecutionError, e:
-                    LOG.error(_('Creating config drive failed with error: %s'),
-                              e, instance=instance)
-                    raise
+                    with excutils.save_and_reraise_exception():
+                        LOG.error(_('Creating config drive failed '
+                                  'with error: %s'),
+                                  e, instance=instance)
 
         # File injection
         elif CONF.libvirt_inject_partition != -2:
@@ -1892,10 +1894,10 @@ class LibvirtDriver(driver.ComputeDriver):
                                      use_cow=CONF.use_cow_images,
                                      mandatory=('files',))
                 except Exception as e:
-                    LOG.error(_('Error injecting data into image '
-                                '%(img_id)s (%(e)s)') % locals(),
-                              instance=instance)
-                    raise
+                    with excutils.save_and_reraise_exception():
+                        LOG.error(_('Error injecting data into image '
+                                    '%(img_id)s (%(e)s)') % locals(),
+                                  instance=instance)
 
         if CONF.libvirt_type == 'uml':
             libvirt_utils.chown(image('disk').path, 'root')
@@ -2950,9 +2952,9 @@ class LibvirtDriver(driver.ComputeDriver):
         try:
             ret = self._conn.compareCPU(cpu.to_xml(), 0)
         except libvirt.libvirtError, e:
-            ret = e.message
-            LOG.error(m % locals())
-            raise
+            with excutils.save_and_reraise_exception():
+                ret = e.message
+                LOG.error(m % locals())
 
         if ret <= 0:
             LOG.error(m % locals())
diff --git a/nova/wsgi.py b/nova/wsgi.py
index b9e204b90..72e464919 100644
--- a/nova/wsgi.py
+++ b/nova/wsgi.py
@@ -34,6 +34,7 @@ import webob.dec
 import webob.exc
 
 from nova import exception
+from nova.openstack.common import excutils
 from nova.openstack.common import log as logging
 
 wsgi_opts = [
@@ -175,9 +176,9 @@ class Server(object):
                                     CONF.tcp_keepidle)
 
             except Exception:
-                LOG.error(_("Failed to start %(name)s on %(host)s"
-                            ":%(port)s with SSL support") % self.__dict__)
-                raise
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_("Failed to start %(name)s on %(host)s"
+                                ":%(port)s with SSL support") % self.__dict__)
 
         wsgi_kwargs = {
             'func': eventlet.wsgi.server,