Send api.fault notification on API service faults.

Exceptions caught by the API middleware will now send "api.fault"
notifications indicating the offending URL and the underlying exception.
The allows the billing/usage systems to track requests end-to-end without
having to monitor log files. Disabled by default.
Enable with --notify_api_faults

Change-Id: Ied2783259413ba12d686b44b33e0707645e51afb

3 files changed, 47 insertions, 0 deletions

diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py
index 12dd8ae83..ada6024cf 100644
--- a/nova/api/openstack/__init__.py
+++ b/nova/api/openstack/__init__.py
@@ -25,6 +25,7 @@ import webob.dec
 import webob.exc
 
 from nova.api.openstack import wsgi
+from nova import notifications
 from nova.openstack.common import log as logging
 from nova import utils
 from nova import wsgi as base_wsgi
@@ -70,6 +71,8 @@ class FaultWrapper(base_wsgi.Middleware):
         if safe:
             outer.explanation = '%s: %s' % (inner.__class__.__name__,
                                             unicode(inner))
+
+        notifications.send_api_fault(req.url, status, inner)
         return wsgi.Fault(outer)
 
     @webob.dec.wsgify(RequestClass=wsgi.Request)
diff --git a/nova/notifications.py b/nova/notifications.py
index 73b318c5c..2df5b95f7 100644
--- a/nova/notifications.py
+++ b/nova/notifications.py
@@ -45,9 +45,29 @@ notify_any_opt = cfg.BoolOpt('notify_on_any_change', default=False,
          'state changes.  Valid values are False for no notifications, '
          'True for notifications on any instance changes.')
 
+notify_api_faults = cfg.BoolOpt('notify_api_faults', default=False,
+    help='If set, send api.fault notifications on caught exceptions '
+         'in the API service.')
+
+
 FLAGS = flags.FLAGS
 FLAGS.register_opt(notify_state_opt)
 FLAGS.register_opt(notify_any_opt)
+FLAGS.register_opt(notify_api_faults)
+
+
+def send_api_fault(url, status, exception):
+    """Send an api.fault notification."""
+
+    if not FLAGS.notify_api_faults:
+        return
+
+    payload = {'url': url, 'exception': str(exception), 'status': status}
+
+    publisher_id = notifier_api.publisher_id("api")
+
+    notifier_api.notify(None, publisher_id, 'api.fault', notifier_api.ERROR,
+                        payload)
 
 
 def send_update(context, old_instance, new_instance, service=None, host=None):
diff --git a/nova/tests/test_notifications.py b/nova/tests/test_notifications.py
index 94922941e..d451d852c 100644
--- a/nova/tests/test_notifications.py
+++ b/nova/tests/test_notifications.py
@@ -86,6 +86,30 @@ class NotificationsTestCase(test.TestCase):
             inst.update(params)
         return db.instance_create(self.context, inst)
 
+    def test_send_api_fault_disabled(self):
+        self.flags(notify_api_faults=False)
+        notifications.send_api_fault("http://example.com/foo", 500, None)
+        self.assertEquals(0, len(test_notifier.NOTIFICATIONS))
+
+    def test_send_api_fault(self):
+        self.flags(notify_api_faults=True)
+        exception = None
+        try:
+            # Get a real exception with a call stack.
+            raise test.TestingException("junk")
+        except test.TestingException, e:
+            exception = e
+
+        notifications.send_api_fault("http://example.com/foo", 500, exception)
+
+        self.assertEquals(1, len(test_notifier.NOTIFICATIONS))
+        n = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(n['priority'], 'ERROR')
+        self.assertEquals(n['event_type'], 'api.fault')
+        self.assertEquals(n['payload']['url'], 'http://example.com/foo')
+        self.assertEquals(n['payload']['status'], 500)
+        self.assertTrue(n['payload']['exception'] is not None)
+
     def test_notif_disabled(self):
 
         # test config disable of the notifcations