Changed shared_ip_group detail routing

7 files changed, 330 insertions, 15 deletions

diff --git a/bin/nova-api-paste b/bin/nova-api-paste
index 6ee833a18..419f0bbdc 100755
--- a/bin/nova-api-paste
+++ b/bin/nova-api-paste
@@ -21,7 +21,6 @@
 """Starter script for Nova API."""
 
 import gettext
-import logging
 import os
 import sys
 
@@ -38,6 +37,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
 gettext.install('nova', unicode=1)
 
 from nova import flags
+from nova import log as logging
 from nova import wsgi
 
 LOG = logging.getLogger('nova.api')
diff --git a/bin/nova-console b/bin/nova-console
new file mode 100755
index 000000000..802cc80b6
--- /dev/null
+++ b/bin/nova-console
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2010 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Starter script for Nova Console Proxy."""
+
+import eventlet
+eventlet.monkey_patch()
+
+import gettext
+import os
+import sys
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+gettext.install('nova', unicode=1)
+
+from nova import service
+from nova import utils
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    service.serve()
+    service.wait()
diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge
index 828aba3d1..1a994d956 100755
--- a/bin/nova-dhcpbridge
+++ b/bin/nova-dhcpbridge
@@ -22,7 +22,6 @@ Handle lease database updates from DHCP servers.
 """
 
 import gettext
-import logging
 import os
 import sys
 
@@ -39,6 +38,7 @@ gettext.install('nova', unicode=1)
 from nova import context
 from nova import db
 from nova import flags
+from nova import log as logging
 from nova import rpc
 from nova import utils
 from nova.network import linux_net
@@ -49,11 +49,13 @@ flags.DECLARE('network_size', 'nova.network.manager')
 flags.DECLARE('num_networks', 'nova.network.manager')
 flags.DECLARE('update_dhcp_on_disassociate', 'nova.network.manager')
 
+LOG = logging.getLogger('nova.dhcpbridge')
+
 
 def add_lease(mac, ip_address, _hostname, _interface):
     """Set the IP that was assigned by the DHCP server."""
     if FLAGS.fake_rabbit:
-        logging.debug("leasing ip")
+        LOG.debug(_("leasing ip"))
         network_manager = utils.import_object(FLAGS.network_manager)
         network_manager.lease_fixed_ip(context.get_admin_context(),
                                        mac,
@@ -68,14 +70,14 @@ def add_lease(mac, ip_address, _hostname, _interface):
 
 def old_lease(mac, ip_address, hostname, interface):
     """Update just as add lease."""
-    logging.debug("Adopted old lease or got a change of mac/hostname")
+    LOG.debug(_("Adopted old lease or got a change of mac/hostname"))
     add_lease(mac, ip_address, hostname, interface)
 
 
 def del_lease(mac, ip_address, _hostname, _interface):
     """Called when a lease expires."""
     if FLAGS.fake_rabbit:
-        logging.debug("releasing ip")
+        LOG.debug(_("releasing ip"))
         network_manager = utils.import_object(FLAGS.network_manager)
         network_manager.release_fixed_ip(context.get_admin_context(),
                                          mac,
@@ -100,6 +102,7 @@ def main():
     flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile)
     utils.default_flagfile(flagfile)
     argv = FLAGS(sys.argv)
+    logging.basicConfig()
     interface = os.environ.get('DNSMASQ_INTERFACE', 'br0')
     if int(os.environ.get('TESTING', '0')):
         FLAGS.fake_rabbit = True
@@ -117,9 +120,9 @@ def main():
         mac = argv[2]
         ip = argv[3]
         hostname = argv[4]
-        logging.debug("Called %s for mac %s with ip %s and "
-                      "hostname %s on interface %s",
-                      action, mac, ip, hostname, interface)
+        LOG.debug(_("Called %s for mac %s with ip %s and "
+                    "hostname %s on interface %s"),
+                  action, mac, ip, hostname, interface)
         globals()[action + '_lease'](mac, ip, hostname, interface)
     else:
         print init_leases(interface)
diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor
index 5dac3ffe6..7dca02014 100755
--- a/bin/nova-instancemonitor
+++ b/bin/nova-instancemonitor
@@ -23,7 +23,6 @@
 
 import gettext
 import os
-import logging
 import sys
 from twisted.application import service
 
@@ -37,19 +36,23 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
 
 gettext.install('nova', unicode=1)
 
+from nova import log as logging
 from nova import utils
 from nova import twistd
 from nova.compute import monitor
 
+# TODO(todd): shouldn't this be done with flags?  And what about verbose?
 logging.getLogger('boto').setLevel(logging.WARN)
 
+LOG = logging.getLogger('nova.instancemonitor')
+
 
 if __name__ == '__main__':
     utils.default_flagfile()
     twistd.serve(__file__)
 
 if __name__ == '__builtin__':
-    logging.warn('Starting instance monitor')
+    LOG.warn(_('Starting instance monitor'))
     # pylint: disable-msg=C0103
     monitor = monitor.InstanceMonitor()
 
diff --git a/bin/nova-logspool b/bin/nova-logspool
new file mode 100644
index 000000000..097459b12
--- /dev/null
+++ b/bin/nova-logspool
@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""
+Tools for working with logs generated by nova components
+"""
+
+
+import json
+import os
+import re
+import sys
+
+
+class Request(object):
+    def __init__(self):
+        self.time = ""
+        self.host = ""
+        self.logger = ""
+        self.message = ""
+        self.trace = ""
+        self.env = ""
+        self.request_id = ""
+
+    def add_error_line(self, error_line):
+        self.time = " ".join(error_line.split(" ")[:3])
+        self.host = error_line.split(" ")[3]
+        self.logger = error_line.split("(")[1].split(" ")[0]
+        self.request_id = error_line.split("[")[1].split(" ")[0]
+        error_lines = error_line.split("#012")
+        self.message = self.clean_log_line(error_lines.pop(0))
+        self.trace = "\n".join([self.clean_trace(l) for l in error_lines])
+
+    def add_environment_line(self, env_line):
+        self.env = self.clean_env_line(env_line)
+
+    def clean_log_line(self, line):
+        """Remove log format for time, level, etc: split after context"""
+        return line.split('] ')[-1]
+
+    def clean_env_line(self, line):
+        """Also has an 'Environment: ' string in the message"""
+        return re.sub(r'^Environment: ', '', self.clean_log_line(line))
+
+    def clean_trace(self, line):
+        """trace has a different format, so split on TRACE:"""
+        return line.split('TRACE: ')[-1]
+
+    def to_dict(self):
+        return {'traceback': self.trace, 'message': self.message,
+                'host': self.host, 'env': self.env, 'logger': self.logger,
+                'request_id': self.request_id}
+
+
+class LogReader(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self._errors = {}
+
+    def process(self, spooldir):
+        with open(self.filename) as f:
+            line = f.readline()
+            while len(line) > 0:
+                parts = line.split(" ")
+                level = (len(parts) < 6) or parts[5]
+                if level == 'ERROR':
+                    self.handle_logged_error(line)
+                elif level == '[-]' and self.last_error:
+                    # twisted stack trace line
+                    clean_line = " ".join(line.split(" ")[6:])
+                    self.last_error.trace = self.last_error.trace + clean_line
+                else:
+                    self.last_error = None
+                line = f.readline()
+        self.update_spool(spooldir)
+
+    def handle_logged_error(self, line):
+        request_id = re.search(r' \[([A-Z0-9\-/]+)', line)
+        if not request_id:
+            raise Exception("Unable to parse request id from %s" % line)
+        request_id = request_id.group(1)
+        data = self._errors.get(request_id, Request())
+        if self.is_env_line(line):
+            data.add_environment_line(line)
+        elif self.is_error_line(line):
+            data.add_error_line(line)
+        else:
+            # possibly error from twsited
+            data.add_error_line(line)
+        self.last_error = data
+        self._errors[request_id] = data
+
+    def is_env_line(self, line):
+        return re.search('Environment: ', line)
+
+    def is_error_line(self, line):
+        return re.search('raised', line)
+
+    def update_spool(self, directory):
+        processed_dir = "%s/processed" % directory
+        self._ensure_dir_exists(processed_dir)
+        for rid, value in self._errors.iteritems():
+            if not self.has_been_processed(processed_dir, rid):
+                with open("%s/%s" % (directory, rid), "w") as spool:
+                    spool.write(json.dumps(value.to_dict()))
+        self.flush_old_processed_spool(processed_dir)
+
+    def _ensure_dir_exists(self, d):
+        mkdir = False
+        try:
+            os.stat(d)
+        except:
+            mkdir = True
+        if mkdir:
+            os.mkdir(d)
+
+    def has_been_processed(self, processed_dir, rid):
+        rv = False
+        try:
+            os.stat("%s/%s" % (processed_dir, rid))
+            rv = True
+        except:
+            pass
+        return rv
+
+    def flush_old_processed_spool(self, processed_dir):
+        keys = self._errors.keys()
+        procs = os.listdir(processed_dir)
+        for p in procs:
+            if p not in keys:
+                # log has rotated and the old error won't be seen again
+                os.unlink("%s/%s" % (processed_dir, p))
+
+if __name__ == '__main__':
+    filename = '/var/log/nova.log'
+    spooldir = '/var/spool/nova'
+    if len(sys.argv) > 1:
+        filename = sys.argv[1]
+    if len(sys.argv) > 2:
+        spooldir = sys.argv[2]
+    LogReader(filename).process(spooldir)
diff --git a/bin/nova-manage b/bin/nova-manage
index 3416c1a52..3f5957190 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -55,8 +55,8 @@
 
 import datetime
 import gettext
-import logging
 import os
+import re
 import sys
 import time
 
@@ -333,6 +333,11 @@ class ProjectCommands(object):
         arguments: name project_manager [description]"""
         self.manager.create_project(name, project_manager, description)
 
+    def modify(self, name, project_manager, description=None):
+        """Modifies a project
+        arguments: name project_manager [description]"""
+        self.manager.modify_project(name, project_manager, description)
+
     def delete(self, name):
         """Deletes an existing project
         arguments: name"""
@@ -499,6 +504,15 @@ class ServiceCommands(object):
         db.service_update(ctxt, svc['id'], {'disabled': True})
 
 
+class LogCommands(object):
+    def request(self, request_id, logfile='/var/log/nova.log'):
+        """Show all fields in the log for the given request.  Assumes you
+        haven't changed the log format too much.
+        ARGS: request_id [logfile]"""
+        lines = utils.execute("cat %s | grep '\[%s '" % (logfile, request_id))
+        print re.sub('#012', "\n", "\n".join(lines))
+
+
 CATEGORIES = [
     ('user', UserCommands),
     ('project', ProjectCommands),
@@ -507,7 +521,8 @@ CATEGORIES = [
     ('vpn', VpnCommands),
     ('floating', FloatingIpCommands),
     ('network', NetworkCommands),
-    ('service', ServiceCommands)]
+    ('service', ServiceCommands),
+    ('log', LogCommands)]
 
 
 def lazy_match(name, key_value_tuples):
@@ -546,9 +561,6 @@ def main():
     utils.default_flagfile()
     argv = FLAGS(sys.argv)
 
-    if FLAGS.verbose:
-        logging.getLogger().setLevel(logging.DEBUG)
-
     script_name = argv.pop(0)
     if len(argv) < 1:
         print script_name + " category action [<args>]"
diff --git a/bin/nova-spoolsentry b/bin/nova-spoolsentry
new file mode 100644
index 000000000..ab20268a9
--- /dev/null
+++ b/bin/nova-spoolsentry
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import base64
+import json
+import logging
+import os
+import shutil
+import sys
+import urllib
+import urllib2
+try:
+    import cPickle as pickle
+except:
+    import pickle
+
+
+class SpoolSentry(object):
+    def __init__(self, spool_dir, sentry_url, key=None):
+        self.spool_dir = spool_dir
+        self.sentry_url = sentry_url
+        self.key = key
+
+    def process(self):
+        for fname in os.listdir(self.spool_dir):
+            if fname == "processed":
+                continue
+            try:
+                sourcefile = "%s/%s" % (self.spool_dir, fname)
+                with open(sourcefile) as f:
+                    fdata = f.read()
+                data_from_json = json.loads(fdata)
+                data = self.build_data(data_from_json)
+                self.send_data(data)
+                destfile = "%s/processed/%s" % (self.spool_dir, fname)
+                shutil.move(sourcefile, destfile)
+            except:
+                logging.exception("Unable to upload record %s", fname)
+                raise
+
+    def build_data(self, filejson):
+        env = {'SERVER_NAME': 'unknown', 'SERVER_PORT': '0000',
+               'SCRIPT_NAME': '/unknown/', 'PATH_INFO': 'unknown'}
+        if filejson['env']:
+            env = json.loads(filejson['env'])
+        url = "http://%s:%s%s%s" % (env['SERVER_NAME'], env['SERVER_PORT'],
+                                    env['SCRIPT_NAME'], env['PATH_INFO'])
+        rv = {'logger': filejson['logger'], 'level': logging.ERROR,
+              'server_name': filejson['host'], 'url': url,
+              'message': filejson['message'],
+              'traceback': filejson['traceback']}
+        rv['data'] = {}
+        if filejson['env']:
+            rv['data']['META'] = env
+        if filejson['request_id']:
+            rv['data']['request_id'] = filejson['request_id']
+        return rv
+
+    def send_data(self, data):
+        data = {
+            'data': base64.b64encode(pickle.dumps(data).encode('zlib')),
+            'key': self.key
+        }
+        req = urllib2.Request(self.sentry_url)
+        res = urllib2.urlopen(req, urllib.urlencode(data))
+        if res.getcode() != 200:
+            raise Exception("Bad HTTP code: %s" % res.getcode())
+        txt = res.read()
+
+if __name__ == '__main__':
+    sentryurl = 'http://127.0.0.1/sentry/store/'
+    key = ''
+    spooldir = '/var/spool/nova'
+    if len(sys.argv) > 1:
+        sentryurl = sys.argv[1]
+    if len(sys.argv) > 2:
+        key = sys.argv[2]
+    if len(sys.argv) > 3:
+        spooldir = sys.argv[3]
+    SpoolSentry(spooldir, sentryurl, key).process()