merged trunk

12 files changed, 760 insertions, 146 deletions

diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy
new file mode 100755
index 000000000..2bc407658
--- /dev/null
+++ b/bin/nova-ajax-console-proxy
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# pylint: disable-msg=C0103
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""Ajax Console Proxy Server"""
+
+from eventlet import greenthread
+from eventlet.green import urllib2
+
+import exceptions
+import gettext
+import logging
+import os
+import sys
+import time
+import urlparse
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+gettext.install('nova', unicode=1)
+
+from nova import flags
+from nova import log as logging
+from nova import rpc
+from nova import utils
+from nova import wsgi
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_integer('ajax_console_idle_timeout', 300,
+                     'Seconds before idle connection destroyed')
+
+LOG = logging.getLogger('nova.ajax_console_proxy')
+LOG.setLevel(logging.DEBUG)
+LOG.addHandler(logging.StreamHandler())
+
+
+class AjaxConsoleProxy(object):
+    tokens = {}
+
+    def __call__(self, env, start_response):
+        try:
+            req_url = '%s://%s%s?%s' % (env['wsgi.url_scheme'],
+                                        env['HTTP_HOST'],
+                                        env['PATH_INFO'],
+                                        env['QUERY_STRING'])
+            if 'HTTP_REFERER' in env:
+                auth_url = env['HTTP_REFERER']
+            else:
+                auth_url = req_url
+
+            auth_params = urlparse.parse_qs(urlparse.urlparse(auth_url).query)
+            parsed_url = urlparse.urlparse(req_url)
+
+            auth_info = AjaxConsoleProxy.tokens[auth_params['token'][0]]
+            args = auth_info['args']
+            auth_info['last_activity'] = time.time()
+
+            remote_url = ("http://%s:%s%s?token=%s" % (
+                          str(args['host']),
+                          str(args['port']),
+                          parsed_url.path,
+                          str(args['token'])))
+
+            opener = urllib2.urlopen(remote_url, env['wsgi.input'].read())
+            body = opener.read()
+            info = opener.info()
+
+            start_response("200 OK", info.dict.items())
+            return body
+        except (exceptions.KeyError):
+            if env['PATH_INFO'] != '/favicon.ico':
+                LOG.audit("Unauthorized request %s, %s"
+                          % (req_url, str(env)))
+            start_response("401 NOT AUTHORIZED", [])
+            return "Not Authorized"
+        except Exception:
+            start_response("500 ERROR", [])
+            return "Server Error"
+
+    def register_listeners(self):
+        class Callback:
+            def __call__(self, data, message):
+                if data['method'] == 'authorize_ajax_console':
+                    AjaxConsoleProxy.tokens[data['args']['token']] =  \
+                        {'args': data['args'], 'last_activity': time.time()}
+
+        conn = rpc.Connection.instance(new=True)
+        consumer = rpc.TopicConsumer(
+                        connection=conn,
+                        topic=FLAGS.ajax_console_proxy_topic)
+        consumer.register_callback(Callback())
+
+        def delete_expired_tokens():
+            now = time.time()
+            to_delete = []
+            for k, v in AjaxConsoleProxy.tokens.items():
+                if now - v['last_activity'] > FLAGS.ajax_console_idle_timeout:
+                    to_delete.append(k)
+
+            for k in to_delete:
+                del AjaxConsoleProxy.tokens[k]
+
+        utils.LoopingCall(consumer.fetch, auto_ack=True,
+                          enable_callbacks=True).start(0.1)
+        utils.LoopingCall(delete_expired_tokens).start(1)
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    FLAGS(sys.argv)
+    server = wsgi.Server()
+    acp = AjaxConsoleProxy()
+    acp.register_listeners()
+    server.start(acp, FLAGS.ajax_console_proxy_port, host='0.0.0.0')
+    server.wait()
diff --git a/bin/nova-api b/bin/nova-api
index 1c671201e..7b4fbeab1 100755
--- a/bin/nova-api
+++ b/bin/nova-api
@@ -34,23 +34,53 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
 
 gettext.install('nova', unicode=1)
 
-from nova import api
 from nova import flags
-from nova import utils
+from nova import log as logging
 from nova import wsgi
 
+logging.basicConfig()
+LOG = logging.getLogger('nova.api')
+LOG.setLevel(logging.DEBUG)
 
 FLAGS = flags.FLAGS
-flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port')
-flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host')
-flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port')
-flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host')
 
+API_ENDPOINTS = ['ec2', 'osapi']
 
-if __name__ == '__main__':
-    utils.default_flagfile()
-    FLAGS(sys.argv)
+
+def run_app(paste_config_file):
+    LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file)
+    apps = []
+    for api in API_ENDPOINTS:
+        config = wsgi.load_paste_configuration(paste_config_file, api)
+        if config is None:
+            LOG.debug(_("No paste configuration for app: %s"), api)
+            continue
+        LOG.debug(_("App Config: %s\n%r"), api, config)
+        wsgi.paste_config_to_flags(config, {
+            "verbose": FLAGS.verbose,
+            "%s_host" % api: config.get('host', '0.0.0.0'),
+            "%s_port" % api: getattr(FLAGS, "%s_port" % api)})
+        LOG.info(_("Running %s API"), api)
+        app = wsgi.load_paste_app(paste_config_file, api)
+        apps.append((app, getattr(FLAGS, "%s_port" % api),
+                     getattr(FLAGS, "%s_host" % api)))
+    if len(apps) == 0:
+        LOG.error(_("No known API applications configured in %s."),
+                  paste_config_file)
+        return
+
+    # NOTE(todd): redo logging config, verbose could be set in paste config
+    logging.basicConfig()
     server = wsgi.Server()
-    server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host)
-    server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host)
+    for app in apps:
+        server.start(*app)
     server.wait()
+
+
+if __name__ == '__main__':
+    FLAGS(sys.argv)
+    conf = wsgi.paste_config_file('nova-api.conf')
+    if conf:
+        run_app(conf)
+    else:
+        LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf')
diff --git a/bin/nova-api-paste b/bin/nova-api-paste
deleted file mode 100755
index 6ee833a18..000000000
--- a/bin/nova-api-paste
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-# pylint: disable-msg=C0103
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-"""Starter script for Nova API."""
-
-import gettext
-import logging
-import os
-import sys
-
-from paste import deploy
-
-# If ../nova/__init__.py exists, add ../ to Python search path, so that
-# it will override what happens to be installed in /usr/(local/)lib/python...
-possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
-                                   os.pardir,
-                                   os.pardir))
-if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
-    sys.path.insert(0, possible_topdir)
-
-gettext.install('nova', unicode=1)
-
-from nova import flags
-from nova import wsgi
-
-LOG = logging.getLogger('nova.api')
-LOG.setLevel(logging.DEBUG)
-LOG.addHandler(logging.StreamHandler())
-
-FLAGS = flags.FLAGS
-
-API_ENDPOINTS = ['ec2', 'openstack']
-
-
-def load_configuration(paste_config):
-    """Load the paste configuration from the config file and return it."""
-    config = None
-    # Try each known name to get the global DEFAULTS, which will give ports
-    for name in API_ENDPOINTS:
-        try:
-            config = deploy.appconfig("config:%s" % paste_config, name=name)
-        except LookupError:
-            pass
-        if config:
-            verbose = config.get('verbose', None)
-            if verbose:
-                FLAGS.verbose = int(verbose) == 1
-                if FLAGS.verbose:
-                    logging.getLogger().setLevel(logging.DEBUG)
-            return config
-    LOG.debug(_("Paste config at %s has no secion for known apis"),
-              paste_config)
-    print _("Paste config at %s has no secion for any known apis") % \
-          paste_config
-    os.exit(1)
-
-
-def launch_api(paste_config_file, section, server, port, host):
-    """Launch an api server from the specified port and IP."""
-    LOG.debug(_("Launching %s api on %s:%s"), section, host, port)
-    app = deploy.loadapp('config:%s' % paste_config_file, name=section)
-    server.start(app, int(port), host)
-
-
-def run_app(paste_config_file):
-    LOG.debug(_("Using paste.deploy config at: %s"), configfile)
-    config = load_configuration(paste_config_file)
-    LOG.debug(_("Configuration: %r"), config)
-    server = wsgi.Server()
-    ip = config.get('host', '0.0.0.0')
-    for api in API_ENDPOINTS:
-        port = config.get("%s_port" % api, None)
-        if not port:
-            continue
-        host = config.get("%s_host" % api, ip)
-        launch_api(configfile, api, server, port, host)
-    LOG.debug(_("All api servers launched, now waiting"))
-    server.wait()
-
-
-if __name__ == '__main__':
-    FLAGS(sys.argv)
-    configfiles = ['/etc/nova/nova-api.conf']
-    if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
-        configfiles.insert(0,
-                       os.path.join(possible_topdir, 'etc', 'nova-api.conf'))
-    for configfile in configfiles:
-        if os.path.exists(configfile):
-            run_app(configfile)
-            break
-        else:
-            LOG.debug(_("Skipping missing configuration: %s"), configfile)
diff --git a/bin/nova-combined b/bin/nova-combined
index 53322f1a0..913c866bf 100755
--- a/bin/nova-combined
+++ b/bin/nova-combined
@@ -36,23 +36,20 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
 
 gettext.install('nova', unicode=1)
 
-from nova import api
 from nova import flags
+from nova import log as logging
 from nova import service
 from nova import utils
 from nova import wsgi
 
 
 FLAGS = flags.FLAGS
-flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port')
-flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host')
-flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port')
-flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host')
 
 
 if __name__ == '__main__':
     utils.default_flagfile()
     FLAGS(sys.argv)
+    logging.basicConfig()
 
     compute = service.Service.create(binary='nova-compute')
     network = service.Service.create(binary='nova-network')
@@ -62,7 +59,22 @@ if __name__ == '__main__':
 
     service.serve(compute, network, volume, scheduler)
 
-    server = wsgi.Server()
-    server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host)
-    server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host)
-    server.wait()
+    apps = []
+    paste_config_file = wsgi.paste_config_file('nova-api.conf')
+    for api in ['osapi', 'ec2']:
+        config = wsgi.load_paste_configuration(paste_config_file, api)
+        if config is None:
+            continue
+        wsgi.paste_config_to_flags(config, {
+            "verbose": FLAGS.verbose,
+            "%s_host" % api: config.get('host', '0.0.0.0'),
+            "%s_port" % api: getattr(FLAGS, "%s_port" % api)})
+        app = wsgi.load_paste_app(paste_config_file, api)
+        apps.append((app, getattr(FLAGS, "%s_port" % api),
+                     getattr(FLAGS, "%s_host" % api)))
+    if len(apps) > 0:
+        logging.basicConfig()
+        server = wsgi.Server()
+        for app in apps:
+            server.start(*app)
+        server.wait()
diff --git a/bin/nova-console b/bin/nova-console
new file mode 100755
index 000000000..802cc80b6
--- /dev/null
+++ b/bin/nova-console
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2010 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Starter script for Nova Console Proxy."""
+
+import eventlet
+eventlet.monkey_patch()
+
+import gettext
+import os
+import sys
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+gettext.install('nova', unicode=1)
+
+from nova import service
+from nova import utils
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    service.serve()
+    service.wait()
diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge
index 828aba3d1..1a994d956 100755
--- a/bin/nova-dhcpbridge
+++ b/bin/nova-dhcpbridge
@@ -22,7 +22,6 @@ Handle lease database updates from DHCP servers.
 """
 
 import gettext
-import logging
 import os
 import sys
 
@@ -39,6 +38,7 @@ gettext.install('nova', unicode=1)
 from nova import context
 from nova import db
 from nova import flags
+from nova import log as logging
 from nova import rpc
 from nova import utils
 from nova.network import linux_net
@@ -49,11 +49,13 @@ flags.DECLARE('network_size', 'nova.network.manager')
 flags.DECLARE('num_networks', 'nova.network.manager')
 flags.DECLARE('update_dhcp_on_disassociate', 'nova.network.manager')
 
+LOG = logging.getLogger('nova.dhcpbridge')
+
 
 def add_lease(mac, ip_address, _hostname, _interface):
     """Set the IP that was assigned by the DHCP server."""
     if FLAGS.fake_rabbit:
-        logging.debug("leasing ip")
+        LOG.debug(_("leasing ip"))
         network_manager = utils.import_object(FLAGS.network_manager)
         network_manager.lease_fixed_ip(context.get_admin_context(),
                                        mac,
@@ -68,14 +70,14 @@ def add_lease(mac, ip_address, _hostname, _interface):
 
 def old_lease(mac, ip_address, hostname, interface):
     """Update just as add lease."""
-    logging.debug("Adopted old lease or got a change of mac/hostname")
+    LOG.debug(_("Adopted old lease or got a change of mac/hostname"))
     add_lease(mac, ip_address, hostname, interface)
 
 
 def del_lease(mac, ip_address, _hostname, _interface):
     """Called when a lease expires."""
     if FLAGS.fake_rabbit:
-        logging.debug("releasing ip")
+        LOG.debug(_("releasing ip"))
         network_manager = utils.import_object(FLAGS.network_manager)
         network_manager.release_fixed_ip(context.get_admin_context(),
                                          mac,
@@ -100,6 +102,7 @@ def main():
     flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile)
     utils.default_flagfile(flagfile)
     argv = FLAGS(sys.argv)
+    logging.basicConfig()
     interface = os.environ.get('DNSMASQ_INTERFACE', 'br0')
     if int(os.environ.get('TESTING', '0')):
         FLAGS.fake_rabbit = True
@@ -117,9 +120,9 @@ def main():
         mac = argv[2]
         ip = argv[3]
         hostname = argv[4]
-        logging.debug("Called %s for mac %s with ip %s and "
-                      "hostname %s on interface %s",
-                      action, mac, ip, hostname, interface)
+        LOG.debug(_("Called %s for mac %s with ip %s and "
+                    "hostname %s on interface %s"),
+                  action, mac, ip, hostname, interface)
         globals()[action + '_lease'](mac, ip, hostname, interface)
     else:
         print init_leases(interface)
diff --git a/bin/nova-direct-api b/bin/nova-direct-api
new file mode 100755
index 000000000..e7dd14fb2
--- /dev/null
+++ b/bin/nova-direct-api
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# pylint: disable-msg=C0103
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""Starter script for Nova Direct API."""
+
+import gettext
+import os
+import sys
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+gettext.install('nova', unicode=1)
+
+from nova import flags
+from nova import utils
+from nova import wsgi
+from nova.api import direct
+from nova.compute import api as compute_api
+
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer('direct_port', 8001, 'Direct API port')
+flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host')
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    FLAGS(sys.argv)
+
+    direct.register_service('compute', compute_api.ComputeAPI())
+    direct.register_service('reflect', direct.Reflection())
+    router = direct.Router()
+    with_json = direct.JsonParamsMiddleware(router)
+    with_req = direct.PostParamsMiddleware(with_json)
+    with_auth = direct.DelegatedAuthMiddleware(with_req)
+
+    server = wsgi.Server()
+    server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host)
+    server.wait()
diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor
index 5dac3ffe6..7dca02014 100755
--- a/bin/nova-instancemonitor
+++ b/bin/nova-instancemonitor
@@ -23,7 +23,6 @@
 
 import gettext
 import os
-import logging
 import sys
 from twisted.application import service
 
@@ -37,19 +36,23 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
 
 gettext.install('nova', unicode=1)
 
+from nova import log as logging
 from nova import utils
 from nova import twistd
 from nova.compute import monitor
 
+# TODO(todd): shouldn't this be done with flags?  And what about verbose?
 logging.getLogger('boto').setLevel(logging.WARN)
 
+LOG = logging.getLogger('nova.instancemonitor')
+
 
 if __name__ == '__main__':
     utils.default_flagfile()
     twistd.serve(__file__)
 
 if __name__ == '__builtin__':
-    logging.warn('Starting instance monitor')
+    LOG.warn(_('Starting instance monitor'))
     # pylint: disable-msg=C0103
     monitor = monitor.InstanceMonitor()
 
diff --git a/bin/nova-logspool b/bin/nova-logspool
new file mode 100644
index 000000000..097459b12
--- /dev/null
+++ b/bin/nova-logspool
@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""
+Tools for working with logs generated by nova components
+"""
+
+
+import json
+import os
+import re
+import sys
+
+
+class Request(object):
+    def __init__(self):
+        self.time = ""
+        self.host = ""
+        self.logger = ""
+        self.message = ""
+        self.trace = ""
+        self.env = ""
+        self.request_id = ""
+
+    def add_error_line(self, error_line):
+        self.time = " ".join(error_line.split(" ")[:3])
+        self.host = error_line.split(" ")[3]
+        self.logger = error_line.split("(")[1].split(" ")[0]
+        self.request_id = error_line.split("[")[1].split(" ")[0]
+        error_lines = error_line.split("#012")
+        self.message = self.clean_log_line(error_lines.pop(0))
+        self.trace = "\n".join([self.clean_trace(l) for l in error_lines])
+
+    def add_environment_line(self, env_line):
+        self.env = self.clean_env_line(env_line)
+
+    def clean_log_line(self, line):
+        """Remove log format for time, level, etc: split after context"""
+        return line.split('] ')[-1]
+
+    def clean_env_line(self, line):
+        """Also has an 'Environment: ' string in the message"""
+        return re.sub(r'^Environment: ', '', self.clean_log_line(line))
+
+    def clean_trace(self, line):
+        """trace has a different format, so split on TRACE:"""
+        return line.split('TRACE: ')[-1]
+
+    def to_dict(self):
+        return {'traceback': self.trace, 'message': self.message,
+                'host': self.host, 'env': self.env, 'logger': self.logger,
+                'request_id': self.request_id}
+
+
+class LogReader(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self._errors = {}
+
+    def process(self, spooldir):
+        with open(self.filename) as f:
+            line = f.readline()
+            while len(line) > 0:
+                parts = line.split(" ")
+                level = (len(parts) < 6) or parts[5]
+                if level == 'ERROR':
+                    self.handle_logged_error(line)
+                elif level == '[-]' and self.last_error:
+                    # twisted stack trace line
+                    clean_line = " ".join(line.split(" ")[6:])
+                    self.last_error.trace = self.last_error.trace + clean_line
+                else:
+                    self.last_error = None
+                line = f.readline()
+        self.update_spool(spooldir)
+
+    def handle_logged_error(self, line):
+        request_id = re.search(r' \[([A-Z0-9\-/]+)', line)
+        if not request_id:
+            raise Exception("Unable to parse request id from %s" % line)
+        request_id = request_id.group(1)
+        data = self._errors.get(request_id, Request())
+        if self.is_env_line(line):
+            data.add_environment_line(line)
+        elif self.is_error_line(line):
+            data.add_error_line(line)
+        else:
+            # possibly error from twsited
+            data.add_error_line(line)
+        self.last_error = data
+        self._errors[request_id] = data
+
+    def is_env_line(self, line):
+        return re.search('Environment: ', line)
+
+    def is_error_line(self, line):
+        return re.search('raised', line)
+
+    def update_spool(self, directory):
+        processed_dir = "%s/processed" % directory
+        self._ensure_dir_exists(processed_dir)
+        for rid, value in self._errors.iteritems():
+            if not self.has_been_processed(processed_dir, rid):
+                with open("%s/%s" % (directory, rid), "w") as spool:
+                    spool.write(json.dumps(value.to_dict()))
+        self.flush_old_processed_spool(processed_dir)
+
+    def _ensure_dir_exists(self, d):
+        mkdir = False
+        try:
+            os.stat(d)
+        except:
+            mkdir = True
+        if mkdir:
+            os.mkdir(d)
+
+    def has_been_processed(self, processed_dir, rid):
+        rv = False
+        try:
+            os.stat("%s/%s" % (processed_dir, rid))
+            rv = True
+        except:
+            pass
+        return rv
+
+    def flush_old_processed_spool(self, processed_dir):
+        keys = self._errors.keys()
+        procs = os.listdir(processed_dir)
+        for p in procs:
+            if p not in keys:
+                # log has rotated and the old error won't be seen again
+                os.unlink("%s/%s" % (processed_dir, p))
+
+if __name__ == '__main__':
+    filename = '/var/log/nova.log'
+    spooldir = '/var/spool/nova'
+    if len(sys.argv) > 1:
+        filename = sys.argv[1]
+    if len(sys.argv) > 2:
+        spooldir = sys.argv[2]
+    LogReader(filename).process(spooldir)
diff --git a/bin/nova-manage b/bin/nova-manage
index 3416c1a52..d0901ddfc 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -55,8 +55,8 @@
 
 import datetime
 import gettext
-import logging
 import os
+import re
 import sys
 import time
 
@@ -77,18 +77,22 @@ from nova import crypto
 from nova import db
 from nova import exception
 from nova import flags
+from nova import log as logging
 from nova import quota
 from nova import utils
 from nova.auth import manager
 from nova.cloudpipe import pipelib
+from nova.db import migration
 
 
+logging.basicConfig()
 FLAGS = flags.FLAGS
 flags.DECLARE('fixed_range', 'nova.network.manager')
 flags.DECLARE('num_networks', 'nova.network.manager')
 flags.DECLARE('network_size', 'nova.network.manager')
 flags.DECLARE('vlan_start', 'nova.network.manager')
 flags.DECLARE('vpn_start', 'nova.network.manager')
+flags.DECLARE('fixed_range_v6', 'nova.network.manager')
 
 
 class VpnCommands(object):
@@ -333,6 +337,11 @@ class ProjectCommands(object):
         arguments: name project_manager [description]"""
         self.manager.create_project(name, project_manager, description)
 
+    def modify(self, name, project_manager, description=None):
+        """Modifies a project
+        arguments: name project_manager [description]"""
+        self.manager.modify_project(name, project_manager, description)
+
     def delete(self, name):
         """Deletes an existing project
         arguments: name"""
@@ -432,11 +441,12 @@ class NetworkCommands(object):
     """Class for managing networks."""
 
     def create(self, fixed_range=None, num_networks=None,
-               network_size=None, vlan_start=None, vpn_start=None):
+               network_size=None, vlan_start=None, vpn_start=None,
+               fixed_range_v6=None):
         """Creates fixed ips for host by range
         arguments: [fixed_range=FLAG], [num_networks=FLAG],
                    [network_size=FLAG], [vlan_start=FLAG],
-                   [vpn_start=FLAG]"""
+                   [vpn_start=FLAG], [fixed_range_v6=FLAG]"""
         if not fixed_range:
             fixed_range = FLAGS.fixed_range
         if not num_networks:
@@ -447,11 +457,13 @@ class NetworkCommands(object):
             vlan_start = FLAGS.vlan_start
         if not vpn_start:
             vpn_start = FLAGS.vpn_start
+        if not fixed_range_v6:
+            fixed_range_v6 = FLAGS.fixed_range_v6
         net_manager = utils.import_object(FLAGS.network_manager)
         net_manager.create_networks(context.get_admin_context(),
                                     fixed_range, int(num_networks),
                                     int(network_size), int(vlan_start),
-                                    int(vpn_start))
+                                    int(vpn_start), fixed_range_v6)
 
 
 class ServiceCommands(object):
@@ -499,6 +511,30 @@ class ServiceCommands(object):
         db.service_update(ctxt, svc['id'], {'disabled': True})
 
 
+class LogCommands(object):
+    def request(self, request_id, logfile='/var/log/nova.log'):
+        """Show all fields in the log for the given request.  Assumes you
+        haven't changed the log format too much.
+        ARGS: request_id [logfile]"""
+        lines = utils.execute("cat %s | grep '\[%s '" % (logfile, request_id))
+        print re.sub('#012', "\n", "\n".join(lines))
+
+
+class DbCommands(object):
+    """Class for managing the database."""
+
+    def __init__(self):
+        pass
+
+    def sync(self, version=None):
+        """Sync the database up to the most recent version."""
+        return migration.db_sync(version)
+
+    def version(self):
+        """Print the current database version."""
+        print migration.db_version()
+
+
 CATEGORIES = [
     ('user', UserCommands),
     ('project', ProjectCommands),
@@ -507,7 +543,9 @@ CATEGORIES = [
     ('vpn', VpnCommands),
     ('floating', FloatingIpCommands),
     ('network', NetworkCommands),
-    ('service', ServiceCommands)]
+    ('service', ServiceCommands),
+    ('log', LogCommands),
+    ('db', DbCommands)]
 
 
 def lazy_match(name, key_value_tuples):
@@ -546,9 +584,6 @@ def main():
     utils.default_flagfile()
     argv = FLAGS(sys.argv)
 
-    if FLAGS.verbose:
-        logging.getLogger().setLevel(logging.DEBUG)
-
     script_name = argv.pop(0)
     if len(argv) < 1:
         print script_name + " category action [<args>]"
diff --git a/bin/nova-spoolsentry b/bin/nova-spoolsentry
new file mode 100644
index 000000000..ab20268a9
--- /dev/null
+++ b/bin/nova-spoolsentry
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import base64
+import json
+import logging
+import os
+import shutil
+import sys
+import urllib
+import urllib2
+try:
+    import cPickle as pickle
+except:
+    import pickle
+
+
+class SpoolSentry(object):
+    def __init__(self, spool_dir, sentry_url, key=None):
+        self.spool_dir = spool_dir
+        self.sentry_url = sentry_url
+        self.key = key
+
+    def process(self):
+        for fname in os.listdir(self.spool_dir):
+            if fname == "processed":
+                continue
+            try:
+                sourcefile = "%s/%s" % (self.spool_dir, fname)
+                with open(sourcefile) as f:
+                    fdata = f.read()
+                data_from_json = json.loads(fdata)
+                data = self.build_data(data_from_json)
+                self.send_data(data)
+                destfile = "%s/processed/%s" % (self.spool_dir, fname)
+                shutil.move(sourcefile, destfile)
+            except:
+                logging.exception("Unable to upload record %s", fname)
+                raise
+
+    def build_data(self, filejson):
+        env = {'SERVER_NAME': 'unknown', 'SERVER_PORT': '0000',
+               'SCRIPT_NAME': '/unknown/', 'PATH_INFO': 'unknown'}
+        if filejson['env']:
+            env = json.loads(filejson['env'])
+        url = "http://%s:%s%s%s" % (env['SERVER_NAME'], env['SERVER_PORT'],
+                                    env['SCRIPT_NAME'], env['PATH_INFO'])
+        rv = {'logger': filejson['logger'], 'level': logging.ERROR,
+              'server_name': filejson['host'], 'url': url,
+              'message': filejson['message'],
+              'traceback': filejson['traceback']}
+        rv['data'] = {}
+        if filejson['env']:
+            rv['data']['META'] = env
+        if filejson['request_id']:
+            rv['data']['request_id'] = filejson['request_id']
+        return rv
+
+    def send_data(self, data):
+        data = {
+            'data': base64.b64encode(pickle.dumps(data).encode('zlib')),
+            'key': self.key
+        }
+        req = urllib2.Request(self.sentry_url)
+        res = urllib2.urlopen(req, urllib.urlencode(data))
+        if res.getcode() != 200:
+            raise Exception("Bad HTTP code: %s" % res.getcode())
+        txt = res.read()
+
+if __name__ == '__main__':
+    sentryurl = 'http://127.0.0.1/sentry/store/'
+    key = ''
+    spooldir = '/var/spool/nova'
+    if len(sys.argv) > 1:
+        sentryurl = sys.argv[1]
+    if len(sys.argv) > 2:
+        key = sys.argv[2]
+    if len(sys.argv) > 3:
+        spooldir = sys.argv[3]
+    SpoolSentry(spooldir, sentryurl, key).process()
diff --git a/bin/stack b/bin/stack
new file mode 100755
index 000000000..7a6ce5960
--- /dev/null
+++ b/bin/stack
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""CLI for the Direct API."""
+
+import eventlet
+eventlet.monkey_patch()
+
+import os
+import pprint
+import sys
+import textwrap
+import urllib
+import urllib2
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+import gflags
+from nova import utils
+
+
+FLAGS = gflags.FLAGS
+gflags.DEFINE_string('host', '127.0.0.1', 'Direct API host')
+gflags.DEFINE_integer('port', 8001, 'Direct API host')
+gflags.DEFINE_string('user', 'user1', 'Direct API username')
+gflags.DEFINE_string('project', 'proj1', 'Direct API project')
+
+
+USAGE = """usage: stack [options] <controller> <method> [arg1=value arg2=value]
+
+  `stack help` should output the list of available controllers
+  `stack <controller>` should output the available methods for that controller
+  `stack help <controller>` should do the same
+  `stack help <controller> <method>` should output info for a method
+"""
+
+
+def format_help(d):
+    """Format help text, keys are labels and values are descriptions."""
+    indent = max([len(k) for k in d])
+    out = []
+    for k, v in d.iteritems():
+        t = textwrap.TextWrapper(initial_indent='   %s   ' % k.ljust(indent),
+                                 subsequent_indent=' ' * (indent + 6))
+        out.extend(t.wrap(v))
+    return out
+
+
+def help_all():
+    rv = do_request('reflect', 'get_controllers')
+    out = format_help(rv)
+    return (USAGE + str(FLAGS.MainModuleHelp()) +
+            '\n\nAvailable controllers:\n' +
+            '\n'.join(out) + '\n')
+
+
+def help_controller(controller):
+    rv = do_request('reflect', 'get_methods')
+    methods = dict([(k.split('/')[2], v) for k, v in rv.iteritems()
+               if k.startswith('/%s' % controller)])
+    return ('Available methods for %s:\n' % controller +
+            '\n'.join(format_help(methods)))
+
+
+def help_method(controller, method):
+    rv = do_request('reflect',
+                    'get_method_info',
+                    {'method': '/%s/%s' % (controller, method)})
+
+    sig = '%s(%s):' % (method, ', '.join(['='.join(x) for x in rv['args']]))
+    out = textwrap.wrap(sig, subsequent_indent=' ' * len('%s(' % method))
+    out.append('\n' + rv['doc'])
+    return '\n'.join(out)
+
+
+def do_request(controller, method, params=None):
+    if params:
+        data = urllib.urlencode(params)
+    else:
+        data = None
+
+    url = 'http://%s:%s/%s/%s' % (FLAGS.host, FLAGS.port, controller, method)
+    headers = {'X-OpenStack-User': FLAGS.user,
+               'X-OpenStack-Project': FLAGS.project}
+
+    req = urllib2.Request(url, data, headers)
+    resp = urllib2.urlopen(req)
+    return utils.loads(resp.read())
+
+
+if __name__ == '__main__':
+    args = FLAGS(sys.argv)
+
+    cmd = args.pop(0)
+    if not args:
+        print help_all()
+        sys.exit()
+
+    first = args.pop(0)
+    if first == 'help':
+        action = help_all
+        params = []
+        if args:
+            params.append(args.pop(0))
+            action = help_controller
+        if args:
+            params.append(args.pop(0))
+            action = help_method
+        print action(*params)
+        sys.exit(0)
+
+    controller = first
+    if not args:
+        print help_controller(controller)
+        sys.exit()
+
+    method = args.pop(0)
+    params = {}
+    for x in args:
+        key, value = x.split('=', 1)
+        params[key] = value
+
+    pprint.pprint(do_request(controller, method, params))