Merged trunk

35 files changed, 667 insertions, 364 deletions

diff --git a/.bzrignore b/.bzrignore
index 93fc868a3..c3a502a1a 100644
--- a/.bzrignore
+++ b/.bzrignore
@@ -1 +1,2 @@
 run_tests.err.log
+.nova-venv
diff --git a/.gitignore b/.gitignore
index 9db87ac29..2afc7a32c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,4 @@ keys
 build/*
 build-stamp
 nova.egg-info
+.nova-venv
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..cd7e233e1
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,31 @@
+venv=.nova-venv
+with_venv=tools/with_venv.sh
+
+build:
+	# Nothing to do
+
+default_test_type:= $(shell if [ -e $(venv) ]; then echo venv; else echo system; fi)
+
+test: test-$(default_test_type)
+
+test-venv: $(venv)
+	$(with_venv) python run_tests.py
+
+test-system:
+	python run_tests.py
+
+clean:
+	rm -rf _trial_temp
+	rm -rf keys
+	rm -rf instances
+	rm -rf networks
+	rm -f run_tests.err.log
+
+clean-all: clean
+	rm -rf $(venv)
+
+$(venv):
+	@echo "You need to install the Nova virtualenv before you can run this."
+	@echo ""
+	@echo "Please run tools/install_venv.py"
+	@exit 1
diff --git a/bin/nova-compute b/bin/nova-compute
index 4738abd4b..e0c12354f 100755
--- a/bin/nova-compute
+++ b/bin/nova-compute
@@ -19,81 +19,14 @@
 
 """
   Twistd daemon for the nova compute nodes.
-  Receives messages via AMQP, manages pool of worker threads
-  for async tasks.
 """
 
-import logging
-import os
-import sys
-
-# NOTE(termie): kludge so that we can run this from the bin directory in the
-#               checkout without having to screw with paths
-NOVA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'nova')
-if os.path.exists(NOVA_PATH):
-    sys.path.insert(0, os.path.dirname(NOVA_PATH))
-
-from twisted.internet import task
-from twisted.application import service
-
-from nova import flags
-from nova import rpc
 from nova import twistd
-from nova.compute import node
-from nova.objectstore import image # For the images_path flag
-
-
-FLAGS = flags.FLAGS
-# NOTE(termie): This file will necessarily be re-imported under different
-#               context when the twistd.serve() call is made below so any
-#               flags we define here will have to be conditionally defined,
-#               flags defined by imported modules are safe.
-if 'compute_report_state_interval' not in FLAGS:
-    flags.DEFINE_integer('compute_report_state_interval', 10,
-                         'seconds between nodes reporting state to cloud',
-                         lower_bound=1)
-logging.getLogger().setLevel(logging.DEBUG)
-
-def main():
-    logging.warn('Starting compute node')
-    n = node.Node()
-    d = n.adopt_instances()
-    d.addCallback(lambda x: logging.info('Adopted %d instances', x))
-
-    conn = rpc.Connection.instance()
-    consumer_all = rpc.AdapterConsumer(
-            connection=conn,
-            topic='%s' % FLAGS.compute_topic,
-            proxy=n)
-
-    consumer_node = rpc.AdapterConsumer(
-            connection=conn,
-            topic='%s.%s' % (FLAGS.compute_topic, FLAGS.node_name),
-            proxy=n)
-
-    bin_name = os.path.basename(__file__)
-    pulse = task.LoopingCall(n.report_state, FLAGS.node_name, bin_name)
-    pulse.start(interval=FLAGS.compute_report_state_interval, now=False)
-
-    consumer_all.attach_to_twisted()
-    consumer_node.attach_to_twisted()
-
-    # This is the parent service that twistd will be looking for when it
-    # parses this file, return it so that we can get it into globals below
-    application = service.Application(bin_name)
-    n.setServiceParent(application)
-    return application
+from nova.compute import service
 
 
-# NOTE(termie): When this script is executed from the commandline what it will
-#               actually do is tell the twistd application runner that it
-#               should run this file as a twistd application (see below).
 if __name__ == '__main__':
     twistd.serve(__file__)
 
-# NOTE(termie): When this script is loaded by the twistd application runner
-#               this code path will be executed and twistd will expect a
-#               variable named 'application' to be available, it will then
-#               handle starting it and stopping it.
 if __name__ == '__builtin__':
-    application = main()
+    application = service.ComputeService.create()
diff --git a/bin/nova-network b/bin/nova-network
new file mode 100755
index 000000000..52d6cb70a
--- /dev/null
+++ b/bin/nova-network
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+  Twistd daemon for the nova network nodes.
+"""
+
+from nova import twistd
+from nova.network import service
+
+
+if __name__ == '__main__':
+    twistd.serve(__file__)
+
+if __name__ == '__builtin__':
+    application = service.NetworkService.create()
diff --git a/bin/nova-volume b/bin/nova-volume
index 7d4b65205..f7a8fad37 100755
--- a/bin/nova-volume
+++ b/bin/nova-volume
@@ -18,77 +18,15 @@
 #    under the License.
 
 """
-    Tornado Storage daemon manages AoE volumes via AMQP messaging.
+  Twistd daemon for the nova volume nodes.
 """
 
-import logging
-import os
-import sys
-
-# NOTE(termie): kludge so that we can run this from the bin directory in the
-#               checkout without having to screw with paths
-NOVA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'nova')
-if os.path.exists(NOVA_PATH):
-    sys.path.insert(0, os.path.dirname(NOVA_PATH))
-
-from twisted.internet import task
-from twisted.application import service
-
-from nova import flags
-from nova import rpc
 from nova import twistd
-from nova.volume import storage
-
-
-FLAGS = flags.FLAGS
-# NOTE(termie): This file will necessarily be re-imported under different
-#               context when the twistd.serve() call is made below so any
-#               flags we define here will have to be conditionally defined,
-#               flags defined by imported modules are safe.
-if 'volume_report_state_interval' not in FLAGS:
-    flags.DEFINE_integer('volume_report_state_interval', 10,
-                         'seconds between nodes reporting state to cloud',
-                         lower_bound=1)
-
-
-def main():
-    logging.warn('Starting volume node')
-    bs = storage.BlockStore()
-
-    conn = rpc.Connection.instance()
-    consumer_all = rpc.AdapterConsumer(
-            connection=conn,
-            topic='%s' % FLAGS.storage_topic,
-            proxy=bs)
-
-    consumer_node = rpc.AdapterConsumer(
-            connection=conn,
-            topic='%s.%s' % (FLAGS.storage_topic, FLAGS.node_name),
-            proxy=bs)
-
-    bin_name = os.path.basename(__file__)
-    pulse = task.LoopingCall(bs.report_state, FLAGS.node_name, bin_name)
-    pulse.start(interval=FLAGS.volume_report_state_interval, now=False)
-
-    consumer_all.attach_to_twisted()
-    consumer_node.attach_to_twisted()
-
-    # This is the parent service that twistd will be looking for when it
-    # parses this file, return it so that we can get it into globals below
-    application = service.Application(bin_name)
-    bs.setServiceParent(application)
-    return application
+from nova.volume import service
 
 
-# NOTE(termie): When this script is executed from the commandline what it will
-#               actually do is tell the twistd application runner that it
-#               should run this file as a twistd application (see below).
 if __name__ == '__main__':
     twistd.serve(__file__)
 
-# NOTE(termie): When this script is loaded by the twistd application runner
-#               this code path will be executed and twistd will expect a
-#               variable named 'application' to be available, it will then
-#               handle starting it and stopping it.
 if __name__ == '__builtin__':
-    application = main()
+    application = service.VolumeService.create()
diff --git a/debian/nova-common.install b/debian/nova-common.install
index 9b1bbf147..93251363a 100644
--- a/debian/nova-common.install
+++ b/debian/nova-common.install
@@ -1,10 +1,9 @@
-bin/nova-manage usr/bin
-debian/nova-manage.conf etc/nova
+bin/nova-manage                       usr/bin
+debian/nova-manage.conf               etc/nova
 nova/auth/novarc.template             usr/share/nova
 nova/cloudpipe/client.ovpn.template   usr/share/nova
 nova/compute/libvirt.xml.template     usr/share/nova
 nova/compute/interfaces.template      usr/share/nova
-usr/lib/python*/*-packages/nova/*
 CA/openssl.cnf.tmpl                   var/lib/nova/CA
 CA/geninter.sh                        var/lib/nova/CA
 CA/genrootca.sh                       var/lib/nova/CA
diff --git a/debian/nova-dhcp.conf b/debian/nova-dhcpbridge.conf
index 68cb8903e..68cb8903e 100644
--- a/debian/nova-dhcp.conf
+++ b/debian/nova-dhcpbridge.conf
diff --git a/nova/auth/manager.py b/nova/auth/manager.py
index 8c8c7377c..b7702d64c 100644
--- a/nova/auth/manager.py
+++ b/nova/auth/manager.py
@@ -342,7 +342,7 @@ class AuthManager(object):
 
     def authenticate(self, access, signature, params, verb='GET',
                      server_string='127.0.0.1:8773', path='/',
-                     verify_signature=True):
+                     check_type='ec2', headers=None):
         """Authenticates AWS request using access key and signature
 
         If the project is not specified, attempts to authenticate to
@@ -367,8 +367,14 @@ class AuthManager(object):
         @type path: str
         @param path: Web request path.
 
-        @type verify_signature: bool
-        @param verify_signature: Whether to verify the signature.
+        @type check_type: str
+        @param check_type: Type of signature to check. 'ec2' for EC2, 's3' for
+                           S3. Any other value will cause signature not to be
+                           checked.
+
+        @type headers: list
+        @param headers: HTTP headers passed with the request (only needed for
+                        s3 signature checks)
 
         @rtype: tuple (User, Project)
         @return: User and project that the request represents.
@@ -376,7 +382,9 @@ class AuthManager(object):
         # TODO(vish): check for valid timestamp
         (access_key, sep, project_id) = access.partition(':')
 
+        logging.info('Looking up user: %r', access_key)
         user = self.get_user_from_access_key(access_key)
+        logging.info('user: %r', user)
         if user == None:
             raise exception.NotFound('No user found for access key %s' %
                                      access_key)
@@ -394,7 +402,14 @@ class AuthManager(object):
                                                                   project):
             raise exception.NotFound('User %s is not a member of project %s' %
                                      (user.id, project.id))
-        if verify_signature:
+        if check_type == 's3':
+            expected_signature = signer.Signer(user.secret.encode()).s3_authorization(headers, verb, path)
+            logging.debug('user.secret: %s', user.secret)
+            logging.debug('expected_signature: %s', expected_signature)
+            logging.debug('signature: %s', signature)
+            if signature != expected_signature:
+                raise exception.NotAuthorized('Signature does not match')
+        elif check_type == 'ec2':
             # NOTE(vish): hmac can't handle unicode, so encode ensures that
             #             secret isn't unicode
             expected_signature = signer.Signer(user.secret.encode()).generate(
diff --git a/nova/auth/signer.py b/nova/auth/signer.py
index 83831bfac..7d7471575 100644
--- a/nova/auth/signer.py
+++ b/nova/auth/signer.py
@@ -48,6 +48,7 @@ import hashlib
 import hmac
 import logging
 import urllib
+import boto.utils
 
 from nova.exception import Error
 
@@ -59,6 +60,13 @@ class Signer(object):
         if hashlib.sha256:
             self.hmac_256 = hmac.new(secret_key, digestmod=hashlib.sha256)
 
+    def s3_authorization(self, headers, verb, path):
+        c_string = boto.utils.canonical_string(verb, path, headers)
+        hmac = self.hmac.copy()
+        hmac.update(c_string)
+        b64_hmac = base64.encodestring(hmac.digest()).strip()
+        return b64_hmac
+
     def generate(self, params, verb, server_string, path):
         if params['SignatureVersion'] == '0':
             return self._calc_signature_0(params)
diff --git a/nova/compute/linux_net.py b/nova/compute/linux_net.py
index 48e07da66..861ce779b 100644
--- a/nova/compute/linux_net.py
+++ b/nova/compute/linux_net.py
@@ -29,7 +29,7 @@ from nova import flags
 FLAGS=flags.FLAGS
 
 flags.DEFINE_string('dhcpbridge_flagfile',
-                    '/etc/nova-dhcpbridge.conf',
+                    '/etc/nova/nova-dhcpbridge.conf',
                     'location of flagfile for dhcpbridge')
 
 def execute(cmd, addl_env=None):
diff --git a/nova/compute/network.py b/nova/compute/network.py
index b5b3c3b5d..62d892e58 100644
--- a/nova/compute/network.py
+++ b/nova/compute/network.py
@@ -144,7 +144,7 @@ class Vlan(datastore.BasicModel):
     @datastore.absorb_connection_error
     def destroy(self):
         set_name = self._redis_set_name(self.__class__.__name__)
-        datastore.Redis.instance().hdel(set_name, self.project)
+        datastore.Redis.instance().hdel(set_name, self.project_id)
 
     def subnet(self):
         vlan = int(self.vlan_id)
@@ -529,6 +529,7 @@ def get_vlan_for_project(project_id):
                 #             don't orphan any VLANs.  It is basically
                 #             garbage collection for after projects abandoned
                 #             their reference.
+                vlan.destroy()
                 vlan.project_id = project_id
                 vlan.save()
                 return vlan
diff --git a/nova/compute/node.py b/nova/compute/service.py
index 7cae86d02..9998dc6c3 100644
--- a/nova/compute/node.py
+++ b/nova/compute/service.py
@@ -17,22 +17,23 @@
 #    under the License.
 
 """
-Compute Node:
+Compute Service:
 
-    Runs on each compute node, managing the
+    Runs on each compute host, managing the
     hypervisor using libvirt.
 
 """
 
 import base64
+import boto.utils
 import json
 import logging
 import os
 import shutil
 import sys
+import time
 from twisted.internet import defer
 from twisted.internet import task
-from twisted.application import service
 
 
 try:
@@ -44,12 +45,14 @@ from nova import exception
 from nova import fakevirt
 from nova import flags
 from nova import process
+from nova import service
 from nova import utils
+from nova.auth import signer, manager
 from nova.compute import disk
 from nova.compute import model
 from nova.compute import network
 from nova.objectstore import image # for image_path flag
-from nova.volume import storage
+from nova.volume import service as volume_service
 
 
 FLAGS = flags.FLAGS
@@ -78,13 +81,13 @@ def _image_url(path):
     return "%s:%s/_images/%s" % (FLAGS.s3_host, FLAGS.s3_port, path)
 
 
-class Node(object, service.Service):
+class ComputeService(service.Service):
     """
     Manages the running instances.
     """
     def __init__(self):
         """ load configuration options for this node and connect to libvirt """
-        super(Node, self).__init__()
+        super(ComputeService, self).__init__()
         self._instances = {}
         self._conn = self._get_connection()
         self.instdir = model.InstanceDirectory()
@@ -221,7 +224,7 @@ class Node(object, service.Service):
     @exception.wrap_exception
     def attach_volume(self, instance_id = None,
                       volume_id = None, mountpoint = None):
-        volume = storage.get_volume(volume_id)
+        volume = volume_service.get_volume(volume_id)
         yield self._init_aoe()
         yield process.simple_execute(
                 "sudo virsh attach-disk %s /dev/etherd/%s %s" %
@@ -242,7 +245,7 @@ class Node(object, service.Service):
         """ detach a volume from an instance """
         # despite the documentation, virsh detach-disk just wants the device
         # name without the leading /dev/
-        volume = storage.get_volume(volume_id)
+        volume = volume_service.get_volume(volume_id)
         target = volume['mountpoint'].rpartition('/dev/')[2]
         yield process.simple_execute(
                 "sudo virsh detach-disk %s %s " % (instance_id, target))
@@ -450,9 +453,25 @@ class Instance(object):
 
     def _fetch_s3_image(self, image, path):
         url = _image_url('%s/image' % image)
-        d = process.simple_execute(
-                'curl --silent %s -o %s' % (url, path))
-        return d
+
+        # This should probably move somewhere else, like e.g. a download_as
+        # method on User objects and at the same time get rewritten to use
+        # twisted web client.
+        headers = {}
+        headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
+
+        user_id = self.datamodel['user_id']
+        user = manager.AuthManager().get_user(user_id)
+        uri = '/' + url.partition('/')[2]
+        auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', uri)
+        headers['Authorization'] = 'AWS %s:%s' % (user.access, auth)
+
+        cmd = ['/usr/bin/curl', '--silent', url]
+        for (k,v) in headers.iteritems():
+            cmd += ['-H', '%s: %s' % (k,v)]
+
+        cmd += ['-o', path]
+        return process.SharedPool().execute(executable=cmd[0], args=cmd[1:])
 
     def _fetch_local_image(self, image, path):
         source = _image_path('%s/image' % image)
diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py
index 8c6c05566..76ca35320 100644
--- a/nova/endpoint/cloud.py
+++ b/nova/endpoint/cloud.py
@@ -23,7 +23,6 @@ datastore.
 """
 
 import base64
-import json
 import logging
 import os
 import time
@@ -38,9 +37,9 @@ from nova.auth import rbac
 from nova.auth import manager
 from nova.compute import model
 from nova.compute import network
-from nova.compute import node
+from nova.compute import service as compute_service
 from nova.endpoint import images
-from nova.volume import storage
+from nova.volume import service as volume_service
 
 
 FLAGS = flags.FLAGS
@@ -76,7 +75,7 @@ class CloudController(object):
     def volumes(self):
         """ returns a list of all volumes """
         for volume_id in datastore.Redis.instance().smembers("volumes"):
-            volume = storage.get_volume(volume_id)
+            volume = volume_service.get_volume(volume_id)
             yield volume
 
     def __str__(self):
@@ -103,7 +102,7 @@ class CloudController(object):
         result = {}
         for instance in self.instdir.all:
             if instance['project_id'] == project_id:
-                line = '%s slots=%d' % (instance['private_dns_name'], node.INSTANCE_TYPES[instance['instance_type']]['vcpus'])
+                line = '%s slots=%d' % (instance['private_dns_name'], compute_service.INSTANCE_TYPES[instance['instance_type']]['vcpus'])
                 if instance['key_name'] in result:
                     result[instance['key_name']].append(line)
                 else:
@@ -296,8 +295,8 @@ class CloudController(object):
 
     @rbac.allow('projectmanager', 'sysadmin')
     def create_volume(self, context, size, **kwargs):
-        # TODO(vish): refactor this to create the volume object here and tell storage to create it
-        res = rpc.call(FLAGS.storage_topic, {"method": "create_volume",
+        # TODO(vish): refactor this to create the volume object here and tell service to create it
+        res = rpc.call(FLAGS.volume_topic, {"method": "create_volume",
                                  "args" : {"size": size,
                                            "user_id": context.user.id,
                                            "project_id": context.project.id}})
@@ -331,7 +330,7 @@ class CloudController(object):
         raise exception.NotFound('Instance %s could not be found' % instance_id)
 
     def _get_volume(self, context, volume_id):
-        volume = storage.get_volume(volume_id)
+        volume = volume_service.get_volume(volume_id)
         if context.user.is_admin() or volume['project_id'] == context.project.id:
             return volume
         raise exception.NotFound('Volume %s could not be found' % volume_id)
@@ -578,7 +577,7 @@ class CloudController(object):
                   "args": {"instance_id" : inst.instance_id}})
             logging.debug("Casting to node for %s's instance with IP of %s" %
                       (context.user.name, inst['private_dns_name']))
-        # TODO: Make the NetworkComputeNode figure out the network name from ip.
+        # TODO: Make Network figure out the network name from ip.
         return defer.succeed(self._format_instances(
                                 context, reservation_id))
 
@@ -628,8 +627,8 @@ class CloudController(object):
     def delete_volume(self, context, volume_id, **kwargs):
         # TODO: return error if not authorized
         volume = self._get_volume(context, volume_id)
-        storage_node = volume['node_name']
-        rpc.cast('%s.%s' % (FLAGS.storage_topic, storage_node),
+        volume_node = volume['node_name']
+        rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node),
                             {"method": "delete_volume",
                              "args" : {"volume_id": volume_id}})
         return defer.succeed(True)
diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py
index 605f9b8e0..de05ba2da 100644
--- a/nova/endpoint/rackspace.py
+++ b/nova/endpoint/rackspace.py
@@ -39,7 +39,6 @@ from nova.compute import model
 from nova.compute import network
 from nova.endpoint import images
 from nova.endpoint import wsgi
-from nova.volume import storage
 
 
 FLAGS = flags.FLAGS
diff --git a/nova/flags.py b/nova/flags.py
index 3ad6a3ad5..ec1390fdd 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -40,7 +40,9 @@ DEFINE_integer('s3_port', 3333, 's3 port')
 DEFINE_string('s3_host', '127.0.0.1', 's3 host')
 #DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on')
 DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on')
-DEFINE_string('storage_topic', 'storage', 'the topic storage nodes listen on')
+DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on')
+DEFINE_string('network_topic', 'network', 'the topic network nodes listen on')
+
 DEFINE_bool('fake_libvirt', False,
                   'whether to use a fake libvirt or not')
 DEFINE_bool('verbose', False, 'show debug output')
diff --git a/nova/network/__init__.py b/nova/network/__init__.py
new file mode 100644
index 000000000..dcc54db09
--- /dev/null
+++ b/nova/network/__init__.py
@@ -0,0 +1,32 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+:mod:`nova.network` -- Network Nodes
+=====================================================
+
+.. automodule:: nova.network
+   :platform: Unix
+   :synopsis: Network is responsible for managing networking
+.. moduleauthor:: Jesse Andrews <jesse@ansolabs.com>
+.. moduleauthor:: Devin Carlen <devin.carlen@gmail.com>
+.. moduleauthor:: Vishvananda Ishaya <vishvananda@yahoo.com>
+.. moduleauthor:: Joshua McKenty <joshua@cognition.ca>
+.. moduleauthor:: Manish Singh <yosh@gimp.org>
+.. moduleauthor:: Andy Smith <andy@anarkystic.com>
+"""
diff --git a/nova/network/service.py b/nova/network/service.py
new file mode 100644
index 000000000..9d87e05e6
--- /dev/null
+++ b/nova/network/service.py
@@ -0,0 +1,35 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Network Nodes are responsible for allocating ips and setting up network
+"""
+
+import logging
+
+from nova import flags
+from nova import service
+
+
+FLAGS = flags.FLAGS
+
+class NetworkService(service.Service):
+    """Allocates ips and sets up networks"""
+
+    def __init__(self):
+        logging.debug("Network node working")
diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py
index 7d997390b..999581c65 100644
--- a/nova/objectstore/handler.py
+++ b/nova/objectstore/handler.py
@@ -47,7 +47,7 @@ import urllib
 
 from twisted.application import internet, service
 from twisted.web.resource import Resource
-from twisted.web import server, static
+from twisted.web import server, static, error
 
 
 from nova import exception
@@ -111,10 +111,10 @@ def get_context(request):
                                                              secret,
                                                              {},
                                                              request.method,
-                                                             request.host,
+                                                             request.getRequestHostname(),
                                                              request.uri,
-                                                             False)
-        # FIXME: check signature here!
+                                                             headers=request.getAllHeaders(),
+                                                             check_type='s3')
         return api.APIRequestContext(None, user, project)
     except exception.Error as ex:
         logging.debug("Authentication Failure: %s" % ex)
@@ -124,15 +124,15 @@ class S3(Resource):
     """Implementation of an S3-like storage server based on local files."""
     def getChild(self, name, request):
         request.context = get_context(request)
-
         if name == '':
             return self
         elif name == '_images':
-            return ImageResource()
+            return ImagesResource()
         else:
             return BucketResource(name)
 
     def render_GET(self, request):
+        logging.debug('List of buckets requested')
         buckets = [b for b in bucket.Bucket.all() if b.is_authorized(request.context)]
 
         render_xml(request, {"ListAllMyBucketsResult": {
@@ -154,7 +154,10 @@ class BucketResource(Resource):
     def render_GET(self, request):
         logging.debug("List keys for bucket %s" % (self.name))
 
-        bucket_object = bucket.Bucket(self.name)
+        try:
+            bucket_object = bucket.Bucket(self.name)
+        except exception.NotFound, e:
+            return error.NoResource(message="No such bucket").render(request)
 
         if not bucket_object.is_authorized(request.context):
             raise exception.NotAuthorized
@@ -170,13 +173,10 @@ class BucketResource(Resource):
 
     def render_PUT(self, request):
         logging.debug("Creating bucket %s" % (self.name))
-        try:
-            print 'user is %s' % request.context
-        except Exception as e:
-            logging.exception(e)
         logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context))
         bucket.Bucket.create(self.name, request.context)
-        return ''
+        request.finish()
+        return server.NOT_DONE_YET
 
     def render_DELETE(self, request):
         logging.debug("Deleting bucket %s" % (self.name))
@@ -234,13 +234,19 @@ class ObjectResource(Resource):
 class ImageResource(Resource):
     isLeaf = True
 
+    def __init__(self, name):
+        Resource.__init__(self)
+        self.img = image.Image(name)
+
+    def render_GET(self, request):
+        return static.File(self.img.image_path, defaultType='application/octet-stream').render_GET(request)
+
+class ImagesResource(Resource):
     def getChild(self, name, request):
         if name == '':
             return self
         else:
-            request.setHeader("Content-Type", "application/octet-stream")
-            img = image.Image(name)
-            return static.File(img.image_path)
+            return ImageResource(name)
 
     def render_GET(self, request):
         """ returns a json listing of all images
@@ -302,9 +308,13 @@ class ImageResource(Resource):
         request.setResponseCode(204)
         return ''
 
-def get_application():
+def get_site():
     root = S3()
-    factory = server.Site(root)
+    site = server.Site(root)
+    return site
+
+def get_application():
+    factory = get_site()
     application = service.Application("objectstore")
     objectStoreService = internet.TCPServer(FLAGS.s3_port, factory)
     objectStoreService.setServiceParent(application)
diff --git a/nova/service.py b/nova/service.py
new file mode 100644
index 000000000..96281bc6b
--- /dev/null
+++ b/nova/service.py
@@ -0,0 +1,103 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Generic Node baseclass for all workers that run on hosts
+"""
+
+import inspect
+import logging
+import os
+
+from twisted.internet import defer
+from twisted.internet import task
+from twisted.application import service
+
+from nova import datastore
+from nova import flags
+from nova import rpc
+from nova.compute import model
+
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_integer('report_interval', 10,
+                     'seconds between nodes reporting state to cloud',
+                     lower_bound=1)
+
+class Service(object, service.Service):
+    """Base class for workers that run on hosts"""
+
+    @classmethod
+    def create(cls,
+               report_interval=None, # defaults to flag
+               bin_name=None, # defaults to basename of executable
+               topic=None): # defaults to basename - "nova-" part
+        """Instantiates class and passes back application object"""
+        if not report_interval:
+            # NOTE(vish): set here because if it is set to flag in the
+            #             parameter list, it wrongly uses the default
+            report_interval = FLAGS.report_interval
+        # NOTE(vish): magic to automatically determine bin_name and topic
+        if not bin_name:
+            bin_name = os.path.basename(inspect.stack()[-1][1])
+        if not topic:
+            topic = bin_name.rpartition("nova-")[2]
+        logging.warn("Starting %s node" % topic)
+        node_instance = cls()
+
+        conn = rpc.Connection.instance()
+        consumer_all = rpc.AdapterConsumer(
+                connection=conn,
+                topic='%s' % topic,
+                proxy=node_instance)
+
+        consumer_node = rpc.AdapterConsumer(
+                connection=conn,
+                topic='%s.%s' % (topic, FLAGS.node_name),
+                proxy=node_instance)
+
+        pulse = task.LoopingCall(node_instance.report_state,
+                                 FLAGS.node_name,
+                                 bin_name)
+        pulse.start(interval=report_interval, now=False)
+
+        consumer_all.attach_to_twisted()
+        consumer_node.attach_to_twisted()
+
+        # This is the parent service that twistd will be looking for when it
+        # parses this file, return it so that we can get it into globals below
+        application = service.Application(bin_name)
+        node_instance.setServiceParent(application)
+        return application
+
+    @defer.inlineCallbacks
+    def report_state(self, nodename, daemon):
+        # TODO(termie): make this pattern be more elegant. -todd
+        try:
+            record = model.Daemon(nodename, daemon)
+            record.heartbeat()
+            if getattr(self, "model_disconnected", False):
+                self.model_disconnected = False
+                logging.error("Recovered model server connection!")
+
+        except datastore.ConnectionError, ex:
+            if not getattr(self, "model_disconnected", False):
+                self.model_disconnected = True
+                logging.exception("model server went away")
+        yield
diff --git a/nova/test.py b/nova/test.py
index 5dcf0b9b0..6fbcab5e4 100644
--- a/nova/test.py
+++ b/nova/test.py
@@ -156,9 +156,9 @@ class BaseTestCase(TrialTestCase):
 
         Example (callback chain, ugly):
 
-        d = self.node.terminate_instance(instance_id) # a Deferred instance
+        d = self.compute.terminate_instance(instance_id) # a Deferred instance
         def _describe(_):
-            d_desc = self.node.describe_instances() # another Deferred instance
+            d_desc = self.compute.describe_instances() # another Deferred instance
             return d_desc
         def _checkDescribe(rv):
             self.assertEqual(rv, [])
@@ -169,8 +169,8 @@ class BaseTestCase(TrialTestCase):
 
         Example (inline callbacks! yay!):
 
-        yield self.node.terminate_instance(instance_id)
-        rv = yield self.node.describe_instances()
+        yield self.compute.terminate_instance(instance_id)
+        rv = yield self.compute.describe_instances()
         self.assertEqual(rv, [])
 
         If the test fits the Inline Callbacks pattern we will automatically
diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py
index 4477a1fe6..9d072866c 100644
--- a/nova/tests/api_unittest.py
+++ b/nova/tests/api_unittest.py
@@ -43,7 +43,11 @@ def boto_to_tornado(method, path, headers, data, host, connection=None):
 
     connection should be a FakeTornadoHttpConnection instance
     """
-    headers = httpserver.HTTPHeaders()
+    try:
+        headers = httpserver.HTTPHeaders()
+    except AttributeError:
+        from tornado import httputil
+        headers = httputil.HTTPHeaders()
     for k, v in headers.iteritems():
         headers[k] = v
 
diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py
index 741973201..344085cc0 100644
--- a/nova/tests/cloud_unittest.py
+++ b/nova/tests/cloud_unittest.py
@@ -28,7 +28,7 @@ from nova import flags
 from nova import rpc
 from nova import test
 from nova.auth import manager
-from nova.compute import node
+from nova.compute import service
 from nova.endpoint import api
 from nova.endpoint import cloud
 
@@ -52,12 +52,12 @@ class CloudTestCase(test.BaseTestCase):
                                                       proxy=self.cloud)
         self.injected.append(self.cloud_consumer.attach_to_tornado(self.ioloop))
 
-        # set up a node
-        self.node = node.Node()
-        self.node_consumer = rpc.AdapterConsumer(connection=self.conn,
+        # set up a service
+        self.compute = service.ComputeService()
+        self.compute_consumer = rpc.AdapterConsumer(connection=self.conn,
                                                      topic=FLAGS.compute_topic,
-                                                     proxy=self.node)
-        self.injected.append(self.node_consumer.attach_to_tornado(self.ioloop))
+                                                     proxy=self.compute)
+        self.injected.append(self.compute_consumer.attach_to_tornado(self.ioloop))
 
         try:
             manager.AuthManager().create_user('admin', 'admin', 'admin')
@@ -75,11 +75,11 @@ class CloudTestCase(test.BaseTestCase):
             logging.debug("Can't test instances without a real virtual env.")
             return
         instance_id = 'foo'
-        inst = yield self.node.run_instance(instance_id)
+        inst = yield self.compute.run_instance(instance_id)
         output = yield self.cloud.get_console_output(self.context, [instance_id])
         logging.debug(output)
         self.assert_(output)
-        rv = yield self.node.terminate_instance(instance_id)
+        rv = yield self.compute.terminate_instance(instance_id)
 
     def test_run_instances(self):
         if FLAGS.fake_libvirt:
@@ -111,7 +111,7 @@ class CloudTestCase(test.BaseTestCase):
              # for instance in reservations[res_id]:
            for instance in reservations[reservations.keys()[0]]:
                logging.debug("Terminating instance %s" % instance['instance_id'])
-               rv = yield self.node.terminate_instance(instance['instance_id'])
+               rv = yield self.compute.terminate_instance(instance['instance_id'])
 
     def test_instance_update_state(self):
         def instance(num):
diff --git a/nova/tests/node_unittest.py b/nova/tests/compute_unittest.py
index 55c957696..5e909abc8 100644
--- a/nova/tests/node_unittest.py
+++ b/nova/tests/compute_unittest.py
@@ -26,7 +26,7 @@ from nova import flags
 from nova import test
 from nova import utils
 from nova.compute import model
-from nova.compute import node
+from nova.compute import service
 
 
 FLAGS = flags.FLAGS
@@ -53,13 +53,13 @@ class InstanceXmlTestCase(test.TrialTestCase):
         # rv = yield first_node.terminate_instance(instance_id)
 
 
-class NodeConnectionTestCase(test.TrialTestCase):
+class ComputeConnectionTestCase(test.TrialTestCase):
     def setUp(self):
         logging.getLogger().setLevel(logging.DEBUG)
-        super(NodeConnectionTestCase, self).setUp()
+        super(ComputeConnectionTestCase, self).setUp()
         self.flags(fake_libvirt=True,
                    fake_storage=True)
-        self.node = node.Node()
+        self.compute = service.ComputeService()
 
     def create_instance(self):
         instdir = model.InstanceDirectory()
@@ -80,48 +80,48 @@ class NodeConnectionTestCase(test.TrialTestCase):
     def test_run_describe_terminate(self):
         instance_id = self.create_instance()
 
-        rv = yield self.node.run_instance(instance_id)
+        rv = yield self.compute.run_instance(instance_id)
 
-        rv = yield self.node.describe_instances()
+        rv = yield self.compute.describe_instances()
         logging.info("Running instances: %s", rv)
         self.assertEqual(rv[instance_id].name, instance_id)
 
-        rv = yield self.node.terminate_instance(instance_id)
+        rv = yield self.compute.terminate_instance(instance_id)
 
-        rv = yield self.node.describe_instances()
+        rv = yield self.compute.describe_instances()
         logging.info("After terminating instances: %s", rv)
         self.assertEqual(rv, {})
 
     @defer.inlineCallbacks
     def test_reboot(self):
         instance_id = self.create_instance()
-        rv = yield self.node.run_instance(instance_id)
+        rv = yield self.compute.run_instance(instance_id)
 
-        rv = yield self.node.describe_instances()
+        rv = yield self.compute.describe_instances()
         self.assertEqual(rv[instance_id].name, instance_id)
 
-        yield self.node.reboot_instance(instance_id)
+        yield self.compute.reboot_instance(instance_id)
 
-        rv = yield self.node.describe_instances()
+        rv = yield self.compute.describe_instances()
         self.assertEqual(rv[instance_id].name, instance_id)
-        rv = yield self.node.terminate_instance(instance_id)
+        rv = yield self.compute.terminate_instance(instance_id)
 
     @defer.inlineCallbacks
     def test_console_output(self):
         instance_id = self.create_instance()
-        rv = yield self.node.run_instance(instance_id)
+        rv = yield self.compute.run_instance(instance_id)
 
-        console = yield self.node.get_console_output(instance_id)
+        console = yield self.compute.get_console_output(instance_id)
         self.assert_(console)
-        rv = yield self.node.terminate_instance(instance_id)
+        rv = yield self.compute.terminate_instance(instance_id)
 
     @defer.inlineCallbacks
     def test_run_instance_existing(self):
         instance_id = self.create_instance()
-        rv = yield self.node.run_instance(instance_id)
+        rv = yield self.compute.run_instance(instance_id)
 
-        rv = yield self.node.describe_instances()
+        rv = yield self.compute.describe_instances()
         self.assertEqual(rv[instance_id].name, instance_id)
 
-        self.assertRaises(exception.Error, self.node.run_instance, instance_id)
-        rv = yield self.node.terminate_instance(instance_id)
+        self.assertRaises(exception.Error, self.compute.run_instance, instance_id)
+        rv = yield self.compute.terminate_instance(instance_id)
diff --git a/nova/tests/future_unittest.py b/nova/tests/future_unittest.py
deleted file mode 100644
index da5470ffe..000000000
--- a/nova/tests/future_unittest.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import logging
-import mox
-import StringIO
-import time
-from tornado import ioloop
-from twisted.internet import defer
-import unittest
-from xml.etree import ElementTree
-
-from nova import cloud
-from nova import exception
-from nova import flags
-from nova import node
-from nova import rpc
-from nova import test
-
-
-FLAGS = flags.FLAGS
-
-
-class AdminTestCase(test.BaseTestCase):
-    def setUp(self):
-        super(AdminTestCase, self).setUp()
-        self.flags(fake_libvirt=True,
-                   fake_rabbit=True)
-
-        self.conn = rpc.Connection.instance()
-
-        logging.getLogger().setLevel(logging.INFO)
-
-        # set up our cloud
-        self.cloud = cloud.CloudController()
-        self.cloud_consumer = rpc.AdapterConsumer(connection=self.conn,
-                                                      topic=FLAGS.cloud_topic,
-                                                      proxy=self.cloud)
-        self.injected.append(self.cloud_consumer.attach_to_tornado(self.ioloop))
-        
-        # set up a node
-        self.node = node.Node()
-        self.node_consumer = rpc.AdapterConsumer(connection=self.conn,
-                                                     topic=FLAGS.compute_topic,
-                                                     proxy=self.node)
-        self.injected.append(self.node_consumer.attach_to_tornado(self.ioloop))
-
-    def test_flush_terminated(self):
-        # Launch an instance
-
-        # Wait until it's running
-
-        # Terminate it
-
-        # Wait until it's terminated
-
-        # Flush terminated nodes
-
-        # ASSERT that it's gone
-        pass
diff --git a/nova/tests/model_unittest.py b/nova/tests/model_unittest.py
index 1b94e5798..34d573854 100644
--- a/nova/tests/model_unittest.py
+++ b/nova/tests/model_unittest.py
@@ -25,7 +25,6 @@ from nova import flags
 from nova import test
 from nova import utils
 from nova.compute import model
-from nova.compute import node
 
 
 FLAGS = flags.FLAGS
diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py
index 8c6d866cd..c90120a6e 100644
--- a/nova/tests/objectstore_unittest.py
+++ b/nova/tests/objectstore_unittest.py
@@ -16,6 +16,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import boto
 import glob
 import hashlib
 import logging
@@ -27,8 +28,12 @@ from nova import flags
 from nova import objectstore
 from nova import test
 from nova.auth import manager
+from nova.objectstore.handler import S3
 from nova.exception import NotEmpty, NotFound, NotAuthorized
 
+from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+from twisted.internet import reactor, threads, defer
+from twisted.web import http, server
 
 FLAGS = flags.FLAGS
 
@@ -156,3 +161,107 @@ class ObjectStoreTestCase(test.BaseTestCase):
         self.context.user = self.um.get_user('user2')
         self.context.project = self.um.get_project('proj2')
         self.assertFalse(my_img.is_authorized(self.context))
+
+
+class TestHTTPChannel(http.HTTPChannel):
+    # Otherwise we end up with an unclean reactor
+    def checkPersistence(self, _, __):
+        return False
+
+
+class TestSite(server.Site):
+    protocol = TestHTTPChannel
+
+
+class S3APITestCase(test.TrialTestCase):
+    def setUp(self):
+        super(S3APITestCase, self).setUp()
+
+        FLAGS.auth_driver='nova.auth.ldapdriver.FakeLdapDriver',
+        FLAGS.buckets_path = os.path.join(oss_tempdir, 'buckets')
+
+        self.um = manager.AuthManager()
+        self.admin_user = self.um.create_user('admin', admin=True)
+        self.admin_project = self.um.create_project('admin', self.admin_user)
+
+        shutil.rmtree(FLAGS.buckets_path)
+        os.mkdir(FLAGS.buckets_path)
+
+        root = S3()
+        self.site = TestSite(root)
+        self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1')
+        self.tcp_port = self.listening_port.getHost().port
+
+
+        if not boto.config.has_section('Boto'):
+            boto.config.add_section('Boto')
+        boto.config.set('Boto', 'num_retries', '0')
+        self.conn = S3Connection(aws_access_key_id=self.admin_user.access,
+                                 aws_secret_access_key=self.admin_user.secret,
+                                 host='127.0.0.1',
+                                 port=self.tcp_port,
+                                 is_secure=False,
+                                 calling_format=OrdinaryCallingFormat())
+
+        # Don't attempt to reuse connections
+        def get_http_connection(host, is_secure):
+            return self.conn.new_http_connection(host, is_secure)
+        self.conn.get_http_connection = get_http_connection
+
+    def _ensure_empty_list(self, l):
+        self.assertEquals(len(l), 0, "List was not empty")
+        return True
+
+    def _ensure_only_bucket(self, l, name):
+        self.assertEquals(len(l), 1, "List didn't have exactly one element in it")
+        self.assertEquals(l[0].name, name, "Wrong name")
+
+    def test_000_list_buckets(self):
+        d = threads.deferToThread(self.conn.get_all_buckets)
+        d.addCallback(self._ensure_empty_list)
+        return d
+
+    def test_001_create_and_delete_bucket(self):
+        bucket_name = 'testbucket'
+
+        d = threads.deferToThread(self.conn.create_bucket, bucket_name)
+        d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets))
+
+        def ensure_only_bucket(l, name):
+            self.assertEquals(len(l), 1, "List didn't have exactly one element in it")
+            self.assertEquals(l[0].name, name, "Wrong name")
+        d.addCallback(ensure_only_bucket, bucket_name)
+
+        d.addCallback(lambda _:threads.deferToThread(self.conn.delete_bucket, bucket_name))
+        d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets))
+        d.addCallback(self._ensure_empty_list)
+        return d
+
+    def test_002_create_bucket_and_key_and_delete_key_again(self):
+        bucket_name = 'testbucket'
+        key_name = 'somekey'
+        key_contents = 'somekey'
+
+        d = threads.deferToThread(self.conn.create_bucket, bucket_name)
+        d.addCallback(lambda b:threads.deferToThread(b.new_key, key_name))
+        d.addCallback(lambda k:threads.deferToThread(k.set_contents_from_string, key_contents))
+        def ensure_key_contents(bucket_name, key_name, contents):
+            bucket = self.conn.get_bucket(bucket_name)
+            key = bucket.get_key(key_name)
+            self.assertEquals(key.get_contents_as_string(), contents,  "Bad contents")
+        d.addCallback(lambda _:threads.deferToThread(ensure_key_contents, bucket_name, key_name, key_contents))
+        def delete_key(bucket_name, key_name):
+            bucket = self.conn.get_bucket(bucket_name)
+            key = bucket.get_key(key_name)
+            key.delete()
+        d.addCallback(lambda _:threads.deferToThread(delete_key, bucket_name, key_name))
+        d.addCallback(lambda _:threads.deferToThread(self.conn.get_bucket, bucket_name))
+        d.addCallback(lambda b:threads.deferToThread(b.get_all_keys))
+        d.addCallback(self._ensure_empty_list)
+        return d
+
+    def tearDown(self):
+        self.um.delete_user('admin')
+        self.um.delete_project('admin')
+        return defer.DeferredList([defer.maybeDeferred(self.listening_port.stopListening)])
+        super(S3APITestCase, self).tearDown()
diff --git a/nova/tests/storage_unittest.py b/nova/tests/volume_unittest.py
index 60576d74f..62144269c 100644
--- a/nova/tests/storage_unittest.py
+++ b/nova/tests/volume_unittest.py
@@ -18,38 +18,38 @@
 
 import logging
 
+from nova import compute
 from nova import exception
 from nova import flags
 from nova import test
-from nova.compute import node
-from nova.volume import storage
+from nova.volume import service as volume_service
 
 
 FLAGS = flags.FLAGS
 
 
-class StorageTestCase(test.TrialTestCase):
+class VolumeTestCase(test.TrialTestCase):
     def setUp(self):
         logging.getLogger().setLevel(logging.DEBUG)
-        super(StorageTestCase, self).setUp()
-        self.mynode = node.Node()
-        self.mystorage = None
+        super(VolumeTestCase, self).setUp()
+        self.compute = compute.service.ComputeService()
+        self.volume = None
         self.flags(fake_libvirt=True,
                    fake_storage=True)
-        self.mystorage = storage.BlockStore()
+        self.volume = volume_service.VolumeService()
 
     def test_run_create_volume(self):
         vol_size = '0'
         user_id = 'fake'
         project_id = 'fake'
-        volume_id = self.mystorage.create_volume(vol_size, user_id, project_id)
+        volume_id = self.volume.create_volume(vol_size, user_id, project_id)
         # TODO(termie): get_volume returns differently than create_volume
         self.assertEqual(volume_id,
-                         storage.get_volume(volume_id)['volume_id'])
+                         volume_service.get_volume(volume_id)['volume_id'])
 
-        rv = self.mystorage.delete_volume(volume_id)
+        rv = self.volume.delete_volume(volume_id)
         self.assertRaises(exception.Error,
-                          storage.get_volume,
+                          volume_service.get_volume,
                           volume_id)
 
     def test_too_big_volume(self):
@@ -57,7 +57,7 @@ class StorageTestCase(test.TrialTestCase):
         user_id = 'fake'
         project_id = 'fake'
         self.assertRaises(TypeError,
-                          self.mystorage.create_volume,
+                          self.volume.create_volume,
                           vol_size, user_id, project_id)
 
     def test_too_many_volumes(self):
@@ -68,26 +68,26 @@ class StorageTestCase(test.TrialTestCase):
         total_slots = FLAGS.slots_per_shelf * num_shelves
         vols = []
         for i in xrange(total_slots):
-            vid = self.mystorage.create_volume(vol_size, user_id, project_id)
+            vid = self.volume.create_volume(vol_size, user_id, project_id)
             vols.append(vid)
-        self.assertRaises(storage.NoMoreVolumes,
-                          self.mystorage.create_volume,
+        self.assertRaises(volume_service.NoMoreVolumes,
+                          self.volume.create_volume,
                           vol_size, user_id, project_id)
         for id in vols:
-            self.mystorage.delete_volume(id)
+            self.volume.delete_volume(id)
 
     def test_run_attach_detach_volume(self):
-        # Create one volume and one node to test with
+        # Create one volume and one compute to test with
         instance_id = "storage-test"
         vol_size = "5"
         user_id = "fake"
         project_id = 'fake'
         mountpoint = "/dev/sdf"
-        volume_id = self.mystorage.create_volume(vol_size, user_id, project_id)
+        volume_id = self.volume.create_volume(vol_size, user_id, project_id)
 
-        volume_obj = storage.get_volume(volume_id)
+        volume_obj = volume_service.get_volume(volume_id)
         volume_obj.start_attach(instance_id, mountpoint)
-        rv = yield self.mynode.attach_volume(volume_id,
+        rv = yield self.compute.attach_volume(volume_id,
                                           instance_id,
                                           mountpoint)
         self.assertEqual(volume_obj['status'], "in-use")
@@ -96,16 +96,16 @@ class StorageTestCase(test.TrialTestCase):
         self.assertEqual(volume_obj['mountpoint'], mountpoint)
 
         self.assertRaises(exception.Error,
-                          self.mystorage.delete_volume,
+                          self.volume.delete_volume,
                           volume_id)
 
-        rv = yield self.mystorage.detach_volume(volume_id)
-        volume_obj = storage.get_volume(volume_id)
+        rv = yield self.volume.detach_volume(volume_id)
+        volume_obj = volume_service.get_volume(volume_id)
         self.assertEqual(volume_obj['status'], "available")
 
-        rv = self.mystorage.delete_volume(volume_id)
+        rv = self.volume.delete_volume(volume_id)
         self.assertRaises(exception.Error,
-                          storage.get_volume,
+                          volume_service.get_volume,
                           volume_id)
 
     def test_multi_node(self):
diff --git a/nova/twistd.py b/nova/twistd.py
index 32a46ce03..ecb6e2892 100644
--- a/nova/twistd.py
+++ b/nova/twistd.py
@@ -22,7 +22,6 @@ manage pid files and support syslogging.
 """
 
 import logging
-import logging.handlers
 import os
 import signal
 import sys
@@ -32,7 +31,6 @@ from twisted.python import log
 from twisted.python import reflect
 from twisted.python import runtime
 from twisted.python import usage
-import UserDict
 
 from nova import flags
 
@@ -161,6 +159,13 @@ def WrapTwistedOptions(wrapped):
             except (AttributeError, KeyError):
                 self._data[key] = value
 
+        def get(self, key, default):
+            key = key.replace('-', '_')
+            try:
+                return getattr(FLAGS, key)
+            except (AttributeError, KeyError):
+                self._data.get(key, default)
+
     return TwistedOptionsToFlags
 
 
@@ -209,9 +214,14 @@ def serve(filename):
         FLAGS.pidfile = '%s.pid' % name
     elif FLAGS.pidfile.endswith('twistd.pid'):
         FLAGS.pidfile = FLAGS.pidfile.replace('twistd.pid', '%s.pid' % name)
-
     if not FLAGS.logfile:
         FLAGS.logfile = '%s.log' % name
+    elif FLAGS.logfile.endswith('twistd.log'):
+        FLAGS.logfile = FLAGS.logfile.replace('twistd.log', '%s.log' % name)
+    if not FLAGS.prefix:
+        FLAGS.prefix = name
+    elif FLAGS.prefix.endswith('twisted'):
+        FLAGS.prefix = FLAGS.prefix.replace('twisted', name)
 
     action = 'start'
     if len(argv) > 1:
@@ -228,8 +238,16 @@ def serve(filename):
         print 'usage: %s [options] [start|stop|restart]' % argv[0]
         sys.exit(1)
 
-    formatter = logging.Formatter(
-        name + '(%(name)s): %(levelname)s %(message)s')
+    class NoNewlineFormatter(logging.Formatter):
+        """Strips newlines from default formatter"""
+        def format(self, record):
+            """Grabs default formatter's output and strips newlines"""
+            data = logging.Formatter.format(self, record)
+            return data.replace("\n", "--")
+
+    # NOTE(vish): syslog-ng doesn't handle newlines from trackbacks very well
+    formatter = NoNewlineFormatter(
+        '(%(name)s): %(levelname)s %(message)s')
     handler = logging.StreamHandler(log.StdioOnnaStick())
     handler.setFormatter(formatter)
     logging.getLogger().addHandler(handler)
@@ -239,11 +257,6 @@ def serve(filename):
     else:
         logging.getLogger().setLevel(logging.WARNING)
 
-    if FLAGS.syslog:
-        syslog = logging.handlers.SysLogHandler(address='/dev/log')
-        syslog.setFormatter(formatter)
-        logging.getLogger().addHandler(syslog)
-
     logging.debug("Full set of FLAGS:")
     for flag in FLAGS:
         logging.debug("%s : %s" % (flag, FLAGS.get(flag, None)))
diff --git a/nova/volume/storage.py b/nova/volume/service.py
index 121bc01e6..87a47f40a 100644
--- a/nova/volume/storage.py
+++ b/nova/volume/service.py
@@ -29,16 +29,15 @@ import shutil
 import socket
 import tempfile
 
-from twisted.application import service
 from twisted.internet import defer
 
 from nova import datastore
 from nova import exception
 from nova import flags
 from nova import process
+from nova import service
 from nova import utils
 from nova import validate
-from nova.compute import model
 
 
 FLAGS = flags.FLAGS
@@ -50,13 +49,13 @@ flags.DEFINE_string('aoe_eth_dev', 'eth0',
                     'Which device to export the volumes on')
 flags.DEFINE_string('storage_name',
                     socket.gethostname(),
-                    'name of this node')
+                    'name of this service')
 flags.DEFINE_integer('first_shelf_id',
                     utils.last_octet(utils.get_my_ip()) * 10,
-                    'AoE starting shelf_id for this node')
+                    'AoE starting shelf_id for this service')
 flags.DEFINE_integer('last_shelf_id',
                     utils.last_octet(utils.get_my_ip()) * 10 + 9,
-                    'AoE starting shelf_id for this node')
+                    'AoE starting shelf_id for this service')
 flags.DEFINE_string('aoe_export_dir',
                     '/var/lib/vblade-persist/vblades',
                     'AoE directory where exports are created')
@@ -65,7 +64,7 @@ flags.DEFINE_integer('slots_per_shelf',
                     'Number of AoE slots per shelf')
 flags.DEFINE_string('storage_availability_zone',
                     'nova',
-                    'availability zone of this node')
+                    'availability zone of this service')
 flags.DEFINE_boolean('fake_storage', False,
                      'Should we make real storage volumes to attach?')
 
@@ -82,14 +81,14 @@ def get_volume(volume_id):
         return volume_class(volume_id=volume_id)
     raise exception.Error("Volume does not exist")
 
-class BlockStore(object, service.Service):
+class VolumeService(service.Service):
     """
-    There is one BlockStore running on each volume node.
-    However, each BlockStore can report on the state of
+    There is one VolumeNode running on each host.
+    However, each VolumeNode can report on the state of
     *all* volumes in the cluster.
     """
     def __init__(self):
-        super(BlockStore, self).__init__()
+        super(VolumeService, self).__init__()
         self.volume_class = Volume
         if FLAGS.fake_storage:
             FLAGS.aoe_export_dir = tempfile.mkdtemp()
@@ -104,22 +103,6 @@ class BlockStore(object, service.Service):
             except Exception, err:
                 pass
 
-    @defer.inlineCallbacks
-    def report_state(self, nodename, daemon):
-        # TODO(termie): make this pattern be more elegant. -todd
-        try:
-            record = model.Daemon(nodename, daemon)
-            record.heartbeat()
-            if getattr(self, "model_disconnected", False):
-                self.model_disconnected = False
-                logging.error("Recovered model server connection!")
-
-        except model.ConnectionError, ex:
-            if not getattr(self, "model_disconnected", False):
-                self.model_disconnected = True
-                logging.exception("model server went away")
-        yield
-
     @validate.rangetest(size=(0, 1000))
     def create_volume(self, size, user_id, project_id):
         """
diff --git a/run_tests.py b/run_tests.py
index 0673388a9..5a8966f02 100644
--- a/run_tests.py
+++ b/run_tests.py
@@ -50,16 +50,16 @@ from nova import flags
 from nova import twistd
 
 from nova.tests.access_unittest import *
+from nova.tests.auth_unittest import *
 from nova.tests.api_unittest import *
 from nova.tests.cloud_unittest import *
+from nova.tests.compute_unittest import *
 from nova.tests.model_unittest import *
 from nova.tests.network_unittest import *
-from nova.tests.node_unittest import *
 from nova.tests.objectstore_unittest import *
 from nova.tests.process_unittest import *
-from nova.tests.storage_unittest import *
-from nova.tests.auth_unittest import *
 from nova.tests.validator_unittest import *
+from nova.tests.volume_unittest import *
 
 
 FLAGS = flags.FLAGS
diff --git a/setup.py b/setup.py
index eb42283ea..f9a616335 100644
--- a/setup.py
+++ b/setup.py
@@ -25,11 +25,10 @@ from setuptools import setup, find_packages
 srcdir = os.path.join(os.path.dirname(sys.argv[0]), 'src')
 
 setup(name='nova',
-      version='0.3.0',
-      description='None Other, Vaguely Awesome',
-      author='nova-core',
-      author_email='nova-core@googlegroups.com',
-      url='http://novacc.org/',
+      version='0.9.0',
+      description='cloud computing fabric controller',
+      author='OpenStack',
+      author_email='nova@lists.launchpad.net',
+      url='http://www.openstack.org/',
       packages = find_packages(),
-
      )
diff --git a/tools/install_venv.py b/tools/install_venv.py
new file mode 100644
index 000000000..0b35fc8e9
--- /dev/null
+++ b/tools/install_venv.py
@@ -0,0 +1,94 @@
+"""
+Installation script for Nova's development virtualenv
+"""
+
+import os
+import subprocess
+import sys
+
+
+ROOT = os.path.dirname(os.path.dirname(__file__))
+VENV = os.path.join(ROOT, '.nova-venv')
+PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires')
+TWISTED_NOVA='http://nova.openstack.org/Twisted-10.0.0Nova.tar.gz'
+
+
+def die(message, *args):
+  print >>sys.stderr, message % args
+  sys.exit(1)
+
+
+def run_command(cmd, redirect_output=True, error_ok=False):
+  # Useful for debugging:
+  #print >>sys.stderr, ' '.join(cmd)
+  if redirect_output:
+    stdout = subprocess.PIPE
+  else:
+    stdout = None
+
+  proc = subprocess.Popen(cmd, stdout=stdout)
+  output = proc.communicate()[0]
+  if not error_ok and proc.returncode != 0:
+    die('Command "%s" failed.\n%s', ' '.join(cmd), output)
+  return output
+
+
+def check_dependencies():
+  """Make sure pip and virtualenv are on the path."""
+  print 'Checking for pip...',
+  if not run_command(['which', 'pip']).strip():
+    die('ERROR: pip not found.\n\nNova development requires pip,'
+        ' please install it using your favorite package management tool')
+  print 'done.'
+
+  print 'Checking for virtualenv...',
+  if not run_command(['which', 'virtualenv']).strip():
+    die('ERROR: virtualenv not found.\n\nNova development requires virtualenv,'
+        ' please install it using your favorite package management tool')
+  print 'done.'
+
+
+def create_virtualenv(venv=VENV):
+  print 'Creating venv...',
+  run_command(['virtualenv', '-q', '--no-site-packages', VENV])
+  print 'done.'
+
+
+def install_dependencies(venv=VENV):
+  print 'Installing dependencies with pip (this can take a while)...'
+  run_command(['pip', 'install', '-E', venv, '-r', PIP_REQUIRES],
+              redirect_output=False)
+  run_command(['pip', 'install', '-E', venv, TWISTED_NOVA],
+              redirect_output=False)
+
+
+def print_help():
+  help = """
+ Nova development environment setup is complete.
+
+ Nova development uses virtualenv to track and manage Python dependencies
+ while in development and testing.
+
+ To activate the Nova virtualenv for the extent of your current shell session
+ you can run:
+
+ $ source .nova-venv/bin/activate 
+
+ Or, if you prefer, you can run commands in the virtualenv on a case by case
+ basis by running:
+
+ $ tools/with_venv.sh <your command>
+
+ Also, make test will automatically use the virtualenv.
+  """
+  print help
+
+
+def main(argv):
+  check_dependencies()
+  create_virtualenv()
+  install_dependencies()
+  print_help()
+
+if __name__ == '__main__':
+  main(sys.argv)
diff --git a/tools/pip-requires b/tools/pip-requires
new file mode 100644
index 000000000..4eb47ca2b
--- /dev/null
+++ b/tools/pip-requires
@@ -0,0 +1,15 @@
+IPy==0.70
+M2Crypto==0.20.2
+amqplib==0.6.1
+anyjson==0.2.4
+boto==2.0b1
+carrot==0.10.5
+lockfile==0.8
+python-daemon==1.5.5
+python-gflags==1.3
+redis==2.0.0
+tornado==1.0
+wsgiref==0.1.2
+zope.interface==3.6.1
+mox==0.5.0
+-f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz
diff --git a/tools/with_venv.sh b/tools/with_venv.sh
new file mode 100755
index 000000000..99d1ac18f
--- /dev/null
+++ b/tools/with_venv.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+TOOLS=`dirname $0`
+VENV=$TOOLS/../.nova-venv
+source $VENV/bin/activate && $@