From b3feee7425334f4f2369edc100ed4422e60e2288 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 19:58:12 +0000 Subject: remove calls to runthis from node --- nova/compute/node.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index 3abd20120..3e39e65fc 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -223,16 +223,20 @@ class Node(object, service.Service): volume_id = None, mountpoint = None): volume = storage.get_volume(volume_id) yield self._init_aoe() - yield utils.runthis("Attached Volume: %s", - "sudo virsh attach-disk %s /dev/etherd/%s %s" - % (instance_id, volume['aoe_device'], mountpoint.split("/")[-1])) + yield process.SharedPool().simple_execute( + "sudo virsh attach-disk %s /dev/etherd/%s %s" % + (instance_id, + volume['aoe_device'], + mountpoint.rpartition('/dev/')[2])) volume.finish_attach() defer.returnValue(True) + @defer.inlineCallbacks def _init_aoe(self): - utils.runthis("Doin an AoE discover, returns %s", "sudo aoe-discover") - utils.runthis("Doin an AoE stat, returns %s", "sudo aoe-stat") + yield process.SharedPool().simple_execute("sudo aoe-discover") + yield process.SharedPool().simple_execute("sudo aoe-stat") + @defer.inlineCallbacks @exception.wrap_exception def detach_volume(self, instance_id, volume_id): """ detach a volume from an instance """ @@ -240,10 +244,10 @@ class Node(object, service.Service): # name without the leading /dev/ volume = storage.get_volume(volume_id) target = volume['mountpoint'].rpartition('/dev/')[2] - utils.runthis("Detached Volume: %s", "sudo virsh detach-disk %s %s " - % (instance_id, target)) + yield process.SharedPool().simple_execute( + "sudo virsh detach-disk %s %s " % (instance_id, target)) volume.finish_detach() - return defer.succeed(True) + defer.returnValue(True) class Group(object): -- cgit From 049f27d00900f4b6e810d35f8e0e1ec3520d053b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 19:58:50 +0000 Subject: change volume code to use twisted --- nova/volume/storage.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/nova/volume/storage.py b/nova/volume/storage.py index de20f30b5..305ef527a 100644 --- a/nova/volume/storage.py +++ b/nova/volume/storage.py @@ -35,6 +35,7 @@ from twisted.internet import defer from nova import datastore from nova import exception from nova import flags +from nova import process from nova import utils from nova import validate @@ -143,17 +144,24 @@ class BlockStore(object): datastore.Redis.instance().srem('volumes:%s' % (FLAGS.storage_name), vol['volume_id']) return True + @defer.inlineCallbacks def _restart_exports(self): if FLAGS.fake_storage: return - utils.runthis("Setting exports to auto: %s", "sudo vblade-persist auto all") - utils.runthis("Starting all exports: %s", "sudo vblade-persist start all") + yield process.SharedPool().simple_execute( + "sudo vblade-persist auto all") + yield process.SharedPool().simple_execute( + "sudo vblade-persist start all") + @defer.inlineCallbacks def _init_volume_group(self): if FLAGS.fake_storage: return - utils.runthis("PVCreate returned: %s", "sudo pvcreate %s" % (FLAGS.storage_dev)) - utils.runthis("VGCreate returned: %s", "sudo vgcreate %s %s" % (FLAGS.volume_group, FLAGS.storage_dev)) + yield process.SharedPool().simple_execute( + "sudo pvcreate %s" % (FLAGS.storage_dev)) + yield process.SharedPool().simple_execute( + "sudo vgcreate %s %s" % (FLAGS.volume_group, + FLAGS.storage_dev)) class Volume(datastore.BasicModel): @@ -227,15 +235,22 @@ class Volume(datastore.BasicModel): self._delete_lv() super(Volume, self).destroy() + @defer.inlineCallbacks def create_lv(self): if str(self['size']) == '0': sizestr = '100M' else: sizestr = '%sG' % self['size'] - utils.runthis("Creating LV: %s", "sudo lvcreate -L %s -n %s %s" % (sizestr, self['volume_id'], FLAGS.volume_group)) + yield process.SharedPool().simple_execute( + "sudo lvcreate -L %s -n %s %s" % (sizestr, + self['volume_id'], + FLAGS.volume_group)) + @defer.inlineCallbacks def _delete_lv(self): - utils.runthis("Removing LV: %s", "sudo lvremove -f %s/%s" % (FLAGS.volume_group, self['volume_id'])) + yield process.SharedPool().simple_execute( + "sudo lvremove -f %s/%s" % (FLAGS.volume_group, + self['volume_id'])) def _setup_export(self): (shelf_id, blade_id) = get_next_aoe_numbers() @@ -245,8 +260,9 @@ class Volume(datastore.BasicModel): self.save() self._exec_export() + @defer.inlineCallbacks def _exec_export(self): - utils.runthis("Creating AOE export: %s", + yield process.SharedPool().simple_execute( "sudo vblade-persist setup %s %s %s /dev/%s/%s" % (self['shelf_id'], self['blade_id'], @@ -254,9 +270,14 @@ class Volume(datastore.BasicModel): FLAGS.volume_group, self['volume_id'])) + @defer.inlineCallbacks def _remove_export(self): - utils.runthis("Stopped AOE export: %s", "sudo vblade-persist stop %s %s" % (self['shelf_id'], self['blade_id'])) - utils.runthis("Destroyed AOE export: %s", "sudo vblade-persist destroy %s %s" % (self['shelf_id'], self['blade_id'])) + yield process.SharedPool().simple_execute( + "sudo vblade-persist stop %s %s" % (self['shelf_id'], + self['blade_id'])) + yield process.SharedPool().simple_execute( + "sudo vblade-persist destroy %s %s" % (self['shelf_id'], + self['blade_id'])) class FakeVolume(Volume): -- cgit From 382381f74ca3423958add26b2578c4e77282a9a0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 20:50:08 +0000 Subject: simplify call to simple_execute --- nova/compute/node.py | 8 ++++---- nova/volume/storage.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index 8874ef17e..7cae86d02 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -223,7 +223,7 @@ class Node(object, service.Service): volume_id = None, mountpoint = None): volume = storage.get_volume(volume_id) yield self._init_aoe() - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo virsh attach-disk %s /dev/etherd/%s %s" % (instance_id, volume['aoe_device'], @@ -233,8 +233,8 @@ class Node(object, service.Service): @defer.inlineCallbacks def _init_aoe(self): - yield process.SharedPool().simple_execute("sudo aoe-discover") - yield process.SharedPool().simple_execute("sudo aoe-stat") + yield process.simple_execute("sudo aoe-discover") + yield process.simple_execute("sudo aoe-stat") @defer.inlineCallbacks @exception.wrap_exception @@ -244,7 +244,7 @@ class Node(object, service.Service): # name without the leading /dev/ volume = storage.get_volume(volume_id) target = volume['mountpoint'].rpartition('/dev/')[2] - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo virsh detach-disk %s %s " % (instance_id, target)) volume.finish_detach() defer.returnValue(True) diff --git a/nova/volume/storage.py b/nova/volume/storage.py index 305ef527a..5424b092f 100644 --- a/nova/volume/storage.py +++ b/nova/volume/storage.py @@ -148,18 +148,18 @@ class BlockStore(object): def _restart_exports(self): if FLAGS.fake_storage: return - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo vblade-persist auto all") - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo vblade-persist start all") @defer.inlineCallbacks def _init_volume_group(self): if FLAGS.fake_storage: return - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo pvcreate %s" % (FLAGS.storage_dev)) - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo vgcreate %s %s" % (FLAGS.volume_group, FLAGS.storage_dev)) @@ -241,14 +241,14 @@ class Volume(datastore.BasicModel): sizestr = '100M' else: sizestr = '%sG' % self['size'] - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo lvcreate -L %s -n %s %s" % (sizestr, self['volume_id'], FLAGS.volume_group)) @defer.inlineCallbacks def _delete_lv(self): - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo lvremove -f %s/%s" % (FLAGS.volume_group, self['volume_id'])) @@ -262,7 +262,7 @@ class Volume(datastore.BasicModel): @defer.inlineCallbacks def _exec_export(self): - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo vblade-persist setup %s %s %s /dev/%s/%s" % (self['shelf_id'], self['blade_id'], @@ -272,10 +272,10 @@ class Volume(datastore.BasicModel): @defer.inlineCallbacks def _remove_export(self): - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo vblade-persist stop %s %s" % (self['shelf_id'], self['blade_id'])) - yield process.SharedPool().simple_execute( + yield process.simple_execute( "sudo vblade-persist destroy %s %s" % (self['shelf_id'], self['blade_id'])) -- cgit From 73af1a84eb682423bf40323387d739778765e138 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 21:52:10 +0000 Subject: make nova-volume start with twisteds daemonize stuff --- bin/nova-compute | 9 +++----- bin/nova-volume | 61 +++++++++++++++++++++++++++++++++++--------------- nova/volume/storage.py | 25 ++++++++++++++++----- 3 files changed, 65 insertions(+), 30 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index 5635efbaf..4b559beb4 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -33,9 +33,6 @@ NOVA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'nova') if os.path.exists(NOVA_PATH): sys.path.insert(0, os.path.dirname(NOVA_PATH)) - -from carrot import connection -from carrot import messaging from twisted.internet import task from twisted.application import service @@ -50,8 +47,8 @@ FLAGS = flags.FLAGS # context when the twistd.serve() call is made below so any # flags we define here will have to be conditionally defined, # flags defined by imported modules are safe. -if 'node_report_state_interval' not in FLAGS: - flags.DEFINE_integer('node_report_state_interval', 10, +if 'compute_report_state_interval' not in FLAGS: + flags.DEFINE_integer('compute_report_state_interval', 10, 'seconds between nodes reporting state to cloud', lower_bound=1) logging.getLogger().setLevel(logging.DEBUG) @@ -75,7 +72,7 @@ def main(): bin_name = os.path.basename(__file__) pulse = task.LoopingCall(n.report_state, FLAGS.node_name, bin_name) - pulse.start(interval=FLAGS.node_report_state_interval, now=False) + pulse.start(interval=FLAGS.compute_report_state_interval, now=False) injected = consumer_all.attach_to_twisted() injected = consumer_node.attach_to_twisted() diff --git a/bin/nova-volume b/bin/nova-volume index df9fb5c7a..64b726627 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -22,22 +22,37 @@ """ import logging -from tornado import ioloop +import os +import sys + +# NOTE(termie): kludge so that we can run this from the bin directory in the +# checkout without having to screw with paths +NOVA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'nova') +if os.path.exists(NOVA_PATH): + sys.path.insert(0, os.path.dirname(NOVA_PATH)) + +from twisted.internet import task +from twisted.application import service from nova import flags from nova import rpc -from nova import server -from nova import utils +from nova import twistd from nova.volume import storage FLAGS = flags.FLAGS -flags.DEFINE_integer('storage_report_state_interval', 10, - 'seconds between broadcasting state to cloud', - lower_bound=1) +# NOTE(termie): This file will necessarily be re-imported under different +# context when the twistd.serve() call is made below so any +# flags we define here will have to be conditionally defined, +# flags defined by imported modules are safe. +if 'volume_report_state_interval' not in FLAGS: + flags.DEFINE_integer('volume_report_state_interval', 10, + 'seconds between nodes reporting state to cloud', + lower_bound=1) -def main(argv): +def main(): + logging.warn('Starting volume node') bs = storage.BlockStore() conn = rpc.Connection.instance() @@ -51,19 +66,29 @@ def main(argv): topic='%s.%s' % (FLAGS.storage_topic, FLAGS.node_name), proxy=bs) - io_inst = ioloop.IOLoop.instance() - scheduler = ioloop.PeriodicCallback( - lambda: bs.report_state(), - FLAGS.storage_report_state_interval * 1000, - io_loop=io_inst) + bin_name = os.path.basename(__file__) + pulse = task.LoopingCall(bs.report_state, FLAGS.node_name, bin_name) + pulse.start(interval=FLAGS.volume_report_state_interval, now=False) + + injected = consumer_all.attach_to_twisted() + injected = consumer_node.attach_to_twisted() - injected = consumer_all.attachToTornado(io_inst) - injected = consumer_node.attachToTornado(io_inst) - scheduler.start() - io_inst.start() + # This is the parent service that twistd will be looking for when it + # parses this file, return it so that we can get it into globals below + application = service.Application(bin_name) + bs.setServiceParent(application) + return application +# NOTE(termie): When this script is executed from the commandline what it will +# actually do is tell the twistd application runner that it +# should run this file as a twistd application (see below). if __name__ == '__main__': - utils.default_flagfile() - server.serve('nova-volume', main) + twistd.serve(__file__) +# NOTE(termie): When this script is loaded by the twistd application runner +# this code path will be executed and twistd will expect a +# variable named 'application' to be available, it will then +# handle starting it and stopping it. +if __name__ == '__builtin__': + application = main() diff --git a/nova/volume/storage.py b/nova/volume/storage.py index 5424b092f..121bc01e6 100644 --- a/nova/volume/storage.py +++ b/nova/volume/storage.py @@ -28,8 +28,8 @@ import os import shutil import socket import tempfile -import time -from tornado import ioloop + +from twisted.application import service from twisted.internet import defer from nova import datastore @@ -38,6 +38,7 @@ from nova import flags from nova import process from nova import utils from nova import validate +from nova.compute import model FLAGS = flags.FLAGS @@ -81,7 +82,7 @@ def get_volume(volume_id): return volume_class(volume_id=volume_id) raise exception.Error("Volume does not exist") -class BlockStore(object): +class BlockStore(object, service.Service): """ There is one BlockStore running on each volume node. However, each BlockStore can report on the state of @@ -103,9 +104,21 @@ class BlockStore(object): except Exception, err: pass - def report_state(self): - #TODO: aggregate the state of the system - pass + @defer.inlineCallbacks + def report_state(self, nodename, daemon): + # TODO(termie): make this pattern be more elegant. -todd + try: + record = model.Daemon(nodename, daemon) + record.heartbeat() + if getattr(self, "model_disconnected", False): + self.model_disconnected = False + logging.error("Recovered model server connection!") + + except model.ConnectionError, ex: + if not getattr(self, "model_disconnected", False): + self.model_disconnected = True + logging.exception("model server went away") + yield @validate.rangetest(size=(0, 1000)) def create_volume(self, size, user_id, project_id): -- cgit From 0cf417c9e9c48352ea11c2333794545467585542 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 21:53:17 +0000 Subject: fix conf file to no longer have daemonize=1 because twistd daemonizes by default --- debian/nova-volume.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/nova-volume.conf b/debian/nova-volume.conf index af3271d3b..820c92918 100644 --- a/debian/nova-volume.conf +++ b/debian/nova-volume.conf @@ -1,4 +1,3 @@ ---daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys --datastore_path=/var/lib/nova/keeper -- cgit From fd25c2699867e16908aaadc3380236f84cc3cc5a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 17:05:02 -0500 Subject: remove spaces from export statements in scripts relating to certs --- CA/geninter.sh | 2 +- nova/cloudpipe/bootscript.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CA/geninter.sh b/CA/geninter.sh index 6c0528d1b..46f8f79ad 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -17,7 +17,7 @@ # under the License. # ARG is the id of the user -export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-intCA-$3 +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$3" mkdir INTER/$1 cd INTER/$1 cp ../../openssl.cnf.tmpl openssl.cnf diff --git a/nova/cloudpipe/bootscript.sh b/nova/cloudpipe/bootscript.sh index 43fc2ecab..82ec2012a 100755 --- a/nova/cloudpipe/bootscript.sh +++ b/nova/cloudpipe/bootscript.sh @@ -24,7 +24,7 @@ export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $1}'` export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $1}'` export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` -export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-vpn-$VPN_IP +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-vpn-$VPN_IP" DHCP_LOWER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 10 }'` DHCP_UPPER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 1 }'` -- cgit From 0506cce7d934ad093f5808606627aa19f43428ef Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 17:55:38 -0500 Subject: Fixed the broken reference to --- CA/geninter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CA/geninter.sh b/CA/geninter.sh index 46f8f79ad..7d6c280d5 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -17,7 +17,7 @@ # under the License. # ARG is the id of the user -export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$3" +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$1" mkdir INTER/$1 cd INTER/$1 cp ../../openssl.cnf.tmpl openssl.cnf -- cgit From 1b6efa80e19a60d71a762683fa1edee02645355c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 22:28:23 -0500 Subject: fix for describe addresses showing everyone's public ips --- nova/endpoint/cloud.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 3b7b4804b..4fa9b5afd 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -453,21 +453,21 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - # TODO(vish): move authorization checking into network.py for address in self.network.host_objs: - #logging.debug(address_record) - address_rv = { - 'public_ip': address['address'], - 'instance_id' : address.get('instance_id', 'free') - } - if context.user.is_admin(): - address_rv['instance_id'] = "%s (%s, %s)" % ( - address['instance_id'], - address['user_id'], - address['project_id'], - ) + # TODO(vish): implement a by_project iterator for addresses + if (context.user.is_admin() or + address['project_id'] == self.project.id): + address_rv = { + 'public_ip': address['address'], + 'instance_id' : address.get('instance_id', 'free') + } + if context.user.is_admin(): + address_rv['instance_id'] = "%s (%s, %s)" % ( + address['instance_id'], + address['user_id'], + address['project_id'], + ) addresses.append(address_rv) - # logging.debug(addresses) return {'addressesSet': addresses} @rbac.allow('netadmin') -- cgit From 2d49a870fe89b3266f908a5711a5d412fa6d7a19 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 21 Jul 2010 08:46:53 -0500 Subject: remove all of the unused saved return values from attach_to_twisted --- bin/nova-compute | 4 ++-- bin/nova-volume | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index 4b559beb4..49710e1b3 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -74,8 +74,8 @@ def main(): pulse = task.LoopingCall(n.report_state, FLAGS.node_name, bin_name) pulse.start(interval=FLAGS.compute_report_state_interval, now=False) - injected = consumer_all.attach_to_twisted() - injected = consumer_node.attach_to_twisted() + consumer_all.attach_to_twisted() + consumer_node.attach_to_twisted() # This is the parent service that twistd will be looking for when it # parses this file, return it so that we can get it into globals below diff --git a/bin/nova-volume b/bin/nova-volume index 64b726627..7d4b65205 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -70,8 +70,8 @@ def main(): pulse = task.LoopingCall(bs.report_state, FLAGS.node_name, bin_name) pulse.start(interval=FLAGS.volume_report_state_interval, now=False) - injected = consumer_all.attach_to_twisted() - injected = consumer_node.attach_to_twisted() + consumer_all.attach_to_twisted() + consumer_node.attach_to_twisted() # This is the parent service that twistd will be looking for when it # parses this file, return it so that we can get it into globals below -- cgit From 302afc13da7a83dcdf8bde0d6370b675c9b14218 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 21 Jul 2010 14:35:39 -0400 Subject: Fixes up Bucket to throw proper NotFound and NotEmpty exceptions in constructor and delete() method, and fixes up objectstore_unittest to properly use assertRaises() to check for proper exceptions and remove the assert_ calls. --- nova/exception.py | 3 +++ nova/objectstore/bucket.py | 4 ++-- nova/tests/objectstore_unittest.py | 35 ++++++++++++----------------------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index bda002d1e..2108123de 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -44,6 +44,9 @@ class Duplicate(Error): class NotAuthorized(Error): pass +class NotEmpty(Error): + pass + def wrap_exception(f): def _wrap(*args, **kw): try: diff --git a/nova/objectstore/bucket.py b/nova/objectstore/bucket.py index 090ef4e61..b42a96233 100644 --- a/nova/objectstore/bucket.py +++ b/nova/objectstore/bucket.py @@ -107,7 +107,7 @@ class Bucket(object): try: return context.user.is_admin() or self.owner_id == context.project.id except Exception, e: - pass + return False def list_keys(self, prefix='', marker=None, max_keys=1000, terse=False): object_names = [] @@ -161,7 +161,7 @@ class Bucket(object): def delete(self): if len(os.listdir(self.path)) > 0: - raise exception.NotAuthorized() + raise exception.NotEmpty() os.rmdir(self.path) os.remove(self.path+'.json') diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index f47ca7f00..c0b6e97a5 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -23,6 +23,7 @@ import os import shutil import tempfile +from nova.exception import NotEmpty, NotFound, NotAuthorized from nova import flags from nova import objectstore from nova import test @@ -96,49 +97,37 @@ class ObjectStoreTestCase(test.BaseTestCase): # another user is not authorized self.context.user = self.um.get_user('user2') self.context.project = self.um.get_project('proj2') - self.assert_(bucket.is_authorized(self.context) == False) + self.assertFalse(bucket.is_authorized(self.context)) # admin is authorized to use bucket self.context.user = self.um.get_user('admin_user') self.context.project = None - self.assert_(bucket.is_authorized(self.context)) + self.assertTrue(bucket.is_authorized(self.context)) # new buckets are empty - self.assert_(bucket.list_keys()['Contents'] == []) + self.assertTrue(bucket.list_keys()['Contents'] == []) # storing keys works bucket['foo'] = "bar" - self.assert_(len(bucket.list_keys()['Contents']) == 1) + self.assertEquals(len(bucket.list_keys()['Contents']), 1) - self.assert_(bucket['foo'].read() == 'bar') + self.assertEquals(bucket['foo'].read(), 'bar') # md5 of key works - self.assert_(bucket['foo'].md5 == hashlib.md5('bar').hexdigest()) - - # deleting non-empty bucket throws exception - exception = False - try: - bucket.delete() - except: - exception = True + self.assertEquals(bucket['foo'].md5, hashlib.md5('bar').hexdigest()) - self.assert_(exception) + # deleting non-empty bucket should throw a NotEmpty exception + self.assertRaises(NotEmpty, bucket.delete) # deleting key del bucket['foo'] - # deleting empty button + # deleting empty bucket bucket.delete() # accessing deleted bucket throws exception - exception = False - try: - objectstore.bucket.Bucket('new_bucket') - except: - exception = True - - self.assert_(exception) + self.assertRaises(NotFound, objectstore.bucket.Bucket, 'new_bucket') def test_images(self): self.context.user = self.um.get_user('user1') @@ -167,7 +156,7 @@ class ObjectStoreTestCase(test.BaseTestCase): # verify image permissions self.context.user = self.um.get_user('user2') self.context.project = self.um.get_project('proj2') - self.assert_(my_img.is_authorized(self.context) == False) + self.assertFalse(my_img.is_authorized(self.context)) # class ApiObjectStoreTestCase(test.BaseTestCase): # def setUp(self): -- cgit From 2c7e49ddeba2e9015c541712e5c52e0d902804b0 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 21 Jul 2010 15:28:43 -0400 Subject: reorder import statement and remove commented-out test case that is the same as api_unittest in objectstore_unittest --- nova/tests/objectstore_unittest.py | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index c0b6e97a5..8ae1f6e78 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -23,11 +23,11 @@ import os import shutil import tempfile -from nova.exception import NotEmpty, NotFound, NotAuthorized from nova import flags from nova import objectstore from nova import test from nova.auth import users +from nova.exception import NotEmpty, NotFound, NotAuthorized FLAGS = flags.FLAGS @@ -157,36 +157,3 @@ class ObjectStoreTestCase(test.BaseTestCase): self.context.user = self.um.get_user('user2') self.context.project = self.um.get_project('proj2') self.assertFalse(my_img.is_authorized(self.context)) - -# class ApiObjectStoreTestCase(test.BaseTestCase): -# def setUp(self): -# super(ApiObjectStoreTestCase, self).setUp() -# FLAGS.fake_users = True -# FLAGS.buckets_path = os.path.join(tempdir, 'buckets') -# FLAGS.images_path = os.path.join(tempdir, 'images') -# FLAGS.ca_path = os.path.join(os.path.dirname(__file__), 'CA') -# -# self.users = users.UserManager.instance() -# self.app = handler.Application(self.users) -# -# self.host = '127.0.0.1' -# -# self.conn = boto.s3.connection.S3Connection( -# aws_access_key_id=user.access, -# aws_secret_access_key=user.secret, -# is_secure=False, -# calling_format=boto.s3.connection.OrdinaryCallingFormat(), -# port=FLAGS.s3_port, -# host=FLAGS.s3_host) -# -# self.mox.StubOutWithMock(self.ec2, 'new_http_connection') -# -# def tearDown(self): -# FLAGS.Reset() -# super(ApiObjectStoreTestCase, self).tearDown() -# -# def test_describe_instances(self): -# self.expect_http() -# self.mox.ReplayAll() -# -# self.assertEqual(self.ec2.get_all_instances(), []) -- cgit From 3142fec2c908689f02e4e24a5174a3dcf2260c4c Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 21 Jul 2010 18:20:04 -0700 Subject: Fixed bug 608505 - was freeing the wrong address (should have freed 'secondaddress', was freeing 'address') --- nova/tests/network_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index f3a5868d1..a1d1789e2 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -137,7 +137,7 @@ class NetworkTestCase(test.TrialTestCase): self.dnsmasq.release_ip(mac3, address3, hostname, net.bridge_name) net = network.get_project_network("project0", "default") rv = network.deallocate_ip(secondaddress) - self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) + self.dnsmasq.release_ip(mac, secondaddress, hostname, net.bridge_name) def test_release_before_deallocate(self): pass -- cgit From a0c29a822aaed756728f2619e176d8c54bb1d4e9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 23 Jul 2010 17:20:21 -0700 Subject: fixed bug where partition code was sometimes failing due to initial dd not being yielded properly --- nova/compute/disk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index 08a22556e..7e31498e5 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -64,8 +64,8 @@ def partition(infile, outfile, local_bytes=0, local_type='ext2', execute=None): last_sector = local_last # e # create an empty file - execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, last_sector, sector_size)) + yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (outfile, last_sector, sector_size)) # make mbr partition yield execute('parted --script %s mklabel msdos' % outfile) -- cgit From eb10c8f1ea41564b5ee2d19054eeb8b65bfc0b33 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 24 Jul 2010 16:22:17 -0700 Subject: Updated URLs in the README file to point to current locations. --- README | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README b/README index f7d21f400..851bca9db 100644 --- a/README +++ b/README @@ -6,15 +6,19 @@ The Choose Your Own Adventure README for Nova: To monitor it from a distance: follow @novacc on twitter -To tame it for use in your own cloud: read http://docs.novacc.org/getting.started.html +To tame it for use in your own cloud: read http://nova.openstack.org/getting.started.html -To study its anatomy: read http://docs.novacc.org/architecture.html +To study its anatomy: read http://nova.openstack.org/architecture.html -To disect it in detail: visit http://github.com/nova/cc +To disect it in detail: visit http://code.launchpad.net/nova -To taunt it with its weaknesses: use http://github.com/nova/cc/issues +To taunt it with its weaknesses: use http://bugs.launchpad.net/nova + +To watch it: http://hudson.openstack.org To hack at it: read HACKING -To watch it: http://test.novacc.org/waterfall +To laugh at its PEP8 problems: http://hudson.openstack.org/job/nova-pep8/violations + +To cry over its pylint problems: http://hudson.openstack.org/job/nova-pylint/violations -- cgit From 87e27afec0c7b683ee35f842abdaccea954f2fba Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 24 Jul 2010 18:06:22 -0700 Subject: Updated sphinx layout to a two-dir layout like swift. Updated a doc string to get rid of a Sphinx warning. --- doc/.gitignore | 1 + doc/build/.gitignore | 1 + doc/source/Makefile | 89 ++++++++++++++++ doc/source/_static/.gitignore | 0 doc/source/_templates/.gitignore | 0 doc/source/architecture.rst | 48 +++++++++ doc/source/auth.rst | 215 +++++++++++++++++++++++++++++++++++++++ doc/source/binaries.rst | 31 ++++++ doc/source/compute.rst | 74 ++++++++++++++ doc/source/conf.py | 202 ++++++++++++++++++++++++++++++++++++ doc/source/endpoint.rst | 91 +++++++++++++++++ doc/source/fakes.rst | 43 ++++++++ doc/source/getting.started.rst | 148 +++++++++++++++++++++++++++ doc/source/index.rst | 55 ++++++++++ doc/source/modules.rst | 34 +++++++ doc/source/network.rst | 88 ++++++++++++++++ doc/source/nova.rst | 91 +++++++++++++++++ doc/source/objectstore.rst | 66 ++++++++++++ doc/source/packages.rst | 29 ++++++ doc/source/storage.rst | 31 ++++++ doc/source/volume.rst | 45 ++++++++ docs/.gitignore | 1 - docs/Makefile | 89 ---------------- docs/_build/.gitignore | 1 - docs/_static/.gitignore | 0 docs/_templates/.gitignore | 0 docs/architecture.rst | 48 --------- docs/auth.rst | 215 --------------------------------------- docs/binaries.rst | 31 ------ docs/compute.rst | 74 -------------- docs/conf.py | 202 ------------------------------------ docs/endpoint.rst | 91 ----------------- docs/fakes.rst | 43 -------- docs/getting.started.rst | 148 --------------------------- docs/index.rst | 56 ---------- docs/modules.rst | 34 ------- docs/network.rst | 88 ---------------- docs/nova.rst | 91 ----------------- docs/objectstore.rst | 66 ------------ docs/packages.rst | 29 ------ docs/storage.rst | 31 ------ docs/volume.rst | 45 -------- nova/compute/disk.py | 3 +- setup.cfg | 4 +- 44 files changed, 1386 insertions(+), 1386 deletions(-) create mode 100644 doc/.gitignore create mode 100644 doc/build/.gitignore create mode 100644 doc/source/Makefile create mode 100644 doc/source/_static/.gitignore create mode 100644 doc/source/_templates/.gitignore create mode 100644 doc/source/architecture.rst create mode 100644 doc/source/auth.rst create mode 100644 doc/source/binaries.rst create mode 100644 doc/source/compute.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/endpoint.rst create mode 100644 doc/source/fakes.rst create mode 100644 doc/source/getting.started.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/modules.rst create mode 100644 doc/source/network.rst create mode 100644 doc/source/nova.rst create mode 100644 doc/source/objectstore.rst create mode 100644 doc/source/packages.rst create mode 100644 doc/source/storage.rst create mode 100644 doc/source/volume.rst delete mode 100644 docs/.gitignore delete mode 100644 docs/Makefile delete mode 100644 docs/_build/.gitignore delete mode 100644 docs/_static/.gitignore delete mode 100644 docs/_templates/.gitignore delete mode 100644 docs/architecture.rst delete mode 100644 docs/auth.rst delete mode 100644 docs/binaries.rst delete mode 100644 docs/compute.rst delete mode 100644 docs/conf.py delete mode 100644 docs/endpoint.rst delete mode 100644 docs/fakes.rst delete mode 100644 docs/getting.started.rst delete mode 100644 docs/index.rst delete mode 100644 docs/modules.rst delete mode 100644 docs/network.rst delete mode 100644 docs/nova.rst delete mode 100644 docs/objectstore.rst delete mode 100644 docs/packages.rst delete mode 100644 docs/storage.rst delete mode 100644 docs/volume.rst diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..88f9974bd --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +_build/* diff --git a/doc/build/.gitignore b/doc/build/.gitignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/doc/build/.gitignore @@ -0,0 +1 @@ +* diff --git a/doc/source/Makefile b/doc/source/Makefile new file mode 100644 index 000000000..b2f74e85a --- /dev/null +++ b/doc/source/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/nova.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/nova.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/_static/.gitignore b/doc/source/_static/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/doc/source/_templates/.gitignore b/doc/source/_templates/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst new file mode 100644 index 000000000..11813d2c8 --- /dev/null +++ b/doc/source/architecture.rst @@ -0,0 +1,48 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +nova System Architecture +======================== + +Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. + +In order to achieve shared-nothing with multiple copies of the same component (especially when the component is an API server that needs to reply with state information in a timely fashion), we need to keep all of our system state in a distributed data system. Updates to system state are written into this system, using atomic transactions when necessary. Requests for state are read out of this system. In limited cases, these read calls are memoized within controllers for short periods of time. (Such a limited case would be, for instance, the current list of system users.) + + +Components +---------- + +Below you will find a helpful explanation. + +:: + + [ User Manager ] ---- ( LDAP ) + | + | / [ Storage ] - ( ATAoE ) + [ API server ] -> [ Cloud ] < AMQP > + | \ [ Nodes ] - ( libvirt/kvm ) + < HTTP > + | + [ S3 ] + + +* API: receives http requests from boto, converts commands to/from API format, and sending requests to cloud controller +* Cloud Controller: global state of system, talks to ldap, s3, and node/storage workers through a queue +* Nodes: worker that spawns instances +* S3: tornado based http/s3 server +* User Manager: create/manage users, which are stored in ldap +* Network Controller: allocate and deallocate IPs and VLANs diff --git a/doc/source/auth.rst b/doc/source/auth.rst new file mode 100644 index 000000000..70aca704a --- /dev/null +++ b/doc/source/auth.rst @@ -0,0 +1,215 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Auth Documentation +================== + +Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles: + +Roles-Based Access Control of AWS-style APIs using SAML Assertions +“Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications” + +Introduction +-------------- + +We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles. +Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment. + +Relationship of US eAuth to RBAC +-------------------------------- + +Typical implementations of US eAuth authentication systems are structured as follows:: + + [ MS Active Directory or other federated LDAP user store ] + --> backends to… + [ SUN Identity Manager or other SAML Policy Controller ] + --> maps URLs to groups… + [ Apache Policy Agent in front of eAuth-secured Web Application ] + +In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion. + +Basic AWS API call structure +---------------------------- + +AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: + +* System User +* System Administrator +* Network Administrator +* Project Manager +* Cloud Administrator +* (IT-Sec?) + +There is an additional, conceptual end-user that may or may not have API access: + +* (EXTERNAL) End-user / Third-party User + +Basic operations are available to any System User: + +* Launch Instance +* Terminate Instance (their own) +* Create keypair +* Delete keypair +* Create, Upload, Delete: Buckets and Keys (Object Store) – their own +* Create, Attach, Delete Volume (Block Store) – their own + +System Administrators: + +* Register/Unregister Machine Image (project-wide) +* Change Machine Image properties (public / private) +* Request / Review CloudAudit Scans + +Network Administrator: + +* Change Firewall Rules, define Security Groups +* Allocate, Associate, Deassociate Public IP addresses + +Project Manager: + +* Launch and Terminate Instances (project-wide) +* CRUD of Object and Block store (project-wide) + +Cloud Administrator: + +* Register / Unregister Kernel and Ramdisk Images +* Register / Unregister Machine Image (any) + +Enhancements +------------ + +* SAML Token passing +* REST interfaces +* SOAP interfaces + +Wrapping the SAML token into the API calls. +Then store the UID (fetched via backchannel) into the instance metadata, providing end-to-end auditability of ownership and responsibility, without PII. + +CloudAudit APIs +--------------- + +* Request formats +* Response formats +* Stateless asynchronous queries + +CloudAudit queries may spawn long-running processes (similar to launching instances, etc.) They need to return a ReservationId in the same fashion, which can be returned in further queries for updates. +RBAC of CloudAudit API calls is critical, since detailed system information is a system vulnerability. + +Type declarations +--------------------- +* Data declarations – Volumes and Objects +* System declarations – Instances + +Existing API calls to launch instances specific a single, combined “type” flag. We propose to extend this with three additional type declarations, mapping to the “Confidentiality, Integrity, Availability” classifications of FIPS 199. An example API call would look like:: + + RunInstances type=m1.large number=1 secgroup=default key=mykey confidentiality=low integrity=low availability=low + +These additional parameters would also apply to creation of block storage volumes (along with the existing parameter of ‘size’), and creation of object storage ‘buckets’. (C.I.A. classifications on a bucket would be inherited by the keys within this bucket.) + +Request Brokering +----------------- + + * Cloud Interop + * IMF Registration / PubSub + * Digital C&A + +Establishing declarative semantics for individual API calls will allow the cloud environment to seamlessly proxy these API calls to external, third-party vendors – when the requested CIA levels match. + +See related work within the Infrastructure 2.0 working group for more information on how the IMF Metadata specification could be utilized to manage registration of these vendors and their C&A credentials. + +Dirty Cloud – Hybrid Data Centers +--------------------------------- + +* CloudAudit bridge interfaces +* Anything in the ARP table + +A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. + +This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment. + +Network discovery protocols (ARP, CDP) can be applied in this case, and existing protocols (SNMP location data, DNS LOC records) overloaded to provide CloudAudit information. + +The Details +----------- + + * Preliminary Roles Definitions + * Categorization of available API calls + * SAML assertion vocabulary + +System limits +------------- + +The following limits need to be defined and enforced: + +* Total number of instances allowed (user / project) +* Total number of instances, per instance type (user / project) +* Total number of volumes (user / project) +* Maximum size of volume +* Cumulative size of all volumes +* Total use of object storage (GB) +* Total number of Public IPs + + +Further Challenges +------------------ + * Prioritization of users / jobs in shared computing environments + * Incident response planning + * Limit launch of instances to specific security groups based on AMI + * Store AMIs in LDAP for added property control + + + +The :mod:`rbac` Module +-------------------------- + +.. automodule:: nova.auth.rbac + :members: + :undoc-members: + :show-inheritance: + +The :mod:`signer` Module +------------------------ + +.. automodule:: nova.auth.signer + :members: + :undoc-members: + :show-inheritance: + +The :mod:`users` Module +----------------------- + +.. automodule:: nova.auth.users + :members: + :undoc-members: + :show-inheritance: + +The :mod:`users_unittest` Module +-------------------------------- + +.. automodule:: nova.tests.users_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`access_unittest` Module +--------------------------------- + +.. automodule:: nova.tests.access_unittest + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/binaries.rst b/doc/source/binaries.rst new file mode 100644 index 000000000..90a9581f7 --- /dev/null +++ b/doc/source/binaries.rst @@ -0,0 +1,31 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Binaries +=============== + +* nova-api +* nova-compute +* nova-manage +* nova-objectstore +* nova-volume + +The configuration of these binaries relies on "flagfiles" using the google +gflags package. If present, the nova.conf file will be used as the flagfile +- otherwise, it must be specified on the command line:: + + $ python node_worker.py --flagfile flagfile diff --git a/doc/source/compute.rst b/doc/source/compute.rst new file mode 100644 index 000000000..5b08dbd5b --- /dev/null +++ b/doc/source/compute.rst @@ -0,0 +1,74 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Compute Documentation +===================== + +This page contains the Compute Package documentation. + + +The :mod:`disk` Module +---------------------- + +.. automodule:: nova.compute.disk + :members: + :undoc-members: + :show-inheritance: + +The :mod:`exception` Module +--------------------------- + +.. automodule:: nova.compute.exception + :members: + :undoc-members: + :show-inheritance: + +The :mod:`model` Module +------------------------- + +.. automodule:: nova.compute.model + :members: + :undoc-members: + :show-inheritance: + +The :mod:`network` Module +------------------------- + +.. automodule:: nova.compute.network + :members: + :undoc-members: + :show-inheritance: + +The :mod:`node` Module +---------------------- + +.. automodule:: nova.compute.node + :members: + :undoc-members: + :show-inheritance: + +RELATED TESTS +--------------- + +The :mod:`node_unittest` Module +------------------------------- + +.. automodule:: nova.tests.node_unittest + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..1c1ae7f48 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# +# nova documentation build configuration file, created by +# sphinx-quickstart on Sat May 1 15:17:47 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.append([os.path.abspath('../nova'), os.path.abspath('..'), os.path.abspath('../bin')]) + + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig'] +todo_include_todos = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'nova' +copyright = u'2010, United States Government as represented by the Administrator of the National Aeronautics and Space Administration.' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.42' +# The full version, including alpha/beta/rc tags. +release = '0.42' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +modindex_common_prefix = ['nova.'] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'novadoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Nova.tex', u'Nova Documentation', + u'Anso Labs, LLC', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'python': ('http://docs.python.org/', None), + 'swift': ('http://swift.openstack.org', None)} + diff --git a/doc/source/endpoint.rst b/doc/source/endpoint.rst new file mode 100644 index 000000000..399df4161 --- /dev/null +++ b/doc/source/endpoint.rst @@ -0,0 +1,91 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Endpoint Documentation +====================== + +This page contains the Endpoint Package documentation. + +The :mod:`admin` Module +----------------------- + +.. automodule:: nova.endpoint.admin + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api` Module +--------------------- + +.. automodule:: nova.endpoint.api + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud` Module +----------------------- + +.. automodule:: nova.endpoint.cloud + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +------------------------ + +.. automodule:: nova.endpoint.images + :members: + :undoc-members: + :show-inheritance: + + +RELATED TESTS +-------------- + +The :mod:`api_unittest` Module +------------------------------ + +.. automodule:: nova.tests.api_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api_integration` Module +--------------------------------- + +.. automodule:: nova.tests.api_integration + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud_unittest` Module +-------------------------------- + +.. automodule:: nova.tests.cloud_unittest + :members: + :undoc-members: + :show-inheritance: + +The :mod:`network_unittest` Module +---------------------------------- + +.. automodule:: nova.tests.network_unittest + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/fakes.rst b/doc/source/fakes.rst new file mode 100644 index 000000000..bea8bc4e9 --- /dev/null +++ b/doc/source/fakes.rst @@ -0,0 +1,43 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Fakes +========== + +The :mod:`fakevirt` Module +-------------------------- + +.. automodule:: nova.fakevirt + :members: + :undoc-members: + :show-inheritance: + +The :mod:`fakeldap` Module +-------------------------- + +.. automodule:: nova.auth.fakeldap + :members: + :undoc-members: + :show-inheritance: + +The :mod:`fakerabbit` Module +---------------------------- + +.. automodule:: nova.fakerabbit + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst new file mode 100644 index 000000000..3eadd0882 --- /dev/null +++ b/doc/source/getting.started.rst @@ -0,0 +1,148 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Getting Started with Nova +========================= + + +GOTTA HAVE A nova.pth file added or it WONT WORK (will write setup.py file soon) + +Create a file named nova.pth in your python libraries directory +(usually /usr/local/lib/python2.6/dist-packages) with a single line that points +to the directory where you checked out the source (that contains the nova/ +directory). + +DEPENDENCIES +------------ + +Related servers we rely on + +* RabbitMQ: messaging queue, used for all communication between components +* OpenLDAP: users, groups (maybe cut) +* ReDIS: Remote Dictionary Store (for fast, shared state data) +* nginx: HTTP server to handle serving large files (because Tornado can't) + +Python libraries we don't vendor + +* M2Crypto: python library interface for openssl +* curl + +Vendored python libaries (don't require any installation) + +* Tornado: scalable non blocking web server for api requests +* Twisted: just for the twisted.internet.defer package +* boto: python api for aws api +* IPy: library for managing ip addresses + +Recommended +----------------- + +* euca2ools: python implementation of aws ec2-tools and ami tools +* build tornado to use C module for evented section + + +Installation +-------------- +:: + + # system libraries and tools + apt-get install -y aoetools vlan curl + modprobe aoe + + # python libraries + apt-get install -y python-setuptools python-dev python-pycurl python-m2crypto + + # ON THE CLOUD CONTROLLER + apt-get install -y rabbitmq-server dnsmasq nginx + # build redis from 2.0.0-rc1 source + # setup ldap (slap.sh as root will remove ldap and reinstall it) + NOVA_PATH/nova/auth/slap.sh + /etc/init.d/rabbitmq-server start + + # ON VOLUME NODE: + apt-get install -y vblade-persist + + # ON THE COMPUTE NODE: + apt-get install -y python-libvirt + apt-get install -y kpartx kvm libvirt-bin + modprobe kvm + + # optional packages + apt-get install -y euca2ools + +Configuration +--------------- + +ON CLOUD CONTROLLER + +* Add yourself to the libvirtd group, log out, and log back in +* fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances +:: + + iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 + iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE + + +* Configure NginX proxy (/etc/nginx/sites-enabled/default) + +:: + + server { + listen 3333 default; + server-name localhost; + client_max_body_size 10m; + + access_log /var/log/nginx/localhost.access.log; + + location ~ /_images/.+ { + root NOVA_PATH/images; + rewrite ^/_images/(.*)$ /$1 break; + } + + location / { + proxy_pass http://localhost:3334/; + } + } + +ON VOLUME NODE + +* create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb) + +:: + + # This creates a 1GB file to create volumes out of + dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 + losetup --show -f MY_FILE_PATH + # replace loop0 below with whatever losetup returns + echo "--storage_dev=/dev/loop0" >> NOVA_PATH/bin/nova.conf + +Running +--------- + +Launch servers + +* rabbitmq +* redis +* slapd +* nginx + +Launch nova components + +* nova-api +* nova-compute +* nova-objectstore +* nova-volume diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..6627fe066 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,55 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Welcome to nova's documentation! +================================ + +Nova is a cloud computing fabric controller (the main part of an IaaS system) built to match the popular AWS EC2 and S3 APIs. +It is written in Python, using the Tornado and Twisted frameworks, and relies on the standard AMQP messaging protocol, +and the Redis distributed KVS. +Nova is intended to be easy to extend, and adapt. For example, it currently uses +an LDAP server for users and groups, but also includes a fake LDAP server, +that stores data in Redis. It has extensive test coverage, and uses the +Sphinx toolkit (the same as Python itself) for code and user documentation. +While Nova is currently in Beta use within several organizations, the codebase +is very much under active development - there are bugs! + +Contents: + +.. toctree:: + :maxdepth: 2 + + getting.started + architecture + network + storage + auth + compute + endpoint + nova + fakes + binaries + modules + packages + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/modules.rst b/doc/source/modules.rst new file mode 100644 index 000000000..82c61f008 --- /dev/null +++ b/doc/source/modules.rst @@ -0,0 +1,34 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Documentation +================== + +This page contains the Nova Modules documentation. + +Modules: +-------- + +.. toctree:: + :maxdepth: 4 + + auth + compute + endpoint + fakes + nova + volume diff --git a/doc/source/network.rst b/doc/source/network.rst new file mode 100644 index 000000000..357a0517f --- /dev/null +++ b/doc/source/network.rst @@ -0,0 +1,88 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +nova Networking +================ + +The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules. + +Components +---------- +There are several key components: + +* NetworkController (Manages address and vlan allocation) +* RoutingNode (NATs public IPs to private IPs, and enforces firewall rules) +* AddressingNode (runs DHCP services for private networks) +* BridgingNode (a subclass of the basic nova ComputeNode) +* TunnelingNode (provides VPN connectivity) + +Component Diagram +----------------- + +Overview:: + + (PUBLIC INTERNET) + | \ + / \ / \ + [RoutingNode] ... [RN] [TunnelingNode] ... [TN] + | \ / | | + | < AMQP > | | + [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] + \ | \ / + / \ / \ / \ / \ + [BridgingNode] ... [BridgingNode] + + + [NetworkController] ... [NetworkController] + \ / + < AMQP > + | + / \ + [CloudController]...[CloudController] + +While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). + +State Model +----------- +Network State consists of the following facts: + +* VLAN assignment (to a project) +* Private Subnet assignment (to a security group) in a VLAN +* Private IP assignments (to running instances) +* Public IP allocations (to a project) +* Public IP associations (to a private IP / running instance) + +While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. + +The Public Traffic Path +----------------------- + +Public Traffic:: + + (PUBLIC INTERNET) + | + <-- [RoutingNode] + | + [AddressingNode] --> | + ( VLAN ) + | <-- [BridgingNode] + | + + +The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. + +Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). diff --git a/doc/source/nova.rst b/doc/source/nova.rst new file mode 100644 index 000000000..4b9c44a5f --- /dev/null +++ b/doc/source/nova.rst @@ -0,0 +1,91 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +NOVA Libraries +=============== + +The :mod:`crypto` Module +------------------------ + +.. automodule:: nova.crypto + :members: + :undoc-members: + :show-inheritance: + +The :mod:`adminclient` Module +----------------------------- + +.. automodule:: nova.adminclient + :members: + :undoc-members: + :show-inheritance: + +The :mod:`datastore` Module +--------------------------- + +.. automodule:: nova.datastore + :members: + :undoc-members: + :show-inheritance: + +The :mod:`exception` Module +--------------------------- + +.. automodule:: nova.exception + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flags` Module +--------------------------- + +.. automodule:: nova.flags + :members: + :undoc-members: + :show-inheritance: + +The :mod:`rpc` Module +--------------------------- + +.. automodule:: nova.rpc + :members: + :undoc-members: + :show-inheritance: + +The :mod:`server` Module +--------------------------- + +.. automodule:: nova.server + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test` Module +--------------------------- + +.. automodule:: nova.test + :members: + :undoc-members: + :show-inheritance: + +The :mod:`utils` Module +--------------------------- + +.. automodule:: nova.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/objectstore.rst b/doc/source/objectstore.rst new file mode 100644 index 000000000..6b8d293f4 --- /dev/null +++ b/doc/source/objectstore.rst @@ -0,0 +1,66 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Objectstore Documentation +========================= + +This page contains the Objectstore Package documentation. + + +The :mod:`bucket` Module +------------------------ + +.. automodule:: nova.objectstore.bucket + :members: + :undoc-members: + :show-inheritance: + +The :mod:`handler` Module +------------------------- + +.. automodule:: nova.objectstore.handler + :members: + :undoc-members: + :show-inheritance: + +The :mod:`image` Module +----------------------- + +.. automodule:: nova.objectstore.image + :members: + :undoc-members: + :show-inheritance: + +The :mod:`stored` Module +------------------------ + +.. automodule:: nova.objectstore.stored + :members: + :undoc-members: + :show-inheritance: + +RELATED TESTS +------------- + +The :mod:`objectstore_unittest` Module +-------------------------------------- + +.. automodule:: nova.tests.objectstore_unittest + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/packages.rst b/doc/source/packages.rst new file mode 100644 index 000000000..6029ad7d7 --- /dev/null +++ b/doc/source/packages.rst @@ -0,0 +1,29 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +nova Packages & Dependencies +============================ + +Nova is being built on Ubuntu Lucid. + +The following packages are required: + + apt-get install python-ipy, python-libvirt, python-boto, python-pycurl, python-twisted, python-daemon, python-redis, python-carrot, python-lockfile + +In addition you need to install python: + + * python-gflags - http://code.google.com/p/python-gflags/ diff --git a/doc/source/storage.rst b/doc/source/storage.rst new file mode 100644 index 000000000..f77e5f0e5 --- /dev/null +++ b/doc/source/storage.rst @@ -0,0 +1,31 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Storage in the Nova Cloud +========================= + +There are three primary classes of storage in a nova cloud environment: + +* Ephemeral Storage (local disk within an instance) +* Volume Storage (network-attached FS) +* Object Storage (redundant KVS with locality and MR) + +.. toctree:: + :maxdepth: 2 + + volume + objectstore diff --git a/doc/source/volume.rst b/doc/source/volume.rst new file mode 100644 index 000000000..619968458 --- /dev/null +++ b/doc/source/volume.rst @@ -0,0 +1,45 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Volume Documentation +==================== + +Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. + +Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. + +AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.) + +The underlying volumes are LVM logical volumes, created on demand within a single large volume group. + + +The :mod:`storage` Module +------------------------- + +.. automodule:: nova.volume.storage + :members: + :undoc-members: + :show-inheritance: + +The :mod:`storage_unittest` Module +---------------------------------- + +.. automodule:: nova.tests.storage_unittest + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index 88f9974bd..000000000 --- a/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -_build/* diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index b2f74e85a..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,89 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/nova.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/nova.qhc" - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/_build/.gitignore b/docs/_build/.gitignore deleted file mode 100644 index 72e8ffc0d..000000000 --- a/docs/_build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/_templates/.gitignore b/docs/_templates/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/architecture.rst b/docs/architecture.rst deleted file mode 100644 index 11813d2c8..000000000 --- a/docs/architecture.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -nova System Architecture -======================== - -Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. - -In order to achieve shared-nothing with multiple copies of the same component (especially when the component is an API server that needs to reply with state information in a timely fashion), we need to keep all of our system state in a distributed data system. Updates to system state are written into this system, using atomic transactions when necessary. Requests for state are read out of this system. In limited cases, these read calls are memoized within controllers for short periods of time. (Such a limited case would be, for instance, the current list of system users.) - - -Components ----------- - -Below you will find a helpful explanation. - -:: - - [ User Manager ] ---- ( LDAP ) - | - | / [ Storage ] - ( ATAoE ) - [ API server ] -> [ Cloud ] < AMQP > - | \ [ Nodes ] - ( libvirt/kvm ) - < HTTP > - | - [ S3 ] - - -* API: receives http requests from boto, converts commands to/from API format, and sending requests to cloud controller -* Cloud Controller: global state of system, talks to ldap, s3, and node/storage workers through a queue -* Nodes: worker that spawns instances -* S3: tornado based http/s3 server -* User Manager: create/manage users, which are stored in ldap -* Network Controller: allocate and deallocate IPs and VLANs diff --git a/docs/auth.rst b/docs/auth.rst deleted file mode 100644 index 70aca704a..000000000 --- a/docs/auth.rst +++ /dev/null @@ -1,215 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Auth Documentation -================== - -Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles: - -Roles-Based Access Control of AWS-style APIs using SAML Assertions -“Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications” - -Introduction --------------- - -We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles. -Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment. - -Relationship of US eAuth to RBAC --------------------------------- - -Typical implementations of US eAuth authentication systems are structured as follows:: - - [ MS Active Directory or other federated LDAP user store ] - --> backends to… - [ SUN Identity Manager or other SAML Policy Controller ] - --> maps URLs to groups… - [ Apache Policy Agent in front of eAuth-secured Web Application ] - -In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion. - -Basic AWS API call structure ----------------------------- - -AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: - -* System User -* System Administrator -* Network Administrator -* Project Manager -* Cloud Administrator -* (IT-Sec?) - -There is an additional, conceptual end-user that may or may not have API access: - -* (EXTERNAL) End-user / Third-party User - -Basic operations are available to any System User: - -* Launch Instance -* Terminate Instance (their own) -* Create keypair -* Delete keypair -* Create, Upload, Delete: Buckets and Keys (Object Store) – their own -* Create, Attach, Delete Volume (Block Store) – their own - -System Administrators: - -* Register/Unregister Machine Image (project-wide) -* Change Machine Image properties (public / private) -* Request / Review CloudAudit Scans - -Network Administrator: - -* Change Firewall Rules, define Security Groups -* Allocate, Associate, Deassociate Public IP addresses - -Project Manager: - -* Launch and Terminate Instances (project-wide) -* CRUD of Object and Block store (project-wide) - -Cloud Administrator: - -* Register / Unregister Kernel and Ramdisk Images -* Register / Unregister Machine Image (any) - -Enhancements ------------- - -* SAML Token passing -* REST interfaces -* SOAP interfaces - -Wrapping the SAML token into the API calls. -Then store the UID (fetched via backchannel) into the instance metadata, providing end-to-end auditability of ownership and responsibility, without PII. - -CloudAudit APIs ---------------- - -* Request formats -* Response formats -* Stateless asynchronous queries - -CloudAudit queries may spawn long-running processes (similar to launching instances, etc.) They need to return a ReservationId in the same fashion, which can be returned in further queries for updates. -RBAC of CloudAudit API calls is critical, since detailed system information is a system vulnerability. - -Type declarations ---------------------- -* Data declarations – Volumes and Objects -* System declarations – Instances - -Existing API calls to launch instances specific a single, combined “type” flag. We propose to extend this with three additional type declarations, mapping to the “Confidentiality, Integrity, Availability” classifications of FIPS 199. An example API call would look like:: - - RunInstances type=m1.large number=1 secgroup=default key=mykey confidentiality=low integrity=low availability=low - -These additional parameters would also apply to creation of block storage volumes (along with the existing parameter of ‘size’), and creation of object storage ‘buckets’. (C.I.A. classifications on a bucket would be inherited by the keys within this bucket.) - -Request Brokering ------------------ - - * Cloud Interop - * IMF Registration / PubSub - * Digital C&A - -Establishing declarative semantics for individual API calls will allow the cloud environment to seamlessly proxy these API calls to external, third-party vendors – when the requested CIA levels match. - -See related work within the Infrastructure 2.0 working group for more information on how the IMF Metadata specification could be utilized to manage registration of these vendors and their C&A credentials. - -Dirty Cloud – Hybrid Data Centers ---------------------------------- - -* CloudAudit bridge interfaces -* Anything in the ARP table - -A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. - -This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment. - -Network discovery protocols (ARP, CDP) can be applied in this case, and existing protocols (SNMP location data, DNS LOC records) overloaded to provide CloudAudit information. - -The Details ------------ - - * Preliminary Roles Definitions - * Categorization of available API calls - * SAML assertion vocabulary - -System limits -------------- - -The following limits need to be defined and enforced: - -* Total number of instances allowed (user / project) -* Total number of instances, per instance type (user / project) -* Total number of volumes (user / project) -* Maximum size of volume -* Cumulative size of all volumes -* Total use of object storage (GB) -* Total number of Public IPs - - -Further Challenges ------------------- - * Prioritization of users / jobs in shared computing environments - * Incident response planning - * Limit launch of instances to specific security groups based on AMI - * Store AMIs in LDAP for added property control - - - -The :mod:`rbac` Module --------------------------- - -.. automodule:: nova.auth.rbac - :members: - :undoc-members: - :show-inheritance: - -The :mod:`signer` Module ------------------------- - -.. automodule:: nova.auth.signer - :members: - :undoc-members: - :show-inheritance: - -The :mod:`users` Module ------------------------ - -.. automodule:: nova.auth.users - :members: - :undoc-members: - :show-inheritance: - -The :mod:`users_unittest` Module --------------------------------- - -.. automodule:: nova.tests.users_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`access_unittest` Module ---------------------------------- - -.. automodule:: nova.tests.access_unittest - :members: - :undoc-members: - :show-inheritance: - - diff --git a/docs/binaries.rst b/docs/binaries.rst deleted file mode 100644 index 90a9581f7..000000000 --- a/docs/binaries.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Binaries -=============== - -* nova-api -* nova-compute -* nova-manage -* nova-objectstore -* nova-volume - -The configuration of these binaries relies on "flagfiles" using the google -gflags package. If present, the nova.conf file will be used as the flagfile -- otherwise, it must be specified on the command line:: - - $ python node_worker.py --flagfile flagfile diff --git a/docs/compute.rst b/docs/compute.rst deleted file mode 100644 index 5b08dbd5b..000000000 --- a/docs/compute.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Compute Documentation -===================== - -This page contains the Compute Package documentation. - - -The :mod:`disk` Module ----------------------- - -.. automodule:: nova.compute.disk - :members: - :undoc-members: - :show-inheritance: - -The :mod:`exception` Module ---------------------------- - -.. automodule:: nova.compute.exception - :members: - :undoc-members: - :show-inheritance: - -The :mod:`model` Module -------------------------- - -.. automodule:: nova.compute.model - :members: - :undoc-members: - :show-inheritance: - -The :mod:`network` Module -------------------------- - -.. automodule:: nova.compute.network - :members: - :undoc-members: - :show-inheritance: - -The :mod:`node` Module ----------------------- - -.. automodule:: nova.compute.node - :members: - :undoc-members: - :show-inheritance: - -RELATED TESTS ---------------- - -The :mod:`node_unittest` Module -------------------------------- - -.. automodule:: nova.tests.node_unittest - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index fb3fd1a30..000000000 --- a/docs/conf.py +++ /dev/null @@ -1,202 +0,0 @@ -# -*- coding: utf-8 -*- -# -# nova documentation build configuration file, created by -# sphinx-quickstart on Sat May 1 15:17:47 2010. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('/Users/jmckenty/Projects/cc')) -sys.path.append([os.path.abspath('../nova'),os.path.abspath('../'),os.path.abspath('../vendor')]) - - -# -- General configuration ----------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig'] -#sphinx_to_github = False -todo_include_todos = True - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'nova' -copyright = u'2010, United States Government as represented by the Administrator of the National Aeronautics and Space Administration.' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.42' -# The full version, including alpha/beta/rc tags. -release = '0.42' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -#unused_docs = [] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -modindex_common_prefix = ['nova.'] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. Major themes that come with -# Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_use_modindex = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'novadoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'nova.tex', u'nova Documentation', - u'Anso Labs, LLC', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_use_modindex = True - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/endpoint.rst b/docs/endpoint.rst deleted file mode 100644 index 399df4161..000000000 --- a/docs/endpoint.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Endpoint Documentation -====================== - -This page contains the Endpoint Package documentation. - -The :mod:`admin` Module ------------------------ - -.. automodule:: nova.endpoint.admin - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api` Module ---------------------- - -.. automodule:: nova.endpoint.api - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud` Module ------------------------ - -.. automodule:: nova.endpoint.cloud - :members: - :undoc-members: - :show-inheritance: - -The :mod:`images` Module ------------------------- - -.. automodule:: nova.endpoint.images - :members: - :undoc-members: - :show-inheritance: - - -RELATED TESTS --------------- - -The :mod:`api_unittest` Module ------------------------------- - -.. automodule:: nova.tests.api_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api_integration` Module ---------------------------------- - -.. automodule:: nova.tests.api_integration - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud_unittest` Module --------------------------------- - -.. automodule:: nova.tests.cloud_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`network_unittest` Module ----------------------------------- - -.. automodule:: nova.tests.network_unittest - :members: - :undoc-members: - :show-inheritance: - - diff --git a/docs/fakes.rst b/docs/fakes.rst deleted file mode 100644 index bea8bc4e9..000000000 --- a/docs/fakes.rst +++ /dev/null @@ -1,43 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Fakes -========== - -The :mod:`fakevirt` Module --------------------------- - -.. automodule:: nova.fakevirt - :members: - :undoc-members: - :show-inheritance: - -The :mod:`fakeldap` Module --------------------------- - -.. automodule:: nova.auth.fakeldap - :members: - :undoc-members: - :show-inheritance: - -The :mod:`fakerabbit` Module ----------------------------- - -.. automodule:: nova.fakerabbit - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/getting.started.rst b/docs/getting.started.rst deleted file mode 100644 index 3eadd0882..000000000 --- a/docs/getting.started.rst +++ /dev/null @@ -1,148 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Getting Started with Nova -========================= - - -GOTTA HAVE A nova.pth file added or it WONT WORK (will write setup.py file soon) - -Create a file named nova.pth in your python libraries directory -(usually /usr/local/lib/python2.6/dist-packages) with a single line that points -to the directory where you checked out the source (that contains the nova/ -directory). - -DEPENDENCIES ------------- - -Related servers we rely on - -* RabbitMQ: messaging queue, used for all communication between components -* OpenLDAP: users, groups (maybe cut) -* ReDIS: Remote Dictionary Store (for fast, shared state data) -* nginx: HTTP server to handle serving large files (because Tornado can't) - -Python libraries we don't vendor - -* M2Crypto: python library interface for openssl -* curl - -Vendored python libaries (don't require any installation) - -* Tornado: scalable non blocking web server for api requests -* Twisted: just for the twisted.internet.defer package -* boto: python api for aws api -* IPy: library for managing ip addresses - -Recommended ------------------ - -* euca2ools: python implementation of aws ec2-tools and ami tools -* build tornado to use C module for evented section - - -Installation --------------- -:: - - # system libraries and tools - apt-get install -y aoetools vlan curl - modprobe aoe - - # python libraries - apt-get install -y python-setuptools python-dev python-pycurl python-m2crypto - - # ON THE CLOUD CONTROLLER - apt-get install -y rabbitmq-server dnsmasq nginx - # build redis from 2.0.0-rc1 source - # setup ldap (slap.sh as root will remove ldap and reinstall it) - NOVA_PATH/nova/auth/slap.sh - /etc/init.d/rabbitmq-server start - - # ON VOLUME NODE: - apt-get install -y vblade-persist - - # ON THE COMPUTE NODE: - apt-get install -y python-libvirt - apt-get install -y kpartx kvm libvirt-bin - modprobe kvm - - # optional packages - apt-get install -y euca2ools - -Configuration ---------------- - -ON CLOUD CONTROLLER - -* Add yourself to the libvirtd group, log out, and log back in -* fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances -:: - - iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 - iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE - - -* Configure NginX proxy (/etc/nginx/sites-enabled/default) - -:: - - server { - listen 3333 default; - server-name localhost; - client_max_body_size 10m; - - access_log /var/log/nginx/localhost.access.log; - - location ~ /_images/.+ { - root NOVA_PATH/images; - rewrite ^/_images/(.*)$ /$1 break; - } - - location / { - proxy_pass http://localhost:3334/; - } - } - -ON VOLUME NODE - -* create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb) - -:: - - # This creates a 1GB file to create volumes out of - dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 - losetup --show -f MY_FILE_PATH - # replace loop0 below with whatever losetup returns - echo "--storage_dev=/dev/loop0" >> NOVA_PATH/bin/nova.conf - -Running ---------- - -Launch servers - -* rabbitmq -* redis -* slapd -* nginx - -Launch nova components - -* nova-api -* nova-compute -* nova-objectstore -* nova-volume diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index ef2e8f63e..000000000 --- a/docs/index.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Welcome to nova's documentation! -================================ - -Nova is a cloud computing fabric controller (the main part of an IaaS system) built to match the popular AWS EC2 and S3 APIs. -It is written in Python, using the Tornado and Twisted frameworks, and relies on the standard AMQP messaging protocol, -and the Redis distributed KVS. -Nova is intended to be easy to extend, and adapt. For example, it currently uses -an LDAP server for users and groups, but also includes a fake LDAP server, -that stores data in Redis. It has extensive test coverage, and uses the -Sphinx toolkit (the same as Python itself) for code and user documentation. -While Nova is currently in Beta use within several organizations, the codebase -is very much under active development - there are bugs! - -Contents: - -.. toctree:: - :maxdepth: 2 - - getting.started - architecture - network - storage - auth - compute - endpoint - nova - fakes - binaries - todo - modules - packages - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/modules.rst b/docs/modules.rst deleted file mode 100644 index 82c61f008..000000000 --- a/docs/modules.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Nova Documentation -================== - -This page contains the Nova Modules documentation. - -Modules: --------- - -.. toctree:: - :maxdepth: 4 - - auth - compute - endpoint - fakes - nova - volume diff --git a/docs/network.rst b/docs/network.rst deleted file mode 100644 index 357a0517f..000000000 --- a/docs/network.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -nova Networking -================ - -The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules. - -Components ----------- -There are several key components: - -* NetworkController (Manages address and vlan allocation) -* RoutingNode (NATs public IPs to private IPs, and enforces firewall rules) -* AddressingNode (runs DHCP services for private networks) -* BridgingNode (a subclass of the basic nova ComputeNode) -* TunnelingNode (provides VPN connectivity) - -Component Diagram ------------------ - -Overview:: - - (PUBLIC INTERNET) - | \ - / \ / \ - [RoutingNode] ... [RN] [TunnelingNode] ... [TN] - | \ / | | - | < AMQP > | | - [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] - \ | \ / - / \ / \ / \ / \ - [BridgingNode] ... [BridgingNode] - - - [NetworkController] ... [NetworkController] - \ / - < AMQP > - | - / \ - [CloudController]...[CloudController] - -While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). - -State Model ------------ -Network State consists of the following facts: - -* VLAN assignment (to a project) -* Private Subnet assignment (to a security group) in a VLAN -* Private IP assignments (to running instances) -* Public IP allocations (to a project) -* Public IP associations (to a private IP / running instance) - -While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. - -The Public Traffic Path ------------------------ - -Public Traffic:: - - (PUBLIC INTERNET) - | - <-- [RoutingNode] - | - [AddressingNode] --> | - ( VLAN ) - | <-- [BridgingNode] - | - - -The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. - -Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). diff --git a/docs/nova.rst b/docs/nova.rst deleted file mode 100644 index 4b9c44a5f..000000000 --- a/docs/nova.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -NOVA Libraries -=============== - -The :mod:`crypto` Module ------------------------- - -.. automodule:: nova.crypto - :members: - :undoc-members: - :show-inheritance: - -The :mod:`adminclient` Module ------------------------------ - -.. automodule:: nova.adminclient - :members: - :undoc-members: - :show-inheritance: - -The :mod:`datastore` Module ---------------------------- - -.. automodule:: nova.datastore - :members: - :undoc-members: - :show-inheritance: - -The :mod:`exception` Module ---------------------------- - -.. automodule:: nova.exception - :members: - :undoc-members: - :show-inheritance: - -The :mod:`flags` Module ---------------------------- - -.. automodule:: nova.flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`rpc` Module ---------------------------- - -.. automodule:: nova.rpc - :members: - :undoc-members: - :show-inheritance: - -The :mod:`server` Module ---------------------------- - -.. automodule:: nova.server - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test` Module ---------------------------- - -.. automodule:: nova.test - :members: - :undoc-members: - :show-inheritance: - -The :mod:`utils` Module ---------------------------- - -.. automodule:: nova.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/objectstore.rst b/docs/objectstore.rst deleted file mode 100644 index 6b8d293f4..000000000 --- a/docs/objectstore.rst +++ /dev/null @@ -1,66 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Objectstore Documentation -========================= - -This page contains the Objectstore Package documentation. - - -The :mod:`bucket` Module ------------------------- - -.. automodule:: nova.objectstore.bucket - :members: - :undoc-members: - :show-inheritance: - -The :mod:`handler` Module -------------------------- - -.. automodule:: nova.objectstore.handler - :members: - :undoc-members: - :show-inheritance: - -The :mod:`image` Module ------------------------ - -.. automodule:: nova.objectstore.image - :members: - :undoc-members: - :show-inheritance: - -The :mod:`stored` Module ------------------------- - -.. automodule:: nova.objectstore.stored - :members: - :undoc-members: - :show-inheritance: - -RELATED TESTS -------------- - -The :mod:`objectstore_unittest` Module --------------------------------------- - -.. automodule:: nova.tests.objectstore_unittest - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/packages.rst b/docs/packages.rst deleted file mode 100644 index 6029ad7d7..000000000 --- a/docs/packages.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -nova Packages & Dependencies -============================ - -Nova is being built on Ubuntu Lucid. - -The following packages are required: - - apt-get install python-ipy, python-libvirt, python-boto, python-pycurl, python-twisted, python-daemon, python-redis, python-carrot, python-lockfile - -In addition you need to install python: - - * python-gflags - http://code.google.com/p/python-gflags/ diff --git a/docs/storage.rst b/docs/storage.rst deleted file mode 100644 index f77e5f0e5..000000000 --- a/docs/storage.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Storage in the Nova Cloud -========================= - -There are three primary classes of storage in a nova cloud environment: - -* Ephemeral Storage (local disk within an instance) -* Volume Storage (network-attached FS) -* Object Storage (redundant KVS with locality and MR) - -.. toctree:: - :maxdepth: 2 - - volume - objectstore diff --git a/docs/volume.rst b/docs/volume.rst deleted file mode 100644 index 619968458..000000000 --- a/docs/volume.rst +++ /dev/null @@ -1,45 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Volume Documentation -==================== - -Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. - -Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. - -AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.) - -The underlying volumes are LVM logical volumes, created on demand within a single large volume group. - - -The :mod:`storage` Module -------------------------- - -.. automodule:: nova.volume.storage - :members: - :undoc-members: - :show-inheritance: - -The :mod:`storage_unittest` Module ----------------------------------- - -.. automodule:: nova.tests.storage_unittest - :members: - :undoc-members: - :show-inheritance: - diff --git a/nova/compute/disk.py b/nova/compute/disk.py index 08a22556e..5749d4c6a 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -40,7 +40,8 @@ def partition(infile, outfile, local_bytes=0, local_type='ext2', execute=None): formatted as ext2. In the diagram below, dashes represent drive sectors. - 0 a b c d e + +-----+------. . .-------+------. . .------+ + | 0 a| b c|d e| +-----+------. . .-------+------. . .------+ | mbr | primary partiton | local partition | +-----+------. . .-------+------. . .------+ diff --git a/setup.cfg b/setup.cfg index 278586962..839472544 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ [build_sphinx] -source-dir = docs -build-dir = docs/_build +source-dir = doc/source +build-dir = doc/build all_files = 1 -- cgit From 480d1b6d5dd12490298b2b4d3e62f40917390bde Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 25 Jul 2010 15:32:10 +0100 Subject: Add an import so that nova-compute sees the images_path flag, so that it can be used on the command line. --- bin/nova-compute | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-compute b/bin/nova-compute index 5635efbaf..63d57a765 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -43,6 +43,7 @@ from nova import flags from nova import rpc from nova import twistd from nova.compute import node +from nova.objectstore import image # For the images_path flag FLAGS = flags.FLAGS -- cgit From fdea01a233e72551e750a5beaca0739ec8173ac3 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 25 Jul 2010 17:28:39 +0100 Subject: Set durable=False on TopicPublisher, so that it matches the flag on TopicConsumer. This ensures that either redeclaration of the control_exchange will use the same flag, and avoid AMQPChannelException. --- nova/rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/rpc.py b/nova/rpc.py index ef463e84b..5a2f4b3ad 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -151,6 +151,7 @@ class TopicPublisher(Publisher): def __init__(self, connection=None, topic="broadcast"): self.routing_key = topic self.exchange = FLAGS.control_exchange + self.durable = False super(TopicPublisher, self).__init__(connection=connection) -- cgit From ad2250ac0080ca35b1fd2747e3f4d0ff07bc90be Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 25 Jul 2010 17:40:41 +0100 Subject: Replace hardcoded "nova" with FLAGS.control_exchange. --- nova/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index ef463e84b..5610ea124 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -242,7 +242,7 @@ def send_message(topic, message, wait=True): consumer.register_callback(generic_response) publisher = messaging.Publisher(connection=Connection.instance(), - exchange="nova", + exchange=FLAGS.control_exchange, exchange_type="topic", routing_key=topic) publisher.send(message) -- cgit From a8c8aed28ce5d1d9eadcbecab03f6bc3bec8e622 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 25 Jul 2010 19:09:12 +0100 Subject: Fix references to get_argument, fixing internal error when calling euca-deregister. --- nova/objectstore/handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index c670ee02f..fd1ed848c 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -273,8 +273,8 @@ class ImageResource(Resource): def render_POST(self, request): """ update image attributes: public/private """ - image_id = self.get_argument('image_id', u'') - operation = self.get_argument('operation', u'') + image_id = get_argument(request, 'image_id', u'') + operation = get_argument(request, 'operation', u'') image_object = image.Image(image_id) @@ -287,7 +287,7 @@ class ImageResource(Resource): def render_DELETE(self, request): """ delete a registered image """ - image_id = self.get_argument("image_id", u"") + image_id = get_argument(request, "image_id", u"") image_object = image.Image(image_id) if not image.is_authorized(request.context): -- cgit From 0278767e0dc41444b889f904e6e49d26be5a54c4 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 25 Jul 2010 19:25:42 +0100 Subject: Fix references to image_object. This caused an internal error when using euca-deregister. --- nova/objectstore/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index fd1ed848c..ae3ffa0eb 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -278,7 +278,7 @@ class ImageResource(Resource): image_object = image.Image(image_id) - if not image.is_authorized(request.context): + if not image_object.is_authorized(request.context): raise exception.NotAuthorized image_object.set_public(operation=='add') @@ -290,7 +290,7 @@ class ImageResource(Resource): image_id = get_argument(request, "image_id", u"") image_object = image.Image(image_id) - if not image.is_authorized(request.context): + if not image_object.is_authorized(request.context): raise exception.NotAuthorized image_object.delete() -- cgit