diff options
29 files changed, 380 insertions, 373 deletions
diff --git a/.bzrignore b/.bzrignore new file mode 100644 index 000000000..93fc868a3 --- /dev/null +++ b/.bzrignore @@ -0,0 +1 @@ +run_tests.err.log diff --git a/CA/geninter.sh b/CA/geninter.sh index 6c0528d1b..7d6c280d5 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -17,7 +17,7 @@ # under the License. # ARG is the id of the user -export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-intCA-$3 +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$1" mkdir INTER/$1 cd INTER/$1 cp ../../openssl.cnf.tmpl openssl.cnf diff --git a/bin/dhcpleasor.py b/bin/nova-dhcpbridge index 4a3f374d5..cc827c50e 100755 --- a/bin/dhcpleasor.py +++ b/bin/nova-dhcpbridge @@ -18,23 +18,27 @@ # under the License. """ -dhcpleasor.py +nova-dhcpbridge Handle lease database updates from DHCP servers. """ -import sys -import os import logging +import os +import sys + +#TODO(joshua): there is concern that the user dnsmasq runs under will not +# have nova in the path. This should be verified and if it is +# not true the ugly line below can be removed sys.path.append(os.path.abspath(os.path.join(__file__, "../../"))) -logging.debug(sys.path) -import getopt -from os import environ -from nova import rpc from nova import flags +from nova import rpc +from nova import utils from nova.compute import linux_net from nova.compute import network + + FLAGS = flags.FLAGS @@ -63,11 +67,12 @@ def init_leases(interface): return res -def main(argv=None): - if argv is None: - argv = sys.argv - interface = environ.get('DNSMASQ_INTERFACE', 'br0') - if int(environ.get('TESTING', '0')): +def main(): + flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) + utils.default_flagfile(flagfile) + argv = FLAGS(sys.argv) + interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') + if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True FLAGS.redis_db = 8 FLAGS.network_size = 32 diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 521f3d5d1..9385fd299 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -18,33 +18,32 @@ # under the License. """ - Tornado daemon for nova objectstore. Supports S3 API. + Twisted daemon for nova objectstore. Supports S3 API. """ import logging -from tornado import httpserver -from tornado import ioloop from nova import flags -from nova import server from nova import utils -from nova.auth import users +from nova import twistd from nova.objectstore import handler FLAGS = flags.FLAGS -def main(argv): +def main(): # FIXME: if this log statement isn't here, no logging # appears from other files and app won't start daemonized - logging.debug('Started HTTP server on %s' % (FLAGS.s3_internal_port)) - app = handler.Application(users.UserManager()) - server = httpserver.HTTPServer(app) - server.listen(FLAGS.s3_internal_port) - ioloop.IOLoop.instance().start() - + logging.debug('Started HTTP server on %s' % (FLAGS.s3_port)) + app = handler.get_application() + print app + return app +# NOTE(soren): Stolen from nova-compute if __name__ == '__main__': + twistd.serve(__file__) + +if __name__ == '__builtin__': utils.default_flagfile() - server.serve('nova-objectstore', main) + application = main() diff --git a/debian/control b/debian/control index 17414bb7a..a6d12f36e 100644 --- a/debian/control +++ b/debian/control @@ -91,7 +91,7 @@ Description: Nova Cloud Computing - API frontend Package: nova-objectstore Architecture: all -Depends: nova-common (= ${binary:Version}), nginx, ${python:Depends}, ${misc:Depends} +Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends} Description: Nova Cloud Computing - object store Nova is a cloud computing fabric controller (the main part of an IaaS system) built to match the popular AWS EC2 and S3 APIs. It is written in diff --git a/debian/nova-api.conf b/debian/nova-api.conf index 9cd4051b1..d0b796878 100644 --- a/debian/nova-api.conf +++ b/debian/nova-api.conf @@ -1,5 +1,6 @@ --daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys +--networks_path=/var/lib/nova/networks +--dhcpbridge_flagfile=/etc/nova/nova-dhcpbridge.conf --fake_users=1 ---datastore_path=/var/lib/nova/keeper diff --git a/debian/nova-api.install b/debian/nova-api.install index 02dbda02d..89615d302 100644 --- a/debian/nova-api.install +++ b/debian/nova-api.install @@ -1,2 +1,3 @@ bin/nova-api usr/bin debian/nova-api.conf etc/nova +debian/nova-dhcpbridge.conf etc/nova diff --git a/debian/nova-compute.conf b/debian/nova-compute.conf index e4ca3fe95..d862f2328 100644 --- a/debian/nova-compute.conf +++ b/debian/nova-compute.conf @@ -1,8 +1,6 @@ --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys ---datastore_path=/var/lib/nova/keeper --instances_path=/var/lib/nova/instances ---networks_path=/var/lib/nova/networks --simple_network_template=/usr/share/nova/interfaces.template --libvirt_xml_template=/usr/share/nova/libvirt.xml.template --vpn_client_template=/usr/share/nova/client.ovpn.template diff --git a/debian/nova-dhcp.conf b/debian/nova-dhcp.conf new file mode 100644 index 000000000..0aafe7549 --- /dev/null +++ b/debian/nova-dhcp.conf @@ -0,0 +1,2 @@ +--networks_path=/var/lib/nova/networks +--fake_users=1 diff --git a/debian/nova-objectstore.conf b/debian/nova-objectstore.conf index af3271d3b..03f5df051 100644 --- a/debian/nova-objectstore.conf +++ b/debian/nova-objectstore.conf @@ -1,7 +1,6 @@ --daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys ---datastore_path=/var/lib/nova/keeper --fake_users=1 --images_path=/var/lib/nova/images --buckets_path=/var/lib/nova/buckets diff --git a/debian/nova-objectstore.install b/debian/nova-objectstore.install index 3ed93ff37..c5b3d997a 100644 --- a/debian/nova-objectstore.install +++ b/debian/nova-objectstore.install @@ -1,3 +1,2 @@ bin/nova-objectstore usr/bin debian/nova-objectstore.conf etc/nova -debian/nova-objectstore.nginx.conf etc/nginx/sites-available diff --git a/debian/nova-objectstore.links b/debian/nova-objectstore.links deleted file mode 100644 index 38e33948e..000000000 --- a/debian/nova-objectstore.links +++ /dev/null @@ -1 +0,0 @@ -/etc/nginx/sites-available/nova-objectstore.nginx.conf /etc/nginx/sites-enabled/nova-objectstore.nginx.conf diff --git a/debian/nova-objectstore.nginx.conf b/debian/nova-objectstore.nginx.conf deleted file mode 100644 index b63424150..000000000 --- a/debian/nova-objectstore.nginx.conf +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 3333 default; - server_name localhost; - client_max_body_size 10m; - - access_log /var/log/nginx/localhost.access.log; - - location ~ /_images/.+ { - root /var/lib/nova/images; - rewrite ^/_images/(.*)$ /$1 break; - } - - location / { - proxy_pass http://localhost:3334/; - } -} - diff --git a/debian/nova-volume.conf b/debian/nova-volume.conf index af3271d3b..03f5df051 100644 --- a/debian/nova-volume.conf +++ b/debian/nova-volume.conf @@ -1,7 +1,6 @@ --daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys ---datastore_path=/var/lib/nova/keeper --fake_users=1 --images_path=/var/lib/nova/images --buckets_path=/var/lib/nova/buckets diff --git a/nova/cloudpipe/bootscript.sh b/nova/cloudpipe/bootscript.sh index 43fc2ecab..82ec2012a 100755 --- a/nova/cloudpipe/bootscript.sh +++ b/nova/cloudpipe/bootscript.sh @@ -24,7 +24,7 @@ export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $1}'` export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $1}'` export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` -export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-vpn-$VPN_IP +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-vpn-$VPN_IP" DHCP_LOWER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 10 }'` DHCP_UPPER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 1 }'` diff --git a/nova/compute/linux_net.py b/nova/compute/linux_net.py index 00c64d81a..48e07da66 100644 --- a/nova/compute/linux_net.py +++ b/nova/compute/linux_net.py @@ -28,12 +28,16 @@ from nova import utils from nova import flags FLAGS=flags.FLAGS -def execute(cmd): +flags.DEFINE_string('dhcpbridge_flagfile', + '/etc/nova-dhcpbridge.conf', + 'location of flagfile for dhcpbridge') + +def execute(cmd, addl_env=None): if FLAGS.fake_network: logging.debug("FAKE NET: %s" % cmd) return "fake", 0 else: - return utils.execute(cmd) + return utils.execute(cmd, addl_env=addl_env) def runthis(desc, cmd): if FLAGS.fake_network: @@ -61,7 +65,7 @@ def remove_rule(cmd): def bind_public_ip(ip, interface): runthis("Binding IP to interface: %s", "sudo ip addr add %s dev %s" % (ip, interface)) - + def unbind_public_ip(ip, interface): runthis("Binding IP to interface: %s", "sudo ip addr del %s dev %s" % (ip, interface)) @@ -99,7 +103,7 @@ def dnsmasq_cmd(net): ' --except-interface=lo', ' --dhcp-range=%s,static,600s' % (net.dhcp_range_start), ' --dhcp-hostsfile=%s' % dhcp_file(net['vlan'], 'conf'), - ' --dhcp-script=%s' % bin_file('dhcpleasor.py'), + ' --dhcp-script=%s' % bin_file('nova-dhcpbridge'), ' --leasefile-ro'] return ''.join(cmd) @@ -139,7 +143,9 @@ def start_dnsmasq(network): if os.path.exists(lease_file): os.unlink(lease_file) - Popen(dnsmasq_cmd(network).split(" ")) + # FLAGFILE in env + env = {'FLAGFILE' : FLAGS.dhcpbridge_flagfile} + execute(dnsmasq_cmd(network), addl_env=env) def stop_dnsmasq(network): """ stops the dnsmasq instance for a given network """ diff --git a/nova/compute/network.py b/nova/compute/network.py index 90d6b2dc6..43011f696 100644 --- a/nova/compute/network.py +++ b/nova/compute/network.py @@ -162,6 +162,7 @@ class Vlan(datastore.BasicModel): class BaseNetwork(datastore.BasicModel): override_type = 'network' + NUM_STATIC_IPS = 3 # Network, Gateway, and CloudPipe @property def identifier(self): @@ -237,11 +238,15 @@ class BaseNetwork(datastore.BasicModel): def available(self): # the .2 address is always CloudPipe # and the top <n> are for vpn clients - for idx in range(3, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): + for idx in range(self.num_static_ips, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): address = str(self.network[idx]) if not address in self.hosts.keys(): yield str(address) + @property + def num_static_ips(self): + return BaseNetwork.NUM_STATIC_IPS + def allocate_ip(self, user_id, project_id, mac): for address in self.available: logging.debug("Allocating IP %s to %s" % (address, project_id)) @@ -251,7 +256,7 @@ class BaseNetwork(datastore.BasicModel): raise compute_exception.NoMoreAddresses("Project %s with network %s" % (project_id, str(self.network))) - def lease_ip(self, ip_str): + def lease_ip(self, ip_str): logging.debug("Leasing allocated IP %s" % (ip_str)) def release_ip(self, ip_str): @@ -566,10 +571,10 @@ def allocate_ip(user_id, project_id, mac): def deallocate_ip(address): return get_network_by_address(address).deallocate_ip(address) - + def release_ip(address): return get_network_by_address(address).release_ip(address) - + def lease_ip(address): return get_network_by_address(address).lease_ip(address) diff --git a/nova/compute/node.py b/nova/compute/node.py index 7146d1279..533670b12 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -58,7 +58,6 @@ class Node(object, service.Service): super(Node, self).__init__() self._instances = {} self._conn = virt_connection.get_connection() - self._pool = process.ProcessPool() self.instdir = model.InstanceDirectory() # TODO(joshua): This needs to ensure system state, specifically: modprobe aoe @@ -70,7 +69,7 @@ class Node(object, service.Service): # inst = self.instdir.get(instance_id) # return inst if self.instdir.exists(instance_id): - return Instance.fromName(self._conn, self._pool, instance_id) + return Instance.fromName(self._conn, instance_id) return None @exception.wrap_exception @@ -80,7 +79,7 @@ class Node(object, service.Service): instance_names = self._conn.list_instances() for name in instance_names: try: - new_inst = Instance.fromName(self._conn, self._pool, name) + new_inst = Instance.fromName(self._conn, name) new_inst.update_state() except: pass @@ -90,7 +89,8 @@ class Node(object, service.Service): def describe_instances(self): retval = {} for inst in self.instdir.by_node(FLAGS.node_name): - retval[inst['instance_id']] = (Instance.fromName(self._conn, self._pool, inst['instance_id'])) + retval[inst['instance_id']] = ( + Instance.fromName(self._conn, inst['instance_id'])) return retval @defer.inlineCallbacks @@ -123,8 +123,7 @@ class Node(object, service.Service): inst['node_name'] = FLAGS.node_name inst.save() # TODO(vish) check to make sure the availability zone matches - new_inst = Instance(self._conn, name=instance_id, - pool=self._pool, data=inst) + new_inst = Instance(self._conn, name=instance_id, data=inst) logging.info("Instances current state is %s", new_inst.state) if new_inst.is_running(): raise exception.Error("Instance is already running") @@ -220,11 +219,8 @@ class Instance(object): SHUTOFF = 0x05 CRASHED = 0x06 - def __init__(self, conn, pool, name, data): + def __init__(self, conn, name, data): """ spawn an instance with a given name """ - # TODO(termie): pool should probably be a singleton instead of being passed - # here and in the classmethods - self._pool = pool self._conn = conn # TODO(vish): this can be removed after data has been updated # data doesn't seem to have a working iterator so in doesn't work @@ -263,11 +259,11 @@ class Instance(object): logging.debug("Finished init of Instance with id of %s" % name) @classmethod - def fromName(cls, conn, pool, name): + def fromName(cls, conn, name): """ use the saved data for reloading the instance """ instdir = model.InstanceDirectory() instance = instdir.get(name) - return cls(conn=conn, pool=pool, name=name, data=instance) + return cls(conn=conn, name=name, data=instance) def set_state(self, state_code, state_description=None): self.datamodel['state'] = state_code diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 51f5c859b..48e47018a 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -454,21 +454,21 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - # TODO(vish): move authorization checking into network.py for address in self.network.host_objs: - #logging.debug(address_record) - address_rv = { - 'public_ip': address['address'], - 'instance_id' : address.get('instance_id', 'free') - } - if context.user.is_admin(): - address_rv['instance_id'] = "%s (%s, %s)" % ( - address['instance_id'], - address['user_id'], - address['project_id'], - ) + # TODO(vish): implement a by_project iterator for addresses + if (context.user.is_admin() or + address['project_id'] == self.project.id): + address_rv = { + 'public_ip': address['address'], + 'instance_id' : address.get('instance_id', 'free') + } + if context.user.is_admin(): + address_rv['instance_id'] = "%s (%s, %s)" % ( + address['instance_id'], + address['user_id'], + address['project_id'], + ) addresses.append(address_rv) - # logging.debug(addresses) return {'addressesSet': addresses} @rbac.allow('netadmin') diff --git a/nova/flags.py b/nova/flags.py index caf2d2e93..dc8fc9d10 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -38,7 +38,6 @@ DEFINE_bool = DEFINE_bool DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') -DEFINE_integer('s3_internal_port', 3334, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') #DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 8377a57a6..c670ee02f 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -1,10 +1,11 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. # -# Copyright 2009 Facebook +# Copyright 2010 OpenStack LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2009 Facebook # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -37,15 +38,21 @@ S3 client with this module:: """ import datetime -import os -import json import logging +import json import multiprocessing -from tornado import escape, web +import os +from tornado import escape import urllib +from twisted.application import internet, service +from twisted.web.resource import Resource +from twisted.web import server, static + + from nova import exception from nova import flags +from nova.auth import users from nova.endpoint import api from nova.objectstore import bucket from nova.objectstore import image @@ -53,241 +60,217 @@ from nova.objectstore import image FLAGS = flags.FLAGS - -def catch_nova_exceptions(target): - # FIXME: find a way to wrap all handlers in the web.Application.__init__ ? - def wrapper(*args, **kwargs): - try: - return target(*args, **kwargs) - except exception.NotFound: - raise web.HTTPError(404) - except exception.NotAuthorized: - raise web.HTTPError(403) - - return wrapper - - -class Application(web.Application): +def render_xml(request, value): + assert isinstance(value, dict) and len(value) == 1 + request.setHeader("Content-Type", "application/xml; charset=UTF-8") + + name = value.keys()[0] + request.write('<?xml version="1.0" encoding="UTF-8"?>\n') + request.write('<' + escape.utf8(name) + + ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') + _render_parts(value.values()[0], request.write) + request.write('</' + escape.utf8(name) + '>') + request.finish() + +def finish(request, content=None): + if content: + request.write(content) + request.finish() + +def _render_parts(value, write_cb): + if isinstance(value, basestring): + write_cb(escape.xhtml_escape(value)) + elif isinstance(value, int) or isinstance(value, long): + write_cb(str(value)) + elif isinstance(value, datetime.datetime): + write_cb(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) + elif isinstance(value, dict): + for name, subvalue in value.iteritems(): + if not isinstance(subvalue, list): + subvalue = [subvalue] + for subsubvalue in subvalue: + write_cb('<' + escape.utf8(name) + '>') + _render_parts(subsubvalue, write_cb) + write_cb('</' + escape.utf8(name) + '>') + else: + raise Exception("Unknown S3 value type %r", value) + +def get_argument(request, key, default_value): + if key in request.args: + return request.args[key][0] + return default_value + +def get_context(request): + try: + # Authorization Header format: 'AWS <access>:<secret>' + authorization_header = request.getHeader('Authorization') + if not authorization_header: + raise exception.NotAuthorized + access, sep, secret = authorization_header.split(' ')[1].rpartition(':') + um = users.UserManager.instance() + print 'um %s' % um + (user, project) = um.authenticate(access, secret, {}, request.method, request.host, request.uri, False) + # FIXME: check signature here! + return api.APIRequestContext(None, user, project) + except exception.Error as ex: + logging.debug("Authentication Failure: %s" % ex) + raise exception.NotAuthorized + +class S3(Resource): """Implementation of an S3-like storage server based on local files.""" - def __init__(self, user_manager): - web.Application.__init__(self, [ - (r"/", RootHandler), - (r"/_images/(.+)", ImageDownloadHandler), - (r"/_images/", ImageHandler), - (r"/([^/]+)/(.+)", ObjectHandler), - (r"/([^/]+)/", BucketHandler), - ]) - self.buckets_path = os.path.abspath(FLAGS.buckets_path) - self.images_path = os.path.abspath(FLAGS.images_path) - - if not os.path.exists(self.buckets_path): - raise Exception("buckets_path does not exist") - if not os.path.exists(self.images_path): - raise Exception("images_path does not exist") - self.user_manager = user_manager - - -class BaseRequestHandler(web.RequestHandler): - SUPPORTED_METHODS = ("PUT", "GET", "DELETE", "HEAD") - - @property - def context(self): - if not hasattr(self, '_context'): - try: - # Authorization Header format: 'AWS <access>:<secret>' - access, sep, secret = self.request.headers['Authorization'].split(' ')[1].rpartition(':') - (user, project) = self.application.user_manager.authenticate(access, secret, {}, self.request.method, self.request.host, self.request.path, False) - # FIXME: check signature here! - self._context = api.APIRequestContext(self, user, project) - except exception.Error, ex: - logging.debug("Authentication Failure: %s" % ex) - raise web.HTTPError(403) - return self._context - - def render_xml(self, value): - assert isinstance(value, dict) and len(value) == 1 - self.set_header("Content-Type", "application/xml; charset=UTF-8") - name = value.keys()[0] - parts = [] - parts.append('<' + escape.utf8(name) + - ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') - self._render_parts(value.values()[0], parts) - parts.append('</' + escape.utf8(name) + '>') - self.finish('<?xml version="1.0" encoding="UTF-8"?>\n' + - ''.join(parts)) - - def _render_parts(self, value, parts=[]): - if isinstance(value, basestring): - parts.append(escape.xhtml_escape(value)) - elif isinstance(value, int) or isinstance(value, long): - parts.append(str(value)) - elif isinstance(value, datetime.datetime): - parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) - elif isinstance(value, dict): - for name, subvalue in value.iteritems(): - if not isinstance(subvalue, list): - subvalue = [subvalue] - for subsubvalue in subvalue: - parts.append('<' + escape.utf8(name) + '>') - self._render_parts(subsubvalue, parts) - parts.append('</' + escape.utf8(name) + '>') - else: - raise Exception("Unknown S3 value type %r", value) - - def head(self, *args, **kwargs): - return self.get(*args, **kwargs) + def getChild(self, name, request): + request.context = get_context(request) + if name == '': + return self + elif name == '_images': + return ImageResource() + else: + return BucketResource(name) -class RootHandler(BaseRequestHandler): - def get(self): - buckets = [b for b in bucket.Bucket.all() if b.is_authorized(self.context)] + def render_GET(self, request): + buckets = [b for b in bucket.Bucket.all() if b.is_authorized(request.context)] - self.render_xml({"ListAllMyBucketsResult": { + render_xml(request, {"ListAllMyBucketsResult": { "Buckets": {"Bucket": [b.metadata for b in buckets]}, }}) + return server.NOT_DONE_YET + +class BucketResource(Resource): + def __init__(self, name): + Resource.__init__(self) + self.name = name + def getChild(self, name, request): + if name == '': + return self + else: + return ObjectResource(bucket.Bucket(self.name), name) -class BucketHandler(BaseRequestHandler): - @catch_nova_exceptions - def get(self, bucket_name): - logging.debug("List keys for bucket %s" % (bucket_name)) + def render_GET(self, request): + logging.debug("List keys for bucket %s" % (self.name)) - bucket_object = bucket.Bucket(bucket_name) + bucket_object = bucket.Bucket(self.name) - if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + if not bucket_object.is_authorized(request.context): + raise exception.NotAuthorized - prefix = self.get_argument("prefix", u"") - marker = self.get_argument("marker", u"") - max_keys = int(self.get_argument("max-keys", 1000)) - terse = int(self.get_argument("terse", 0)) + prefix = get_argument(request, "prefix", u"") + marker = get_argument(request, "marker", u"") + max_keys = int(get_argument(request, "max-keys", 1000)) + terse = int(get_argument(request, "terse", 0)) results = bucket_object.list_keys(prefix=prefix, marker=marker, max_keys=max_keys, terse=terse) - self.render_xml({"ListBucketResult": results}) + render_xml(request, {"ListBucketResult": results}) + return server.NOT_DONE_YET - @catch_nova_exceptions - def put(self, bucket_name): - logging.debug("Creating bucket %s" % (bucket_name)) - bucket.Bucket.create(bucket_name, self.context) - self.finish() + def render_PUT(self, request): + logging.debug("Creating bucket %s" % (self.name)) + try: + print 'user is %s' % request.context + except Exception as e: + logging.exception(e) + logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context)) + bucket.Bucket.create(self.name, request.context) + return '' - @catch_nova_exceptions - def delete(self, bucket_name): - logging.debug("Deleting bucket %s" % (bucket_name)) - bucket_object = bucket.Bucket(bucket_name) + def render_DELETE(self, request): + logging.debug("Deleting bucket %s" % (self.name)) + bucket_object = bucket.Bucket(self.name) - if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + if not bucket_object.is_authorized(request.context): + raise exception.NotAuthorized bucket_object.delete() - self.set_status(204) - self.finish() - - -class ObjectHandler(BaseRequestHandler): - @catch_nova_exceptions - def get(self, bucket_name, object_name): - logging.debug("Getting object: %s / %s" % (bucket_name, object_name)) - - bucket_object = bucket.Bucket(bucket_name) + request.setResponseCode(204) + return '' - if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) - obj = bucket_object[urllib.unquote(object_name)] - self.set_header("Content-Type", "application/unknown") - self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp(obj.mtime)) - self.set_header("Etag", '"' + obj.md5 + '"') - self.finish(obj.read()) +class ObjectResource(Resource): + def __init__(self, bucket, name): + Resource.__init__(self) + self.bucket = bucket + self.name = name - @catch_nova_exceptions - def put(self, bucket_name, object_name): - logging.debug("Putting object: %s / %s" % (bucket_name, object_name)) - bucket_object = bucket.Bucket(bucket_name) + def render_GET(self, request): + logging.debug("Getting object: %s / %s" % (self.bucket.name, self.name)) - if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + if not self.bucket.is_authorized(request.context): + raise exception.NotAuthorized - key = urllib.unquote(object_name) - bucket_object[key] = self.request.body - self.set_header("Etag", '"' + bucket_object[key].md5 + '"') - self.finish() + obj = self.bucket[urllib.unquote(self.name)] + request.setHeader("Content-Type", "application/unknown") + request.setHeader("Last-Modified", datetime.datetime.utcfromtimestamp(obj.mtime)) + request.setHeader("Etag", '"' + obj.md5 + '"') + return static.File(obj.path).render_GET(request) - @catch_nova_exceptions - def delete(self, bucket_name, object_name): - logging.debug("Deleting object: %s / %s" % (bucket_name, object_name)) - bucket_object = bucket.Bucket(bucket_name) + def render_PUT(self, request): + logging.debug("Putting object: %s / %s" % (self.bucket.name, self.name)) - if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + if not self.bucket.is_authorized(request.context): + raise exception.NotAuthorized - del bucket_object[urllib.unquote(object_name)] - self.set_status(204) - self.finish() + key = urllib.unquote(self.name) + request.content.seek(0, 0) + self.bucket[key] = request.content.read() + request.setHeader("Etag", '"' + self.bucket[key].md5 + '"') + finish(request) + return server.NOT_DONE_YET + def render_DELETE(self, request): + logging.debug("Deleting object: %s / %s" % (self.bucket.name, self.name)) -class ImageDownloadHandler(BaseRequestHandler): - SUPPORTED_METHODS = ("GET", ) + if not self.bucket.is_authorized(request.context): + raise exception.NotAuthorized - @catch_nova_exceptions - def get(self, image_id): - """ send the decrypted image file + del self.bucket[urllib.unquote(self.name)] + request.setResponseCode(204) + return '' - streaming content through python is slow and should only be used - in development mode. You should serve files via a web server - in production. - """ +class ImageResource(Resource): + isLeaf = True - self.set_header("Content-Type", "application/octet-stream") - - READ_SIZE = 64*1024 - - img = image.Image(image_id) - with open(img.image_path, 'rb') as fp: - s = fp.read(READ_SIZE) - while s: - self.write(s) - s = fp.read(READ_SIZE) - - self.finish() - -class ImageHandler(BaseRequestHandler): - SUPPORTED_METHODS = ("POST", "PUT", "GET", "DELETE") + def getChild(self, name, request): + if name == '': + return self + else: + request.setHeader("Content-Type", "application/octet-stream") + img = image.Image(name) + return static.File(img.image_path) - @catch_nova_exceptions - def get(self): + def render_GET(self, request): """ returns a json listing of all images that a user has permissions to see """ - images = [i for i in image.Image.all() if i.is_authorized(self.context)] + images = [i for i in image.Image.all() if i.is_authorized(request.context)] - self.finish(json.dumps([i.metadata for i in images])) + request.write(json.dumps([i.metadata for i in images])) + request.finish() + return server.NOT_DONE_YET - @catch_nova_exceptions - def put(self): + def render_PUT(self, request): """ create a new registered image """ - image_id = self.get_argument('image_id', u'') - image_location = self.get_argument('image_location', u'') + image_id = get_argument(request, 'image_id', u'') + image_location = get_argument(request, 'image_location', u'') image_path = os.path.join(FLAGS.images_path, image_id) if not image_path.startswith(FLAGS.images_path) or \ os.path.exists(image_path): - raise web.HTTPError(403) + raise exception.NotAuthorized bucket_object = bucket.Bucket(image_location.split("/")[0]) manifest = image_location[len(image_location.split('/')[0])+1:] - if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + if not bucket_object.is_authorized(request.context): + raise exception.NotAuthorized p = multiprocessing.Process(target=image.Image.register_aws_image, - args=(image_id, image_location, self.context)) + args=(image_id, image_location, request.context)) p.start() - self.finish() + return '' - @catch_nova_exceptions - def post(self): + def render_POST(self, request): """ update image attributes: public/private """ image_id = self.get_argument('image_id', u'') @@ -295,22 +278,30 @@ class ImageHandler(BaseRequestHandler): image_object = image.Image(image_id) - if not image_object.is_authorized(self.context): - raise web.HTTPError(403) + if not image.is_authorized(request.context): + raise exception.NotAuthorized image_object.set_public(operation=='add') - self.finish() + return '' - @catch_nova_exceptions - def delete(self): + def render_DELETE(self, request): """ delete a registered image """ image_id = self.get_argument("image_id", u"") image_object = image.Image(image_id) - if not image_object.is_authorized(self.context): - raise web.HTTPError(403) + if not image.is_authorized(request.context): + raise exception.NotAuthorized image_object.delete() - self.set_status(204) + request.setResponseCode(204) + return '' + +def get_application(): + root = S3() + factory = server.Site(root) + application = service.Application("objectstore") + objectStoreService = internet.TCPServer(FLAGS.s3_port, factory) + objectStoreService.setServiceParent(application) + return application diff --git a/nova/process.py b/nova/process.py index ff789a08a..d3558ed2e 100644 --- a/nova/process.py +++ b/nova/process.py @@ -85,7 +85,7 @@ class _BackRelay(protocol.ProcessProtocol): def errReceivedIsBad(self, text): if self.deferred is not None: self.onProcessEnded = defer.Deferred() - err = _UnexpectedErrorOutput(text, self.onProcessEnded) + err = UnexpectedErrorOutput(text, self.onProcessEnded) self.deferred.errback(failure.Failure(err)) self.deferred = None self.transport.loseConnection() @@ -152,8 +152,8 @@ def getProcessOutput(executable, args=None, env=None, path=None, reactor=None, d = defer.Deferred() p = BackRelayWithInput( d, startedDeferred=startedDeferred, error_ok=error_ok, input=input) - # VISH: commands come in as unicode, but self.executes needs - # strings or process.spawn raises a deprecation warning + # NOTE(vish): commands come in as unicode, but self.executes needs + # strings or process.spawn raises a deprecation warning executable = str(executable) if not args is None: args = [str(x) for x in args] @@ -171,7 +171,7 @@ class ProcessPool(object): self.size = size and size or FLAGS.process_pool_size self._pool = defer.DeferredSemaphore(self.size) - def simpleExecute(self, cmd, **kw): + def simple_execute(self, cmd, **kw): """ Weak emulation of the old utils.execute() function. This only exists as a way to quickly move old execute methods to @@ -205,34 +205,13 @@ class ProcessPool(object): self._pool.release() return rv +class SharedPool(ProcessPool): + _instance = None + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(SharedPool, cls).__new__( + cls, *args, **kwargs) + return cls._instance -class Pool(object): - """ A simple process pool implementation around mutliprocessing. - - Allows up to `size` processes at a time and queues the rest. - - Using workarounds for multiprocessing behavior described in: - http://pypi.python.org/pypi/twisted.internet.processes/1.0b1 - """ - - def __init__(self, size=None): - self._size = size - self._pool = multiprocessing.Pool(size) - self._registerShutdown() - - def _registerShutdown(self): - reactor.addSystemEventTrigger( - 'during', 'shutdown', self.shutdown, reactor) - - def shutdown(self, reactor=None): - if not self._pool: - return - self._pool.close() - # wait for workers to finish - self._pool.terminate() - self._pool = None - - def apply(self, f, *args, **kw): - """ Add a task to the pool and return a deferred. """ - result = self._pool.apply_async(f, args, kw) - return threads.deferToThread(result.get) +def simple_execute(cmd, **kwargs): + return SharedPool().simple_execute(cmd, **kwargs) diff --git a/nova/rpc.py b/nova/rpc.py index 58a2b29cf..ef463e84b 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -110,6 +110,7 @@ class TopicConsumer(Consumer): self.queue = topic self.routing_key = topic self.exchange = FLAGS.control_exchange + self.durable = False super(TopicConsumer, self).__init__(connection=connection) @@ -195,7 +196,10 @@ def call(topic, msg): conn = Connection.instance() d = defer.Deferred() consumer = DirectConsumer(connection=conn, msg_id=msg_id) - consumer.register_callback(lambda data, message: d.callback(data)) + def deferred_receive(data, message): + message.ack() + d.callback(data) + consumer.register_callback(deferred_receive) injected = consumer.attach_to_tornado() # clean up after the injected listened and return x @@ -233,7 +237,8 @@ def send_message(topic, message, wait=True): exchange=msg_id, auto_delete=True, exchange_type="direct", - routing_key=msg_id) + routing_key=msg_id, + durable=False) consumer.register_callback(generic_response) publisher = messaging.Publisher(connection=Connection.instance(), diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 45ee6dbc7..98568aeae 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -19,16 +19,15 @@ import IPy import os import logging -import unittest from nova import flags from nova import test -from nova import exception -from nova.compute.exception import NoMoreAddresses -from nova.compute import network -from nova.auth import users from nova import utils +from nova.auth import users +from nova.compute import network +from nova.compute.exception import NoMoreAddresses +FLAGS = flags.FLAGS class NetworkTestCase(test.TrialTestCase): def setUp(self): @@ -148,21 +147,42 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_addresses(self): """ - Network size is 32, there are 5 addresses reserved for VPN. - So we should get 23 usable addresses + Here, we test that a proper NoMoreAddresses exception is raised. + + However, the number of available IP addresses depends on the test + environment's setup. + + Network size is set in test fixture's setUp method. + + There are FLAGS.cnt_vpn_clients addresses reserved for VPN (NUM_RESERVED_VPN_IPS) + + And there are NUM_STATIC_IPS that are always reserved by Nova for the necessary + services (gateway, CloudPipe, etc) + + So we should get flags.network_size - (NUM_STATIC_IPS + + NUM_PREALLOCATED_IPS + + NUM_RESERVED_VPN_IPS) + usable addresses """ net = network.get_project_network("project0", "default") + + # Determine expected number of available IP addresses + num_static_ips = net.num_static_ips + num_preallocated_ips = len(net.hosts.keys()) + num_reserved_vpn_ips = flags.FLAGS.cnt_vpn_clients + num_available_ips = flags.FLAGS.network_size - (num_static_ips + num_preallocated_ips + num_reserved_vpn_ips) + hostname = "toomany-hosts" macs = {} addresses = {} - for i in range(0, 22): + for i in range(0, (num_available_ips - 1)): macs[i] = utils.generate_mac() addresses[i] = network.allocate_ip("netuser", "project0", macs[i]) self.dnsmasq.issue_ip(macs[i], addresses[i], hostname, net.bridge_name) self.assertRaises(NoMoreAddresses, network.allocate_ip, "netuser", "project0", utils.generate_mac()) - for i in range(0, 22): + for i in range(0, (num_available_ips - 1)): rv = network.deallocate_ip(addresses[i]) self.dnsmasq.release_ip(macs[i], addresses[i], hostname, net.bridge_name) @@ -180,14 +200,20 @@ def binpath(script): class FakeDNSMasq(object): def issue_ip(self, mac, ip, hostname, interface): - cmd = "%s add %s %s %s" % (binpath('dhcpleasor.py'), mac, ip, hostname) - env = {'DNSMASQ_INTERFACE': interface, 'TESTING' : '1'} + cmd = "%s add %s %s %s" % (binpath('nova-dhcpbridge'), + mac, ip, hostname) + env = {'DNSMASQ_INTERFACE': interface, + 'TESTING' : '1', + 'FLAGFILE' : FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s " % (out, err)) def release_ip(self, mac, ip, hostname, interface): - cmd = "%s del %s %s %s" % (binpath('dhcpleasor.py'), mac, ip, hostname) - env = {'DNSMASQ_INTERFACE': interface, 'TESTING' : '1'} + cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), + mac, ip, hostname) + env = {'DNSMASQ_INTERFACE': interface, + 'TESTING' : '1', + 'FLAGFILE' : FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("RELEASE_IP: %s, %s " % (out, err)) diff --git a/nova/tests/process_unittest.py b/nova/tests/process_unittest.py index 01648961f..1c15b69a0 100644 --- a/nova/tests/process_unittest.py +++ b/nova/tests/process_unittest.py @@ -37,7 +37,7 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stdout(self): pool = process.ProcessPool(2) - d = pool.simpleExecute('echo test') + d = pool.simple_execute('echo test') def _check(rv): self.assertEqual(rv[0], 'test\n') self.assertEqual(rv[1], '') @@ -48,38 +48,38 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stderr(self): pool = process.ProcessPool(2) - d = pool.simpleExecute('cat BAD_FILE', error_ok=1) + d = pool.simple_execute('cat BAD_FILE', error_ok=1) def _check(rv): self.assertEqual(rv[0], '') self.assert_('No such file' in rv[1]) - + d.addCallback(_check) d.addErrback(self.fail) return d def test_execute_unexpected_stderr(self): pool = process.ProcessPool(2) - d = pool.simpleExecute('cat BAD_FILE') + d = pool.simple_execute('cat BAD_FILE') d.addCallback(lambda x: self.fail('should have raised an error')) d.addErrback(lambda failure: failure.trap(IOError)) return d - + def test_max_processes(self): pool = process.ProcessPool(2) - d1 = pool.simpleExecute('sleep 0.01') - d2 = pool.simpleExecute('sleep 0.01') - d3 = pool.simpleExecute('sleep 0.005') - d4 = pool.simpleExecute('sleep 0.005') + d1 = pool.simple_execute('sleep 0.01') + d2 = pool.simple_execute('sleep 0.01') + d3 = pool.simple_execute('sleep 0.005') + d4 = pool.simple_execute('sleep 0.005') called = [] def _called(rv, name): called.append(name) - + d1.addCallback(_called, 'd1') d2.addCallback(_called, 'd2') d3.addCallback(_called, 'd3') d4.addCallback(_called, 'd4') - + # Make sure that d3 and d4 had to wait on the other two and were called # in order # NOTE(termie): there may be a race condition in this test if for some @@ -92,25 +92,31 @@ class ProcessTestCase(test.TrialTestCase): def test_kill_long_process(self): pool = process.ProcessPool(2) - - d1 = pool.simpleExecute('sleep 1') - d2 = pool.simpleExecute('sleep 0.005') + + d1 = pool.simple_execute('sleep 1') + d2 = pool.simple_execute('sleep 0.005') timeout = reactor.callLater(0.1, self.fail, 'should have been killed') - + # kill d1 and wait on it to end then cancel the timeout d2.addCallback(lambda _: d1.process.signalProcess('KILL')) d2.addCallback(lambda _: d1) d2.addBoth(lambda _: timeout.active() and timeout.cancel()) d2.addErrback(self.fail) return d2 - + def test_process_exit_is_contained(self): pool = process.ProcessPool(2) - - d1 = pool.simpleExecute('sleep 1') + + d1 = pool.simple_execute('sleep 1') d1.addCallback(lambda x: self.fail('should have errbacked')) d1.addErrback(lambda fail: fail.trap(IOError)) reactor.callLater(0.05, d1.process.signalProcess, 'KILL') - + return d1 + + def test_shared_pool_is_singleton(self): + pool1 = process.SharedPool() + pool2 = process.SharedPool() + self.assert_(id(pool1) == id(pool2)) + diff --git a/nova/utils.py b/nova/utils.py index c5b935673..9ecceafe0 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -94,7 +94,7 @@ def generate_uid(topic, size=8): def generate_mac(): - mac = [0x00, 0x16, 0x3e, random.randint(0x00, 0x7f), + mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f), random.randint(0x00, 0xff), random.randint(0x00, 0xff) ] return ':'.join(map(lambda x: "%02x" % x, mac)) diff --git a/nova/virt/images.py b/nova/virt/images.py index 0b11c134e..fd74349b1 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -31,22 +31,20 @@ flags.DEFINE_bool('use_s3', True, 'whether to get images from s3 or use local copy') -def fetch(pool, image, path): +def fetch(image, path): if FLAGS.use_s3: f = _fetch_s3_image else: f = _fetch_local_image - return f(pool, image, path) + return f(image, path) -def _fetch_s3_image(pool, image, path): +def _fetch_s3_image(image, path): url = _image_url('%s/image' % image) - d = pool.simpleExecute('curl --silent %s -o %s' % (url, path)) - return d + return process.simple_execute('curl --silent %s -o %s' % (url, path)) -def _fetch_local_image(pool, image, path): +def _fetch_local_image(image, path): source = _image_path('%s/image' % image) - d = pool.simpleExecute('cp %s %s' % (source, path)) - return d + return process.simple_execute('cp %s %s' % (source, path)) def _image_path(path): return os.path.join(FLAGS.images_path, path) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 74fec650e..39ed9bd78 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -175,8 +175,8 @@ class LibvirtConnection(object): basepath = lambda x='': self.basepath(instance, x) # ensure directories exist and are writable - yield self._pool.simpleExecute('mkdir -p %s' % basepath()) - yield self._pool.simpleExecute('chmod 0777 %s' % basepath()) + yield process.simple_execute('mkdir -p %s' % basepath()) + yield process.simple_execute('chmod 0777 %s' % basepath()) # TODO(termie): these are blocking calls, it would be great @@ -187,15 +187,16 @@ class LibvirtConnection(object): f.close() if not os.path.exists(basepath('disk')): - yield images.fetch(self._pool, data['image_id'], basepath('disk-raw')) + yield images.fetch(data['image_id'], basepath('disk-raw')) if not os.path.exists(basepath('kernel')): - yield images.fetch(self._pool, data['kernel_id'], basepath('kernel')) + yield images.fetch(data['kernel_id'], basepath('kernel')) if not os.path.exists(basepath('ramdisk')): - yield images.fetch(self._pool, data['ramdisk_id'], basepath('ramdisk')) + yield images.fetch(data['ramdisk_id'], basepath('ramdisk')) - execute = lambda cmd, input=None: self._pool.simpleExecute(cmd=cmd, - input=input, - error_ok=1) + execute = lambda cmd, input=None: \ + process.simple_execute(cmd=cmd, + input=input, + error_ok=1) key = data['key_data'] net = None @@ -212,7 +213,7 @@ class LibvirtConnection(object): yield disk.inject_data(basepath('disk-raw'), key, net, execute=execute) if os.path.exists(basepath('disk')): - yield self._pool.simpleExecute('rm -f %s' % basepath('disk')) + yield process.simple_execute('rm -f %s' % basepath('disk')) bytes = (instance_types.INSTANCE_TYPES[data['instance_type']]['local_gb'] * 1024 * 1024 * 1024) diff --git a/run_tests.py b/run_tests.py index eb26459c5..db8a582ea 100644 --- a/run_tests.py +++ b/run_tests.py @@ -39,6 +39,7 @@ Due to our use of multiprocessing it we frequently get some ignorable """ import __main__ +import os import sys @@ -66,6 +67,9 @@ FLAGS = flags.FLAGS flags.DEFINE_bool('flush_db', True, 'Flush the database before running fake tests') +flags.DEFINE_string('tests_stderr', 'run_tests.err.log', + 'Path to where to pipe STDERR during test runs. Default = "run_tests.err.log"') + if __name__ == '__main__': OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) config = OptionsClass() @@ -85,6 +89,11 @@ if __name__ == '__main__': else: from nova.tests.real_flags import * + # Establish redirect for STDERR + sys.stderr.flush() + err = open(FLAGS.tests_stderr, 'w+', 0) + os.dup2(err.fileno(), sys.stderr.fileno()) + if len(argv) == 1 and len(config['tests']) == 0: # If no tests were specified run the ones imported in this file # NOTE(termie): "tests" is not a flag, just some Trial related stuff |