From 0a5c5f8130fce42b1edcfb67c702e25f51aefa13 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 12 Jun 2010 18:24:27 -0700 Subject: implement image serving in objectstore so nginx isn't required in development reviewed by yosh --- nova/objectstore/handler.py | 26 ++++++++++++++++++++++++++ nova/objectstore/image.py | 4 ++++ 2 files changed, 30 insertions(+) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index a7fff12fc..3f00bb0c4 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -71,6 +71,7 @@ class Application(web.Application): def __init__(self, user_manager): web.Application.__init__(self, [ (r"/", RootHandler), + (r"/_images/(.+)", ImageDownloadHandler), (r"/_images/", ImageHandler), (r"/([^/]+)/(.+)", ObjectHandler), (r"/([^/]+)/", BucketHandler), @@ -224,6 +225,31 @@ class ObjectHandler(BaseRequestHandler): self.finish() +class ImageDownloadHandler(BaseRequestHandler): + SUPPORTED_METHODS = ("GET", ) + + @catch_nova_exceptions + def get(self, image_id): + """ send the decrypted image file + + streaming content through python is slow and should only be used + in development mode. You should serve files via a web server + in production. + """ + + self.set_header("Content-Type", "application/octet-stream") + + READ_SIZE = 64*1024 + + img = image.Image(image_id) + with open(img.image_path, 'rb') as fp: + s = fp.read(READ_SIZE) + while s: + self.write(s) + s = fp.read(READ_SIZE) + + self.finish() + class ImageHandler(BaseRequestHandler): SUPPORTED_METHODS = ("POST", "PUT", "GET", "DELETE") diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 892ada00c..b8dae4077 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -47,6 +47,10 @@ class Image(object): not os.path.isdir(self.path): raise exception.NotFound + @property + def image_path(self): + return os.path.join(self.path, 'image') + def delete(self): for fn in ['info.json', 'image']: try: -- cgit From aabc316aa734107e82a6dd0317028f9a254f24bc Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 15 Jun 2010 10:42:34 -0700 Subject: first go at moving from tornado to twisted --- nova/objectstore/handler.py | 169 ++++++++++++++++++++++++++++++-------------- 1 file changed, 114 insertions(+), 55 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 3f00bb0c4..d9369afbf 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -33,18 +33,26 @@ S3 client with this module:: """ import datetime -import os -import urllib -import json import logging +import json import multiprocessing +import os +import re +import time +import urllib from nova import vendor -from tornado import escape, web + +from twisted.web import resource +from twisted.web import server +from twisted.internet import reactor + +from tornado import escape # FIXME(ja): move to non-tornado escape from nova import exception from nova import flags +from nova.auth import users from nova.endpoint import api from nova.objectstore import bucket from nova.objectstore import image @@ -53,54 +61,102 @@ from nova.objectstore import image FLAGS = flags.FLAGS -def catch_nova_exceptions(target): - # FIXME: find a way to wrap all handlers in the web.Application.__init__ ? - def wrapper(*args, **kwargs): - try: - return target(*args, **kwargs) - except exception.NotFound: - raise web.HTTPError(404) - except exception.NotAuthorized: - raise web.HTTPError(403) - - return wrapper +class Application(resource.Resource): + """Implementation of an S3-like storage server based on local files.""" + isLeaf = True -class Application(web.Application): - """Implementation of an S3-like storage server based on local files.""" - def __init__(self, user_manager): - web.Application.__init__(self, [ - (r"/", RootHandler), + def __init__(self): + # fixme(ja): optomize by compiling regexps? + self.handlers = [ (r"/_images/(.+)", ImageDownloadHandler), (r"/_images/", ImageHandler), (r"/([^/]+)/(.+)", ObjectHandler), (r"/([^/]+)/", BucketHandler), - ]) + (r"/", RootHandler), + ] self.buckets_path = os.path.abspath(FLAGS.buckets_path) self.images_path = os.path.abspath(FLAGS.images_path) if not os.path.exists(self.buckets_path): - raise Exception("buckets_path does not exist") + raise Exception("buckets_path %s does not exist" % self.buckets_path) if not os.path.exists(self.images_path): - raise Exception("images_path does not exist") - self.user_manager = user_manager + raise Exception("images_path %s does not exist" % self.images_path) + + def render_GET(self, request): + return self.route(request) + + def render_PUT(self, request): + return self.route(request) + + def render_POST(self, request): + return self.route(request) + + def render_DELETE(self, request): + return self.route(request) + + def route(self, request): + start_time = time.time() + + for regexp, handler in self.handlers: + match = re.search(regexp, request.path) + if match: + try: + print 'match: %s' % request.path + func = getattr(handler(request), request.method.lower()) + #print 'func: %s' % func + params = match.groups() + #print 'args: %s' % str(params) + response = func(*params) + #print 'resp: %s' % response + except exception.NotFound: + request.setResponseCode(404) + response = 'Not Found' + except exception.NotAuthorized: + request.setResponseCode(403) + response = 'Not Authorized' + except Exception, e: + request.setResponseCode(500) + response = 'Internal Error: %s' % e + break + + duration = (time.time() - start_time) * 1000 + logging.info("%d %s %s %0.1fms %s" % (request.code, request.method, request.uri, + duration, str(handler).split('.')[-1].split("'")[0])) + return response + + +class BaseRequestHandler(object): + SUPPORTED_METHODS = ("PUT", "GET", "DELETE", "HEAD") + def __init__(self, request): + self.request = request -class BaseRequestHandler(web.RequestHandler): - SUPPORTED_METHODS = ("PUT", "GET", "DELETE", "HEAD") + def set_header(self, name, value): + self.request.setHeader(name, value) + + def write(self, content): + self.request.write(content) + + def finish(self, content=None): + if content: + self.request.write(content) + self.request.finish() @property def context(self): if not hasattr(self, '_context'): try: # Authorization Header format: 'AWS :' - access, sep, secret = self.request.headers['Authorization'].split(' ')[1].rpartition(':') - (user, project) = self.application.user_manager.authenticate(access, secret, {}, self.request.method, self.request.host, self.request.path, False) + access, sep, secret = self.request.getHeader('Authorization').split(' ')[1].rpartition(':') + um = users.UserManager.instance() + print 'um %s' % um + (user, project) = um.authenticate(access, secret, {}, self.request.method, self.request.host, self.request.uri, False) # FIXME: check signature here! self._context = api.APIRequestContext(self, user, project) except exception.Error, ex: logging.debug("Authentication Failure: %s" % ex) - raise web.HTTPError(403) + raise exception.NotAuthorized return self._context def render_xml(self, value): @@ -138,6 +194,7 @@ class BaseRequestHandler(web.RequestHandler): class RootHandler(BaseRequestHandler): + def get(self): buckets = [b for b in bucket.Bucket.all() if b.is_authorized(self.context)] @@ -147,14 +204,13 @@ class RootHandler(BaseRequestHandler): class BucketHandler(BaseRequestHandler): - @catch_nova_exceptions def get(self, bucket_name): logging.debug("List keys for bucket %s" % (bucket_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized prefix = self.get_argument("prefix", u"") marker = self.get_argument("marker", u"") @@ -164,19 +220,21 @@ class BucketHandler(BaseRequestHandler): results = bucket_object.list_keys(prefix=prefix, marker=marker, max_keys=max_keys, terse=terse) self.render_xml({"ListBucketResult": results}) - @catch_nova_exceptions def put(self, bucket_name): logging.debug("Creating bucket %s" % (bucket_name)) + try: + print 'user is %s' % self.context + except Exception, e: + logging.exception(e) bucket.Bucket.create(bucket_name, self.context) self.finish() - @catch_nova_exceptions def delete(self, bucket_name): logging.debug("Deleting bucket %s" % (bucket_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized bucket_object.delete() self.set_status(204) @@ -184,14 +242,13 @@ class BucketHandler(BaseRequestHandler): class ObjectHandler(BaseRequestHandler): - @catch_nova_exceptions def get(self, bucket_name, object_name): logging.debug("Getting object: %s / %s" % (bucket_name, object_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized obj = bucket_object[urllib.unquote(object_name)] self.set_header("Content-Type", "application/unknown") @@ -199,26 +256,28 @@ class ObjectHandler(BaseRequestHandler): self.set_header("Etag", '"' + obj.md5 + '"') self.finish(obj.read()) - @catch_nova_exceptions def put(self, bucket_name, object_name): logging.debug("Putting object: %s / %s" % (bucket_name, object_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized key = urllib.unquote(object_name) - bucket_object[key] = self.request.body + print 'seeking' + self.request.content.seek(0, 0) + print 'writing' + bucket_object[key] = self.request.content.read() + print 'etag %s' % bucket_object[key].md5 self.set_header("Etag", '"' + bucket_object[key].md5 + '"') self.finish() - @catch_nova_exceptions def delete(self, bucket_name, object_name): logging.debug("Deleting object: %s / %s" % (bucket_name, object_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized del bucket_object[urllib.unquote(object_name)] self.set_status(204) @@ -228,7 +287,6 @@ class ObjectHandler(BaseRequestHandler): class ImageDownloadHandler(BaseRequestHandler): SUPPORTED_METHODS = ("GET", ) - @catch_nova_exceptions def get(self, image_id): """ send the decrypted image file @@ -239,21 +297,21 @@ class ImageDownloadHandler(BaseRequestHandler): self.set_header("Content-Type", "application/octet-stream") - READ_SIZE = 64*1024 + READ_SIZE = 1024*1024 img = image.Image(image_id) with open(img.image_path, 'rb') as fp: - s = fp.read(READ_SIZE) - while s: - self.write(s) - s = fp.read(READ_SIZE) + chunk = fp.read(READ_SIZE) + while chunk: + self.write(chunk) + self.flush() + chunk = fp.read(READ_SIZE) self.finish() class ImageHandler(BaseRequestHandler): SUPPORTED_METHODS = ("POST", "PUT", "GET", "DELETE") - @catch_nova_exceptions def get(self): """ returns a json listing of all images that a user has permissions to see """ @@ -262,7 +320,6 @@ class ImageHandler(BaseRequestHandler): self.finish(json.dumps([i.metadata for i in images])) - @catch_nova_exceptions def put(self): """ create a new registered image """ @@ -272,20 +329,19 @@ class ImageHandler(BaseRequestHandler): image_path = os.path.join(FLAGS.images_path, image_id) if not image_path.startswith(FLAGS.images_path) or \ os.path.exists(image_path): - raise web.HTTPError(403) + raise exception.NotAuthorized bucket_object = bucket.Bucket(image_location.split("/")[0]) manifest = image_location[len(image_location.split('/')[0])+1:] if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized p = multiprocessing.Process(target=image.Image.create,args= (image_id, image_location, self.context)) p.start() self.finish() - @catch_nova_exceptions def post(self): """ update image attributes: public/private """ @@ -295,21 +351,24 @@ class ImageHandler(BaseRequestHandler): image_object = image.Image(image_id) if not image.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized image_object.set_public(operation=='add') self.finish() - @catch_nova_exceptions def delete(self): """ delete a registered image """ image_id = self.get_argument("image_id", u"") image_object = image.Image(image_id) if not image.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized image_object.delete() self.set_status(204) + +factory = server.Site(Application()) +reactor.listenTCP(3333, factory) +reactor.run() -- cgit From b81b0f2ecf3ef9bcba71a581ccd0ed3729398fba Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sun, 20 Jun 2010 15:08:25 -0700 Subject: update spacing --- docs/getting.started.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/getting.started.rst b/docs/getting.started.rst index 9d7808a27..63e435906 100644 --- a/docs/getting.started.rst +++ b/docs/getting.started.rst @@ -1,12 +1,12 @@ .. Copyright [2010] [Anso Labs, LLC] - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,19 +67,19 @@ Installation # ON THE CLOUD CONTROLLER apt-get install -y rabbitmq-server dnsmasq nginx # build redis from 2.0.0-rc1 source - # setup ldap (slap.sh as root will remove ldap and reinstall it) - NOVA_PATH/nova/auth/slap.sh + # setup ldap (slap.sh as root will remove ldap and reinstall it) + NOVA_PATH/nova/auth/slap.sh /etc/init.d/rabbitmq-server start # ON VOLUME NODE: - apt-get install -y vblade-persist + apt-get install -y vblade-persist # ON THE COMPUTE NODE: apt-get install -y kpartx kvm # optional packages - apt-get install -y euca2ools - + apt-get install -y euca2ools + Configuration --------------- @@ -90,10 +90,10 @@ ON CLOUD CONTROLLER :: iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 - iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE + iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE -* Configure NginX proxy (/etc/nginx/sites-enabled/default) +* Configure NginX proxy (/etc/nginx/sites-enabled/default) :: @@ -137,8 +137,8 @@ Launch servers Launch nova components -* api_worker -* s3_worker -* node_worker -* storage_worker +* nova-api +* nova-compute +* nova-objectstore +* nova-volume -- cgit From 0b396e1315a76112ab978a677e96d7b3a371faa9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 15 Jul 2010 23:11:33 +0000 Subject: Fixes to dhcp lease code to use a flagfile --- bin/dhcpleasor.py | 90 ---------------------------------------- bin/nova-dhcpbridge | 94 ++++++++++++++++++++++++++++++++++++++++++ debian/nova-api.conf | 3 +- debian/nova-api.install | 1 + debian/nova-compute.conf | 2 - debian/nova-dhcp.conf | 2 + debian/nova-objectstore.conf | 1 - debian/nova-volume.conf | 1 - nova/compute/linux_net.py | 16 ++++--- nova/compute/network.py | 6 +-- nova/tests/network_unittest.py | 23 +++++++---- 11 files changed, 127 insertions(+), 112 deletions(-) delete mode 100755 bin/dhcpleasor.py create mode 100755 bin/nova-dhcpbridge create mode 100644 debian/nova-dhcp.conf diff --git a/bin/dhcpleasor.py b/bin/dhcpleasor.py deleted file mode 100755 index 31f4433fb..000000000 --- a/bin/dhcpleasor.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -dhcpleasor.py - -Handle lease database updates from DHCP servers. -""" - -import sys -import os -import logging -sys.path.append(os.path.abspath(os.path.join(__file__, "../../"))) - -logging.debug(sys.path) -import getopt -from os import environ -from nova.compute import linux_net -from nova.compute import network -from nova import rpc - -from nova import flags -FLAGS = flags.FLAGS - - -def add_lease(mac, ip, hostname, interface): - if FLAGS.fake_rabbit: - network.lease_ip(ip) - else: - rpc.cast(FLAGS.cloud_topic, {"method": "lease_ip", - "args" : {"address": ip}}) - -def old_lease(mac, ip, hostname, interface): - logging.debug("Adopted old lease or got a change of mac/hostname") - -def del_lease(mac, ip, hostname, interface): - if FLAGS.fake_rabbit: - network.release_ip(ip) - else: - rpc.cast(FLAGS.cloud_topic, {"method": "release_ip", - "args" : {"address": ip}}) - -def init_leases(interface): - net = network.get_network_by_interface(interface) - res = "" - for host_name in net.hosts: - res += "%s\n" % linux_net.hostDHCP(net, host_name, net.hosts[host_name]) - return res - - -def main(argv=None): - if argv is None: - argv = sys.argv - interface = environ.get('DNSMASQ_INTERFACE', 'br0') - if int(environ.get('TESTING', '0')): - FLAGS.fake_rabbit = True - FLAGS.redis_db = 8 - FLAGS.network_size = 32 - FLAGS.fake_libvirt=True - FLAGS.fake_network=True - FLAGS.fake_users = True - action = argv[1] - if action in ['add','del','old']: - mac = argv[2] - ip = argv[3] - hostname = argv[4] - logging.debug("Called %s for mac %s with ip %s and hostname %s on interface %s" % (action, mac, ip, hostname, interface)) - globals()[action+'_lease'](mac, ip, hostname, interface) - else: - print init_leases(interface) - exit(0) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge new file mode 100755 index 000000000..cd0eab2a8 --- /dev/null +++ b/bin/nova-dhcpbridge @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +nova-dhcpbridge + +Handle lease database updates from DHCP servers. +""" + +import logging +import os +import sys + +#TODO(joshua): there is concern that the user dnsmasq runs under will not +# have nova in the path. This should be verified and if it is +# not true the ugly line below can be removed +sys.path.append(os.path.abspath(os.path.join(__file__, "../../"))) + +from nova import rpc +from nova import utils + +from nova.compute import linux_net +from nova.compute import network + +from nova import flags +FLAGS = flags.FLAGS + + +def add_lease(mac, ip, hostname, interface): + if FLAGS.fake_rabbit: + network.lease_ip(ip) + else: + rpc.cast(FLAGS.cloud_topic, {"method": "lease_ip", + "args" : {"address": ip}}) + +def old_lease(mac, ip, hostname, interface): + logging.debug("Adopted old lease or got a change of mac/hostname") + +def del_lease(mac, ip, hostname, interface): + if FLAGS.fake_rabbit: + network.release_ip(ip) + else: + rpc.cast(FLAGS.cloud_topic, {"method": "release_ip", + "args" : {"address": ip}}) + +def init_leases(interface): + net = network.get_network_by_interface(interface) + res = "" + for host_name in net.hosts: + res += "%s\n" % linux_net.hostDHCP(net, host_name, net.hosts[host_name]) + return res + + +def main(): + flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) + utils.default_flagfile(flagfile) + argv = FLAGS(sys.argv) + interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') + if int(os.environ.get('TESTING', '0')): + FLAGS.fake_rabbit = True + FLAGS.redis_db = 8 + FLAGS.network_size = 32 + FLAGS.fake_libvirt=True + FLAGS.fake_network=True + FLAGS.fake_users = True + action = argv[1] + if action in ['add','del','old']: + mac = argv[2] + ip = argv[3] + hostname = argv[4] + logging.debug("Called %s for mac %s with ip %s and hostname %s on interface %s" % (action, mac, ip, hostname, interface)) + globals()[action+'_lease'](mac, ip, hostname, interface) + else: + print init_leases(interface) + exit(0) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/debian/nova-api.conf b/debian/nova-api.conf index 9cd4051b1..d0b796878 100644 --- a/debian/nova-api.conf +++ b/debian/nova-api.conf @@ -1,5 +1,6 @@ --daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys +--networks_path=/var/lib/nova/networks +--dhcpbridge_flagfile=/etc/nova/nova-dhcpbridge.conf --fake_users=1 ---datastore_path=/var/lib/nova/keeper diff --git a/debian/nova-api.install b/debian/nova-api.install index 02dbda02d..89615d302 100644 --- a/debian/nova-api.install +++ b/debian/nova-api.install @@ -1,2 +1,3 @@ bin/nova-api usr/bin debian/nova-api.conf etc/nova +debian/nova-dhcpbridge.conf etc/nova diff --git a/debian/nova-compute.conf b/debian/nova-compute.conf index e4ca3fe95..d862f2328 100644 --- a/debian/nova-compute.conf +++ b/debian/nova-compute.conf @@ -1,8 +1,6 @@ --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys ---datastore_path=/var/lib/nova/keeper --instances_path=/var/lib/nova/instances ---networks_path=/var/lib/nova/networks --simple_network_template=/usr/share/nova/interfaces.template --libvirt_xml_template=/usr/share/nova/libvirt.xml.template --vpn_client_template=/usr/share/nova/client.ovpn.template diff --git a/debian/nova-dhcp.conf b/debian/nova-dhcp.conf new file mode 100644 index 000000000..0aafe7549 --- /dev/null +++ b/debian/nova-dhcp.conf @@ -0,0 +1,2 @@ +--networks_path=/var/lib/nova/networks +--fake_users=1 diff --git a/debian/nova-objectstore.conf b/debian/nova-objectstore.conf index af3271d3b..03f5df051 100644 --- a/debian/nova-objectstore.conf +++ b/debian/nova-objectstore.conf @@ -1,7 +1,6 @@ --daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys ---datastore_path=/var/lib/nova/keeper --fake_users=1 --images_path=/var/lib/nova/images --buckets_path=/var/lib/nova/buckets diff --git a/debian/nova-volume.conf b/debian/nova-volume.conf index af3271d3b..03f5df051 100644 --- a/debian/nova-volume.conf +++ b/debian/nova-volume.conf @@ -1,7 +1,6 @@ --daemonize=1 --ca_path=/var/lib/nova/CA --keys_path=/var/lib/nova/keys ---datastore_path=/var/lib/nova/keeper --fake_users=1 --images_path=/var/lib/nova/images --buckets_path=/var/lib/nova/buckets diff --git a/nova/compute/linux_net.py b/nova/compute/linux_net.py index 358f184af..2e5f2519f 100644 --- a/nova/compute/linux_net.py +++ b/nova/compute/linux_net.py @@ -27,12 +27,16 @@ import subprocess from nova import flags FLAGS=flags.FLAGS -def execute(cmd): +flags.DEFINE_string('dhcpbridge_flagfile', + '/etc/nova-dhcpbridge.conf', + 'location of flagfile for dhcpbridge') + +def execute(cmd, addl_env=None): if FLAGS.fake_network: logging.debug("FAKE NET: %s" % cmd) return "fake", 0 else: - return nova.utils.execute(cmd) + return nova.utils.execute(cmd, addl_env=addl_env) def runthis(desc, cmd): if FLAGS.fake_network: @@ -60,7 +64,7 @@ def remove_rule(cmd): def bind_public_ip(ip, interface): runthis("Binding IP to interface: %s", "sudo ip addr add %s dev %s" % (ip, interface)) - + def unbind_public_ip(ip, interface): runthis("Binding IP to interface: %s", "sudo ip addr del %s dev %s" % (ip, interface)) @@ -98,7 +102,7 @@ def dnsmasq_cmd(net): ' --except-interface=lo', ' --dhcp-range=%s,static,600s' % (net.dhcp_range_start), ' --dhcp-hostsfile=%s' % dhcp_file(net['vlan'], 'conf'), - ' --dhcp-script=%s' % bin_file('dhcpleasor.py'), + ' --dhcp-script=%s' % bin_file('nova-dhcpbridge'), ' --leasefile-ro'] return ''.join(cmd) @@ -138,7 +142,9 @@ def start_dnsmasq(network): if os.path.exists(lease_file): os.unlink(lease_file) - Popen(dnsmasq_cmd(network).split(" ")) + # FLAGFILE in env + env = {'FLAGFILE' : FLAGS.dhcpbridge_flagfile} + execute(dnsmasq_cmd(network), addl_env=env) def stop_dnsmasq(network): """ stops the dnsmasq instance for a given network """ diff --git a/nova/compute/network.py b/nova/compute/network.py index 8592d7af7..3904fcff0 100644 --- a/nova/compute/network.py +++ b/nova/compute/network.py @@ -253,7 +253,7 @@ class BaseNetwork(datastore.BasicModel): raise compute_exception.NoMoreAddresses("Project %s with network %s" % (project_id, str(self.network))) - def lease_ip(self, ip_str): + def lease_ip(self, ip_str): logging.debug("Leasing allocated IP %s" % (ip_str)) def release_ip(self, ip_str): @@ -563,10 +563,10 @@ def allocate_ip(user_id, project_id, mac): def deallocate_ip(address): return get_network_by_address(address).deallocate_ip(address) - + def release_ip(address): return get_network_by_address(address).release_ip(address) - + def lease_ip(address): return get_network_by_address(address).lease_ip(address) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index dd1966ffa..d24ae133c 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -18,19 +18,18 @@ import os import logging -import unittest from nova import vendor import IPy from nova import flags from nova import test -from nova import exception -from nova.compute.exception import NoMoreAddresses -from nova.compute import network -from nova.auth import users from nova import utils +from nova.auth import users +from nova.compute import network +from nova.compute.exception import NoMoreAddresses +FLAGS = flags.FLAGS class NetworkTestCase(test.TrialTestCase): def setUp(self): @@ -182,14 +181,20 @@ def binpath(script): class FakeDNSMasq(object): def issue_ip(self, mac, ip, hostname, interface): - cmd = "%s add %s %s %s" % (binpath('dhcpleasor.py'), mac, ip, hostname) - env = {'DNSMASQ_INTERFACE': interface, 'TESTING' : '1'} + cmd = "%s add %s %s %s" % (binpath('nova-dhcpbridge'), + mac, ip, hostname) + env = {'DNSMASQ_INTERFACE': interface, + 'TESTING' : '1', + 'FLAGFILE' : FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s " % (out, err)) def release_ip(self, mac, ip, hostname, interface): - cmd = "%s del %s %s %s" % (binpath('dhcpleasor.py'), mac, ip, hostname) - env = {'DNSMASQ_INTERFACE': interface, 'TESTING' : '1'} + cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), + mac, ip, hostname) + env = {'DNSMASQ_INTERFACE': interface, + 'TESTING' : '1', + 'FLAGFILE' : FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("RELEASE_IP: %s, %s " % (out, err)) -- cgit From c3505507a5b4c0d88164e8e6dfea405c902004ff Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Fri, 16 Jul 2010 10:03:22 -0500 Subject: Adds a flag to redirect STDERR when running run_tests.py. Defaults to a truncate-on-write logfile named run_tests.err.log. Adds ignore rule for generated errlog file. --- .bzrignore | 1 + run_tests.py | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 .bzrignore diff --git a/.bzrignore b/.bzrignore new file mode 100644 index 000000000..93fc868a3 --- /dev/null +++ b/.bzrignore @@ -0,0 +1 @@ +run_tests.err.log diff --git a/run_tests.py b/run_tests.py index bd1587d43..53d4d7ebe 100644 --- a/run_tests.py +++ b/run_tests.py @@ -39,6 +39,7 @@ Due to our use of multiprocessing it we frequently get some ignorable """ import __main__ +import os import sys from nova import vendor @@ -66,6 +67,9 @@ FLAGS = flags.FLAGS flags.DEFINE_bool('flush_db', True, 'Flush the database before running fake tests') +flags.DEFINE_string('tests_stderr', 'run_tests.err.log', + 'Path to where to pipe STDERR during test runs. Default = "run_tests.err.log"') + if __name__ == '__main__': OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) config = OptionsClass() @@ -85,6 +89,11 @@ if __name__ == '__main__': else: from nova.tests.real_flags import * + # Establish redirect for STDERR + sys.stderr.flush() + err = open(FLAGS.tests_stderr, 'w+', 0) + os.dup2(err.fileno(), sys.stderr.fileno()) + if len(argv) == 1 and len(config['tests']) == 0: # If no tests were specified run the ones imported in this file # NOTE(termie): "tests" is not a flag, just some Trial related stuff -- cgit From 680446f0acd8aa7820974d32b5b4093a6d4e7a10 Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Fri, 16 Jul 2010 12:00:15 -0500 Subject: Adds a fix to the idempotency of the test_too_many_addresses test case by adding a simple property to the BaseNetwork class and calculating the number of available IPs by asking the network class to tell the test how many static and preallocated IP addresses are in use before entering the loop to "blow up" the address allocation... --- nova/compute/network.py | 7 ++++++- nova/tests/network_unittest.py | 29 +++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/nova/compute/network.py b/nova/compute/network.py index 8592d7af7..2b271d0ca 100644 --- a/nova/compute/network.py +++ b/nova/compute/network.py @@ -164,6 +164,7 @@ class Vlan(datastore.BasicModel): class BaseNetwork(datastore.BasicModel): override_type = 'network' + NUM_STATIC_IPS = 3 # Network, Gateway, and CloudPipe @property def identifier(self): @@ -239,11 +240,15 @@ class BaseNetwork(datastore.BasicModel): def available(self): # the .2 address is always CloudPipe # and the top are for vpn clients - for idx in range(3, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): + for idx in range(self.num_static_ips, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): address = str(self.network[idx]) if not address in self.hosts.keys(): yield str(address) + @property + def num_static_ips(self): + return BaseNetwork.NUM_STATIC_IPS + def allocate_ip(self, user_id, project_id, mac): for address in self.available: logging.debug("Allocating IP %s to %s" % (address, project_id)) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index dd1966ffa..441a78097 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -150,21 +150,42 @@ class NetworkTestCase(test.TrialTestCase): def test_too_many_addresses(self): """ - Network size is 32, there are 5 addresses reserved for VPN. - So we should get 23 usable addresses + Here, we test that a proper NoMoreAddresses exception is raised. + + However, the number of available IP addresses depends on the test + environment's setup. + + Network size is 32. + + There are FLAGS.cnt_vpn_clients addresses reserved for VPN (NUM_RESERVED_VPN_IPS) + + And there are NUM_CONST_IPS that are always reserved by Nova for the necessary + services (API, CloudPipe, etc) + + So we should get 32 - (NUM_STATIC_IPS + + NUM_PREALLOCATED_IPS + + NUM_RESERVED_VPN_IPS) + usable addresses """ net = network.get_project_network("project0", "default") + + # Determine expected number of available IP addresses + num_static_ips = net.num_static_ips + num_preallocated_ips = len(net.hosts.keys()) + num_reserved_vpn_ips = flags.FLAGS.cnt_vpn_clients + num_available_ips = 32 - (num_static_ips + num_preallocated_ips + num_reserved_vpn_ips) + hostname = "toomany-hosts" macs = {} addresses = {} - for i in range(0, 22): + for i in range(0, (num_available_ips - 1)): macs[i] = utils.generate_mac() addresses[i] = network.allocate_ip("netuser", "project0", macs[i]) self.dnsmasq.issue_ip(macs[i], addresses[i], hostname, net.bridge_name) self.assertRaises(NoMoreAddresses, network.allocate_ip, "netuser", "project0", utils.generate_mac()) - for i in range(0, 22): + for i in range(0, (num_available_ips - 1)): rv = network.deallocate_ip(addresses[i]) self.dnsmasq.release_ip(macs[i], addresses[i], hostname, net.bridge_name) -- cgit From a5668a0e8ebc96d01ec7a26229079ccda6032e75 Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Fri, 16 Jul 2010 12:41:13 -0500 Subject: Quick fix to variable names for consistency in documentation... --- nova/tests/network_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 441a78097..3cfba2cf9 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -159,8 +159,8 @@ class NetworkTestCase(test.TrialTestCase): There are FLAGS.cnt_vpn_clients addresses reserved for VPN (NUM_RESERVED_VPN_IPS) - And there are NUM_CONST_IPS that are always reserved by Nova for the necessary - services (API, CloudPipe, etc) + And there are NUM_STATIC_IPS that are always reserved by Nova for the necessary + services (gateway, CloudPipe, etc) So we should get 32 - (NUM_STATIC_IPS + NUM_PREALLOCATED_IPS + -- cgit From 4b15a647f8153c493fb697eebc4ab17412142d67 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 16 Jul 2010 13:39:17 -0500 Subject: Make S3 API handler more idiomatic Twisted Web-y. --- nova/objectstore/handler.py | 389 ++++++++++++++++++-------------------------- 1 file changed, 158 insertions(+), 231 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 12ff9763b..50c56c83c 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -1,10 +1,11 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. # -# Copyright 2009 Facebook +# Copyright 2010 OpenStack LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2009 Facebook # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -48,8 +49,8 @@ import urllib from nova import vendor -from twisted.web import resource -from twisted.web import server +from twisted.web.resource import Resource +from twisted.web import server, static from twisted.internet import reactor from tornado import escape # FIXME(ja): move to non-tornado escape @@ -63,273 +64,197 @@ from nova.objectstore import image FLAGS = flags.FLAGS - - -class Application(resource.Resource): +FLAGS.fake_users = True + +def render_xml(request, value): + assert isinstance(value, dict) and len(value) == 1 + request.setHeader("Content-Type", "application/xml; charset=UTF-8") + + name = value.keys()[0] + request.write('\n') + request.write('<' + escape.utf8(name) + + ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') + _render_parts(value.values()[0], request.write) + request.write('') + request.finish() + +def finish(request, content=None): + if content: + request.write(content) + request.finish() + +def _render_parts(value, write_cb): + if isinstance(value, basestring): + write_cb(escape.xhtml_escape(value)) + elif isinstance(value, int) or isinstance(value, long): + write_cb(str(value)) + elif isinstance(value, datetime.datetime): + write_cb(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) + elif isinstance(value, dict): + for name, subvalue in value.iteritems(): + if not isinstance(subvalue, list): + subvalue = [subvalue] + for subsubvalue in subvalue: + write_cb('<' + escape.utf8(name) + '>') + _render_parts(subsubvalue, write_cb) + write_cb('') + else: + raise Exception("Unknown S3 value type %r", value) + +def get_argument(request, key, default_value): + if key in request.args: + return request.args[key][0] + return default_value + +def get_context(request): + try: + # Authorization Header format: 'AWS :' + access, sep, secret = request.getHeader('Authorization').split(' ')[1].rpartition(':') + um = users.UserManager.instance() + print 'um %s' % um + (user, project) = um.authenticate(access, secret, {}, request.method, request.host, request.uri, False) + # FIXME: check signature here! + return api.APIRequestContext(None, user, project) + except exception.Error, ex: + logging.debug("Authentication Failure: %s" % ex) + raise exception.NotAuthorized + +class S3(Resource): """Implementation of an S3-like storage server based on local files.""" + def getChild(self, name, request): + request.context = get_context(request) - isLeaf = True - - def __init__(self): - # fixme(ja): optomize by compiling regexps? - self.handlers = [ - (r"/_images/(.+)", ImageDownloadHandler), - (r"/_images/", ImageHandler), - (r"/([^/]+)/(.+)", ObjectHandler), - (r"/([^/]+)/", BucketHandler), - (r"/", RootHandler), - ] - self.buckets_path = os.path.abspath(FLAGS.buckets_path) - self.images_path = os.path.abspath(FLAGS.images_path) - - if not os.path.exists(self.buckets_path): - raise Exception("buckets_path %s does not exist" % self.buckets_path) - if not os.path.exists(self.images_path): - raise Exception("images_path %s does not exist" % self.images_path) - - def render_GET(self, request): - return self.route(request) - - def render_PUT(self, request): - return self.route(request) - - def render_POST(self, request): - return self.route(request) - - def render_DELETE(self, request): - return self.route(request) - - def route(self, request): - start_time = time.time() - - for regexp, handler in self.handlers: - match = re.search(regexp, request.path) - if match: - try: - print 'match: %s' % request.path - func = getattr(handler(request), request.method.lower()) - #print 'func: %s' % func - params = match.groups() - #print 'args: %s' % str(params) - response = func(*params) - #print 'resp: %s' % response - except exception.NotFound: - request.setResponseCode(404) - response = 'Not Found' - except exception.NotAuthorized: - request.setResponseCode(403) - response = 'Not Authorized' - except Exception, e: - request.setResponseCode(500) - response = 'Internal Error: %s' % e - break - - duration = (time.time() - start_time) * 1000 - logging.info("%d %s %s %0.1fms %s" % (request.code, request.method, request.uri, - duration, str(handler).split('.')[-1].split("'")[0])) - return response - - -class BaseRequestHandler(object): - SUPPORTED_METHODS = ("PUT", "GET", "DELETE", "HEAD") - - def __init__(self, request): - self.request = request - - def set_header(self, name, value): - self.request.setHeader(name, value) - - def write(self, content): - self.request.write(content) - - def finish(self, content=None): - if content: - self.request.write(content) - self.request.finish() - - @property - def context(self): - if not hasattr(self, '_context'): - try: - # Authorization Header format: 'AWS :' - access, sep, secret = self.request.getHeader('Authorization').split(' ')[1].rpartition(':') - um = users.UserManager.instance() - print 'um %s' % um - (user, project) = um.authenticate(access, secret, {}, self.request.method, self.request.host, self.request.uri, False) - # FIXME: check signature here! - self._context = api.APIRequestContext(self, user, project) - except exception.Error, ex: - logging.debug("Authentication Failure: %s" % ex) - raise exception.NotAuthorized - return self._context - - def render_xml(self, value): - assert isinstance(value, dict) and len(value) == 1 - self.set_header("Content-Type", "application/xml; charset=UTF-8") - name = value.keys()[0] - parts = [] - parts.append('<' + escape.utf8(name) + - ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') - self._render_parts(value.values()[0], parts) - parts.append('') - self.finish('\n' + - ''.join(parts)) - - def _render_parts(self, value, parts=[]): - if isinstance(value, basestring): - parts.append(escape.xhtml_escape(value)) - elif isinstance(value, int) or isinstance(value, long): - parts.append(str(value)) - elif isinstance(value, datetime.datetime): - parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) - elif isinstance(value, dict): - for name, subvalue in value.iteritems(): - if not isinstance(subvalue, list): - subvalue = [subvalue] - for subsubvalue in subvalue: - parts.append('<' + escape.utf8(name) + '>') - self._render_parts(subsubvalue, parts) - parts.append('') + if name == '': + return self + elif name == '_images': + return ImageResource() else: - raise Exception("Unknown S3 value type %r", value) - - def head(self, *args, **kwargs): - return self.get(*args, **kwargs) - - -class RootHandler(BaseRequestHandler): + return BucketResource(name) - def get(self): - buckets = [b for b in bucket.Bucket.all() if b.is_authorized(self.context)] + def render_GET(self, request): + buckets = [b for b in bucket.Bucket.all() if b.is_authorized(request.context)] - self.render_xml({"ListAllMyBucketsResult": { + render_xml(request, {"ListAllMyBucketsResult": { "Buckets": {"Bucket": [b.metadata for b in buckets]}, }}) + return server.NOT_DONE_YET +class BucketResource(Resource): + def __init__(self, name): + Resource.__init__(self) + self.name = name -class BucketHandler(BaseRequestHandler): - def get(self, bucket_name): - logging.debug("List keys for bucket %s" % (bucket_name)) + def getChild(self, name, request): + if name == '': + return self + else: + return ObjectResource(bucket.Bucket(self.name), name) - bucket_object = bucket.Bucket(bucket_name) + def render_GET(self, request): + logging.debug("List keys for bucket %s" % (self.name)) - if not bucket_object.is_authorized(self.context): + bucket_object = bucket.Bucket(self.name) + + if not bucket_object.is_authorized(request.context): raise exception.NotAuthorized - prefix = self.get_argument("prefix", u"") - marker = self.get_argument("marker", u"") - max_keys = int(self.get_argument("max-keys", 1000)) - terse = int(self.get_argument("terse", 0)) + prefix = get_argument(request, "prefix", u"") + marker = get_argument(request, "marker", u"") + max_keys = int(get_argument(request, "max-keys", 1000)) + terse = int(get_argument(request, "terse", 0)) results = bucket_object.list_keys(prefix=prefix, marker=marker, max_keys=max_keys, terse=terse) - self.render_xml({"ListBucketResult": results}) + render_xml(request, {"ListBucketResult": results}) + return server.NOT_DONE_YET - def put(self, bucket_name): - logging.debug("Creating bucket %s" % (bucket_name)) + def render_PUT(self, request): + logging.debug("Creating bucket %s" % (self.name)) try: - print 'user is %s' % self.context + print 'user is %s' % request.context except Exception, e: logging.exception(e) - bucket.Bucket.create(bucket_name, self.context) - self.finish() + logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context)) + bucket.Bucket.create(self.name, request.context) + return '' - def delete(self, bucket_name): - logging.debug("Deleting bucket %s" % (bucket_name)) - bucket_object = bucket.Bucket(bucket_name) + def render_DELETE(self, request): + logging.debug("Deleting bucket %s" % (self.name)) + bucket_object = bucket.Bucket(self.name) - if not bucket_object.is_authorized(self.context): + if not bucket_object.is_authorized(request.context): raise exception.NotAuthorized bucket_object.delete() - self.set_status(204) - self.finish() + request.setResponseCode(204) + return '' -class ObjectHandler(BaseRequestHandler): - def get(self, bucket_name, object_name): - logging.debug("Getting object: %s / %s" % (bucket_name, object_name)) +class ObjectResource(Resource): + def __init__(self, bucket, name): + Resource.__init__(self) + self.bucket = bucket + self.name = name - bucket_object = bucket.Bucket(bucket_name) + def render_GET(self, request): + logging.debug("Getting object: %s / %s" % (self.bucket.name, self.name)) - if not bucket_object.is_authorized(self.context): + if not self.bucket.is_authorized(request.context): raise exception.NotAuthorized - obj = bucket_object[urllib.unquote(object_name)] - self.set_header("Content-Type", "application/unknown") - self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp(obj.mtime)) - self.set_header("Etag", '"' + obj.md5 + '"') - self.finish(obj.read()) + obj = self.bucket[urllib.unquote(self.name)] + request.setHeader("Content-Type", "application/unknown") + request.setHeader("Last-Modified", datetime.datetime.utcfromtimestamp(obj.mtime)) + request.setHeader("Etag", '"' + obj.md5 + '"') + return static.File(obj.path).render_GET(request) - def put(self, bucket_name, object_name): - logging.debug("Putting object: %s / %s" % (bucket_name, object_name)) - bucket_object = bucket.Bucket(bucket_name) + def render_PUT(self, request): + logging.debug("Putting object: %s / %s" % (self.bucket.name, self.name)) - if not bucket_object.is_authorized(self.context): + if not self.bucket.is_authorized(request.context): raise exception.NotAuthorized - key = urllib.unquote(object_name) - print 'seeking' - self.request.content.seek(0, 0) - print 'writing' - bucket_object[key] = self.request.content.read() - print 'etag %s' % bucket_object[key].md5 - self.set_header("Etag", '"' + bucket_object[key].md5 + '"') - self.finish() + key = urllib.unquote(self.name) + request.content.seek(0, 0) + self.bucket[key] = request.content.read() + request.setHeader("Etag", '"' + self.bucket[key].md5 + '"') + finish(request) + return server.NOT_DONE_YET - def delete(self, bucket_name, object_name): - logging.debug("Deleting object: %s / %s" % (bucket_name, object_name)) - bucket_object = bucket.Bucket(bucket_name) + def render_DELETE(self, request): + logging.debug("Deleting object: %s / %s" % (self.bucket.name, self.name)) - if not bucket_object.is_authorized(self.context): + if not self.bucket.is_authorized(request.context): raise exception.NotAuthorized - del bucket_object[urllib.unquote(object_name)] - self.set_status(204) - self.finish() - - -class ImageDownloadHandler(BaseRequestHandler): - SUPPORTED_METHODS = ("GET", ) + del self.bucket[urllib.unquote(self.name)] + request.setResponseCode(204) + return '' - @catch_nova_exceptions - def get(self, image_id): - """ send the decrypted image file - - streaming content through python is slow and should only be used - in development mode. You should serve files via a web server - in production. - """ - - self.set_header("Content-Type", "application/octet-stream") - - READ_SIZE = 1024*1024 - - img = image.Image(image_id) - with open(img.image_path, 'rb') as fp: - chunk = fp.read(READ_SIZE) - while chunk: - self.write(chunk) - self.flush() - chunk = fp.read(READ_SIZE) - - self.finish() +class ImageResource(Resource): + isLeaf = True -class ImageHandler(BaseRequestHandler): - SUPPORTED_METHODS = ("POST", "PUT", "GET", "DELETE") + def getChild(self, name, request): + if name == '': + return self + else: + request.setHeader("Content-Type", "application/octet-stream") + img = image.Image(name) + return static.File(img.image_path) - def get(self): + def render_GET(self, request): """ returns a json listing of all images that a user has permissions to see """ images = [i for i in image.Image.all() if i.is_authorized(self.context)] - self.finish(json.dumps([i.metadata for i in images])) + request.write(json.dumps([i.metadata for i in images])) + return server.NOT_DONE_YET - def put(self): + def render_PUT(self, request): """ create a new registered image """ - image_id = self.get_argument('image_id', u'') - image_location = self.get_argument('image_location', u'') + image_id = get_argument(request, 'image_id', u'') + image_location = get_argument(request, 'image_location', u'') image_path = os.path.join(FLAGS.images_path, image_id) if not image_path.startswith(FLAGS.images_path) or \ @@ -345,9 +270,9 @@ class ImageHandler(BaseRequestHandler): p = multiprocessing.Process(target=image.Image.register_aws_image, args=(image_id, image_location, self.context)) p.start() - self.finish() + return '' - def post(self): + def render_POST(self): """ update image attributes: public/private """ image_id = self.get_argument('image_id', u'') @@ -360,9 +285,9 @@ class ImageHandler(BaseRequestHandler): image_object.set_public(operation=='add') - self.finish() + return '' - def delete(self): + def render_DELETE(self): """ delete a registered image """ image_id = self.get_argument("image_id", u"") image_object = image.Image(image_id) @@ -372,8 +297,10 @@ class ImageHandler(BaseRequestHandler): image_object.delete() - self.set_status(204) + request.setResponseCode(204) + return '' -factory = server.Site(Application()) +root = S3() +factory = server.Site(root) reactor.listenTCP(3333, factory) reactor.run() -- cgit From ae9e4e81d992fb81c01acd2dfcb1cb3d32956041 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 19:12:36 +0000 Subject: Removed unused Pool from process.py, added a singleton pool called SharedPool, changed calls in node to use singleton pool --- nova/compute/node.py | 41 +++++++++++++++++++------------------ nova/process.py | 46 ++++++++++-------------------------------- nova/tests/process_unittest.py | 44 +++++++++++++++++++++++----------------- 3 files changed, 57 insertions(+), 74 deletions(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index d681ec661..3abd20120 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -87,7 +87,6 @@ class Node(object, service.Service): super(Node, self).__init__() self._instances = {} self._conn = self._get_connection() - self._pool = process.ProcessPool() self.instdir = model.InstanceDirectory() # TODO(joshua): This needs to ensure system state, specifically: modprobe aoe @@ -115,7 +114,7 @@ class Node(object, service.Service): # inst = self.instdir.get(instance_id) # return inst if self.instdir.exists(instance_id): - return Instance.fromName(self._conn, self._pool, instance_id) + return Instance.fromName(self._conn, instance_id) return None @exception.wrap_exception @@ -126,7 +125,7 @@ class Node(object, service.Service): for x in self._conn.listDomainsID()] for name in instance_names: try: - new_inst = Instance.fromName(self._conn, self._pool, name) + new_inst = Instance.fromName(self._conn, name) new_inst.update_state() except: pass @@ -136,7 +135,8 @@ class Node(object, service.Service): def describe_instances(self): retval = {} for inst in self.instdir.by_node(FLAGS.node_name): - retval[inst['instance_id']] = (Instance.fromName(self._conn, self._pool, inst['instance_id'])) + retval[inst['instance_id']] = ( + Instance.fromName(self._conn, inst['instance_id'])) return retval @defer.inlineCallbacks @@ -169,8 +169,7 @@ class Node(object, service.Service): inst['node_name'] = FLAGS.node_name inst.save() # TODO(vish) check to make sure the availability zone matches - new_inst = Instance(self._conn, name=instance_id, - pool=self._pool, data=inst) + new_inst = Instance(self._conn, name=instance_id, data=inst) logging.info("Instances current state is %s", new_inst.state) if new_inst.is_running(): raise exception.Error("Instance is already running") @@ -267,11 +266,8 @@ class Instance(object): SHUTOFF = 0x05 CRASHED = 0x06 - def __init__(self, conn, pool, name, data): + def __init__(self, conn, name, data): """ spawn an instance with a given name """ - # TODO(termie): pool should probably be a singleton instead of being passed - # here and in the classmethods - self._pool = pool self._conn = conn # TODO(vish): this can be removed after data has been updated # data doesn't seem to have a working iterator so in doesn't work @@ -324,11 +320,11 @@ class Instance(object): return libvirt_xml @classmethod - def fromName(cls, conn, pool, name): + def fromName(cls, conn, name): """ use the saved data for reloading the instance """ instdir = model.InstanceDirectory() instance = instdir.get(name) - return cls(conn=conn, pool=pool, name=name, data=instance) + return cls(conn=conn, name=name, data=instance) def set_state(self, state_code, state_description=None): self.datamodel['state'] = state_code @@ -450,12 +446,13 @@ class Instance(object): def _fetch_s3_image(self, image, path): url = _image_url('%s/image' % image) - d = self._pool.simpleExecute('curl --silent %s -o %s' % (url, path)) + d = process.SharedPool().simple_execute( + 'curl --silent %s -o %s' % (url, path)) return d def _fetch_local_image(self, image, path): source = _image_path('%s/image' % image) - d = self._pool.simpleExecute('cp %s %s' % (source, path)) + d = process.SharedPool().simple_execute('cp %s %s' % (source, path)) return d @defer.inlineCallbacks @@ -465,8 +462,10 @@ class Instance(object): basepath = self.basepath # ensure directories exist and are writable - yield self._pool.simpleExecute('mkdir -p %s' % basepath()) - yield self._pool.simpleExecute('chmod 0777 %s' % basepath()) + yield process.SharedPool().simple_execute( + 'mkdir -p %s' % basepath()) + yield process.SharedPool().simple_execute( + 'chmod 0777 %s' % basepath()) # TODO(termie): these are blocking calls, it would be great @@ -492,9 +491,10 @@ class Instance(object): if not os.path.exists(basepath('ramdisk')): yield _fetch_file(data['ramdisk_id'], basepath('ramdisk')) - execute = lambda cmd, input=None: self._pool.simpleExecute(cmd=cmd, - input=input, - error_ok=1) + execute = lambda cmd, input=None: \ + process.SharedPool().simple_execute(cmd=cmd, + input=input, + error_ok=1) key = data['key_data'] net = None @@ -511,7 +511,8 @@ class Instance(object): yield disk.inject_data(basepath('disk-raw'), key, net, execute=execute) if os.path.exists(basepath('disk')): - yield self._pool.simpleExecute('rm -f %s' % basepath('disk')) + yield process.SharedPool().simple_execute( + 'rm -f %s' % basepath('disk')) bytes = (INSTANCE_TYPES[data['instance_type']]['local_gb'] * 1024 * 1024 * 1024) diff --git a/nova/process.py b/nova/process.py index ff789a08a..ebfb2f4ba 100644 --- a/nova/process.py +++ b/nova/process.py @@ -85,7 +85,7 @@ class _BackRelay(protocol.ProcessProtocol): def errReceivedIsBad(self, text): if self.deferred is not None: self.onProcessEnded = defer.Deferred() - err = _UnexpectedErrorOutput(text, self.onProcessEnded) + err = UnexpectedErrorOutput(text, self.onProcessEnded) self.deferred.errback(failure.Failure(err)) self.deferred = None self.transport.loseConnection() @@ -152,8 +152,8 @@ def getProcessOutput(executable, args=None, env=None, path=None, reactor=None, d = defer.Deferred() p = BackRelayWithInput( d, startedDeferred=startedDeferred, error_ok=error_ok, input=input) - # VISH: commands come in as unicode, but self.executes needs - # strings or process.spawn raises a deprecation warning + # NOTE(vish): commands come in as unicode, but self.executes needs + # strings or process.spawn raises a deprecation warning executable = str(executable) if not args is None: args = [str(x) for x in args] @@ -171,7 +171,7 @@ class ProcessPool(object): self.size = size and size or FLAGS.process_pool_size self._pool = defer.DeferredSemaphore(self.size) - def simpleExecute(self, cmd, **kw): + def simple_execute(self, cmd, **kw): """ Weak emulation of the old utils.execute() function. This only exists as a way to quickly move old execute methods to @@ -205,34 +205,10 @@ class ProcessPool(object): self._pool.release() return rv - -class Pool(object): - """ A simple process pool implementation around mutliprocessing. - - Allows up to `size` processes at a time and queues the rest. - - Using workarounds for multiprocessing behavior described in: - http://pypi.python.org/pypi/twisted.internet.processes/1.0b1 - """ - - def __init__(self, size=None): - self._size = size - self._pool = multiprocessing.Pool(size) - self._registerShutdown() - - def _registerShutdown(self): - reactor.addSystemEventTrigger( - 'during', 'shutdown', self.shutdown, reactor) - - def shutdown(self, reactor=None): - if not self._pool: - return - self._pool.close() - # wait for workers to finish - self._pool.terminate() - self._pool = None - - def apply(self, f, *args, **kw): - """ Add a task to the pool and return a deferred. """ - result = self._pool.apply_async(f, args, kw) - return threads.deferToThread(result.get) +class SharedPool(ProcessPool): + _instance = None + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(SharedPool, cls).__new__( + cls, *args, **kwargs) + return cls._instance diff --git a/nova/tests/process_unittest.py b/nova/tests/process_unittest.py index 01648961f..1c15b69a0 100644 --- a/nova/tests/process_unittest.py +++ b/nova/tests/process_unittest.py @@ -37,7 +37,7 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stdout(self): pool = process.ProcessPool(2) - d = pool.simpleExecute('echo test') + d = pool.simple_execute('echo test') def _check(rv): self.assertEqual(rv[0], 'test\n') self.assertEqual(rv[1], '') @@ -48,38 +48,38 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stderr(self): pool = process.ProcessPool(2) - d = pool.simpleExecute('cat BAD_FILE', error_ok=1) + d = pool.simple_execute('cat BAD_FILE', error_ok=1) def _check(rv): self.assertEqual(rv[0], '') self.assert_('No such file' in rv[1]) - + d.addCallback(_check) d.addErrback(self.fail) return d def test_execute_unexpected_stderr(self): pool = process.ProcessPool(2) - d = pool.simpleExecute('cat BAD_FILE') + d = pool.simple_execute('cat BAD_FILE') d.addCallback(lambda x: self.fail('should have raised an error')) d.addErrback(lambda failure: failure.trap(IOError)) return d - + def test_max_processes(self): pool = process.ProcessPool(2) - d1 = pool.simpleExecute('sleep 0.01') - d2 = pool.simpleExecute('sleep 0.01') - d3 = pool.simpleExecute('sleep 0.005') - d4 = pool.simpleExecute('sleep 0.005') + d1 = pool.simple_execute('sleep 0.01') + d2 = pool.simple_execute('sleep 0.01') + d3 = pool.simple_execute('sleep 0.005') + d4 = pool.simple_execute('sleep 0.005') called = [] def _called(rv, name): called.append(name) - + d1.addCallback(_called, 'd1') d2.addCallback(_called, 'd2') d3.addCallback(_called, 'd3') d4.addCallback(_called, 'd4') - + # Make sure that d3 and d4 had to wait on the other two and were called # in order # NOTE(termie): there may be a race condition in this test if for some @@ -92,25 +92,31 @@ class ProcessTestCase(test.TrialTestCase): def test_kill_long_process(self): pool = process.ProcessPool(2) - - d1 = pool.simpleExecute('sleep 1') - d2 = pool.simpleExecute('sleep 0.005') + + d1 = pool.simple_execute('sleep 1') + d2 = pool.simple_execute('sleep 0.005') timeout = reactor.callLater(0.1, self.fail, 'should have been killed') - + # kill d1 and wait on it to end then cancel the timeout d2.addCallback(lambda _: d1.process.signalProcess('KILL')) d2.addCallback(lambda _: d1) d2.addBoth(lambda _: timeout.active() and timeout.cancel()) d2.addErrback(self.fail) return d2 - + def test_process_exit_is_contained(self): pool = process.ProcessPool(2) - - d1 = pool.simpleExecute('sleep 1') + + d1 = pool.simple_execute('sleep 1') d1.addCallback(lambda x: self.fail('should have errbacked')) d1.addErrback(lambda fail: fail.trap(IOError)) reactor.callLater(0.05, d1.process.signalProcess, 'KILL') - + return d1 + + def test_shared_pool_is_singleton(self): + pool1 = process.SharedPool() + pool2 = process.SharedPool() + self.assert_(id(pool1) == id(pool2)) + -- cgit From 52f0ec017c04ad9356eb45d96bcab90a0807d914 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 16 Jul 2010 20:32:51 +0000 Subject: make simple method wrapper for process pool simple_execute --- nova/compute/node.py | 12 ++++++------ nova/process.py | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index 3abd20120..4683f1c8d 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -446,13 +446,13 @@ class Instance(object): def _fetch_s3_image(self, image, path): url = _image_url('%s/image' % image) - d = process.SharedPool().simple_execute( + d = process.simple_execute( 'curl --silent %s -o %s' % (url, path)) return d def _fetch_local_image(self, image, path): source = _image_path('%s/image' % image) - d = process.SharedPool().simple_execute('cp %s %s' % (source, path)) + d = process.simple_execute('cp %s %s' % (source, path)) return d @defer.inlineCallbacks @@ -462,9 +462,9 @@ class Instance(object): basepath = self.basepath # ensure directories exist and are writable - yield process.SharedPool().simple_execute( + yield process.simple_execute( 'mkdir -p %s' % basepath()) - yield process.SharedPool().simple_execute( + yield process.simple_execute( 'chmod 0777 %s' % basepath()) @@ -492,7 +492,7 @@ class Instance(object): yield _fetch_file(data['ramdisk_id'], basepath('ramdisk')) execute = lambda cmd, input=None: \ - process.SharedPool().simple_execute(cmd=cmd, + process.simple_execute(cmd=cmd, input=input, error_ok=1) @@ -511,7 +511,7 @@ class Instance(object): yield disk.inject_data(basepath('disk-raw'), key, net, execute=execute) if os.path.exists(basepath('disk')): - yield process.SharedPool().simple_execute( + yield process.simple_execute( 'rm -f %s' % basepath('disk')) bytes = (INSTANCE_TYPES[data['instance_type']]['local_gb'] diff --git a/nova/process.py b/nova/process.py index ebfb2f4ba..d3558ed2e 100644 --- a/nova/process.py +++ b/nova/process.py @@ -212,3 +212,6 @@ class SharedPool(ProcessPool): cls._instance = super(SharedPool, cls).__new__( cls, *args, **kwargs) return cls._instance + +def simple_execute(cmd, **kwargs): + return SharedPool().simple_execute(cmd, **kwargs) -- cgit From 6892ff871b1a154bbe669bf5cb10eab638fb181b Mon Sep 17 00:00:00 2001 From: Joshua McKenty Date: Fri, 16 Jul 2010 14:02:37 -0700 Subject: Ack messages during call so rabbit leaks less. --- nova/rpc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index 72a84b7f7..99e820ff3 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -197,7 +197,10 @@ def call(topic, msg): conn = Connection.instance() d = defer.Deferred() consumer = DirectConsumer(connection=conn, msg_id=msg_id) - consumer.register_callback(lambda data, message: d.callback(data)) + def deferred_receive(data, message): + message.ack() + d.callback(data) + consumer.register_callback(deferred_receive) injected = consumer.attach_to_tornado() # clean up after the injected listened and return x -- cgit From f6aeb0a121e76aefa8b6af6ae602df76c2419b2e Mon Sep 17 00:00:00 2001 From: Joshua McKenty Date: Fri, 16 Jul 2010 14:07:57 -0700 Subject: Makin the queues non-durable by default --- nova/rpc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index 99e820ff3..c6ebb1160 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -112,6 +112,7 @@ class TopicConsumer(Consumer): self.queue = topic self.routing_key = topic self.exchange = FLAGS.control_exchange + self.durable = False super(TopicConsumer, self).__init__(connection=connection) @@ -238,7 +239,8 @@ def send_message(topic, message, wait=True): exchange=msg_id, auto_delete=True, exchange_type="direct", - routing_key=msg_id) + routing_key=msg_id, + durable=False) consumer.register_callback(generic_response) publisher = messaging.Publisher(connection=Connection.instance(), -- cgit From 9e023095e303f096ac0d2914ed427d2a37d1444d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 17 Jul 2010 23:00:53 -0700 Subject: Replace nova-objectstore with a twistd style wrapper. Add a get_application method to objectstore handler. --- bin/nova-objectstore | 25 +++++++++++-------------- nova/objectstore/handler.py | 29 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 7876864c0..5ac911f09 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -18,35 +18,32 @@ # under the License. """ - Tornado daemon for nova objectstore. Supports S3 API. + Twisted daemon for nova objectstore. Supports S3 API. """ import logging -from nova import vendor -from tornado import httpserver -from tornado import ioloop - from nova import flags -from nova import server from nova import utils -from nova.auth import users +from nova import twistd from nova.objectstore import handler FLAGS = flags.FLAGS -def main(argv): +def main(): # FIXME: if this log statement isn't here, no logging # appears from other files and app won't start daemonized logging.debug('Started HTTP server on %s' % (FLAGS.s3_internal_port)) - app = handler.Application(users.UserManager()) - server = httpserver.HTTPServer(app) - server.listen(FLAGS.s3_internal_port) - ioloop.IOLoop.instance().start() - + app = handler.get_application() + print app + return app +# NOTE(soren): Stolen from nova-compute if __name__ == '__main__': + twistd.serve(__file__) + +if __name__ == '__builtin__': utils.default_flagfile() - server.serve('nova-objectstore', main) + application = main() diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 50c56c83c..63ed34f2d 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -42,13 +42,10 @@ import logging import json import multiprocessing import os -import re import time import urllib - -from nova import vendor - +from twisted.application import internet, service from twisted.web.resource import Resource from twisted.web import server, static from twisted.internet import reactor @@ -64,7 +61,6 @@ from nova.objectstore import image FLAGS = flags.FLAGS -FLAGS.fake_users = True def render_xml(request, value): assert isinstance(value, dict) and len(value) == 1 @@ -264,15 +260,15 @@ class ImageResource(Resource): bucket_object = bucket.Bucket(image_location.split("/")[0]) manifest = image_location[len(image_location.split('/')[0])+1:] - if not bucket_object.is_authorized(self.context): + if not bucket_object.is_authorized(request.context): raise exception.NotAuthorized p = multiprocessing.Process(target=image.Image.register_aws_image, - args=(image_id, image_location, self.context)) + args=(image_id, image_location, request.context)) p.start() return '' - def render_POST(self): + def render_POST(self, request): """ update image attributes: public/private """ image_id = self.get_argument('image_id', u'') @@ -280,19 +276,19 @@ class ImageResource(Resource): image_object = image.Image(image_id) - if not image.is_authorized(self.context): + if not image.is_authorized(request.context): raise exception.NotAuthorized image_object.set_public(operation=='add') return '' - def render_DELETE(self): + def render_DELETE(self, request): """ delete a registered image """ image_id = self.get_argument("image_id", u"") image_object = image.Image(image_id) - if not image.is_authorized(self.context): + if not image.is_authorized(request.context): raise exception.NotAuthorized image_object.delete() @@ -300,7 +296,10 @@ class ImageResource(Resource): request.setResponseCode(204) return '' -root = S3() -factory = server.Site(root) -reactor.listenTCP(3333, factory) -reactor.run() +def get_application(): + root = S3() + factory = server.Site(root) + application = service.Application("objectstore") + objectStoreService = internet.TCPServer(FLAGS.s3_port, factory) + objectStoreService.setServiceParent(application) + return application -- cgit From 2d5124c3f2c6e4e78dc09eb8f38cb125641b9b1c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 17 Jul 2010 23:04:46 -0700 Subject: Remove s3_internal_port setting. Objectstore should be able to handle the beatings now. As such, nginx is no longer needed, so it's removed from the dependencies and the configuration files are removed. --- bin/nova-objectstore | 2 +- debian/control | 2 +- debian/nova-objectstore.install | 1 - debian/nova-objectstore.links | 1 - debian/nova-objectstore.nginx.conf | 17 ----------------- nova/flags.py | 1 - 6 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 debian/nova-objectstore.links delete mode 100644 debian/nova-objectstore.nginx.conf diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 5ac911f09..9385fd299 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -35,7 +35,7 @@ FLAGS = flags.FLAGS def main(): # FIXME: if this log statement isn't here, no logging # appears from other files and app won't start daemonized - logging.debug('Started HTTP server on %s' % (FLAGS.s3_internal_port)) + logging.debug('Started HTTP server on %s' % (FLAGS.s3_port)) app = handler.get_application() print app return app diff --git a/debian/control b/debian/control index 17414bb7a..a6d12f36e 100644 --- a/debian/control +++ b/debian/control @@ -91,7 +91,7 @@ Description: Nova Cloud Computing - API frontend Package: nova-objectstore Architecture: all -Depends: nova-common (= ${binary:Version}), nginx, ${python:Depends}, ${misc:Depends} +Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends} Description: Nova Cloud Computing - object store Nova is a cloud computing fabric controller (the main part of an IaaS system) built to match the popular AWS EC2 and S3 APIs. It is written in diff --git a/debian/nova-objectstore.install b/debian/nova-objectstore.install index 3ed93ff37..c5b3d997a 100644 --- a/debian/nova-objectstore.install +++ b/debian/nova-objectstore.install @@ -1,3 +1,2 @@ bin/nova-objectstore usr/bin debian/nova-objectstore.conf etc/nova -debian/nova-objectstore.nginx.conf etc/nginx/sites-available diff --git a/debian/nova-objectstore.links b/debian/nova-objectstore.links deleted file mode 100644 index 38e33948e..000000000 --- a/debian/nova-objectstore.links +++ /dev/null @@ -1 +0,0 @@ -/etc/nginx/sites-available/nova-objectstore.nginx.conf /etc/nginx/sites-enabled/nova-objectstore.nginx.conf diff --git a/debian/nova-objectstore.nginx.conf b/debian/nova-objectstore.nginx.conf deleted file mode 100644 index b63424150..000000000 --- a/debian/nova-objectstore.nginx.conf +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 3333 default; - server_name localhost; - client_max_body_size 10m; - - access_log /var/log/nginx/localhost.access.log; - - location ~ /_images/.+ { - root /var/lib/nova/images; - rewrite ^/_images/(.*)$ /$1 break; - } - - location / { - proxy_pass http://localhost:3334/; - } -} - diff --git a/nova/flags.py b/nova/flags.py index 22e00a44a..ae8bf98f7 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -37,7 +37,6 @@ DEFINE_bool = DEFINE_bool # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 DEFINE_integer('s3_port', 3333, 's3 port') -DEFINE_integer('s3_internal_port', 3334, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') #DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') -- cgit From c39b3add4b3db1c027c32ea0f145c3d3ab3f4d1c Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Mon, 19 Jul 2010 09:50:22 -0500 Subject: Hmm, serves me right for not understanding the request, eh? :) Now too_many_addresses test case is idempotent in regards to running in isolation and uses self.flags.network_size instead of the magic number 32. --- nova/tests/network_unittest.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 3cfba2cf9..4a9f9b8ab 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -155,16 +155,16 @@ class NetworkTestCase(test.TrialTestCase): However, the number of available IP addresses depends on the test environment's setup. - Network size is 32. + Network size is set in test fixture's setUp method. - There are FLAGS.cnt_vpn_clients addresses reserved for VPN (NUM_RESERVED_VPN_IPS) + There are FLAGS.cnt_vpn_clients addresses reserved for VPN (NUM_RESERVED_VPN_IPS) - And there are NUM_STATIC_IPS that are always reserved by Nova for the necessary - services (gateway, CloudPipe, etc) + And there are NUM_STATIC_IPS that are always reserved by Nova for the necessary + services (gateway, CloudPipe, etc) - So we should get 32 - (NUM_STATIC_IPS + - NUM_PREALLOCATED_IPS + - NUM_RESERVED_VPN_IPS) + So we should get flags.network_size - (NUM_STATIC_IPS + + NUM_PREALLOCATED_IPS + + NUM_RESERVED_VPN_IPS) usable addresses """ net = network.get_project_network("project0", "default") @@ -173,7 +173,7 @@ class NetworkTestCase(test.TrialTestCase): num_static_ips = net.num_static_ips num_preallocated_ips = len(net.hosts.keys()) num_reserved_vpn_ips = flags.FLAGS.cnt_vpn_clients - num_available_ips = 32 - (num_static_ips + num_preallocated_ips + num_reserved_vpn_ips) + num_available_ips = flags.FLAGS.network_size - (num_static_ips + num_preallocated_ips + num_reserved_vpn_ips) hostname = "toomany-hosts" macs = {} -- cgit From 248f297da74b6353b9589b2887c9ab5edad8dc22 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 19 Jul 2010 17:07:52 -0700 Subject: Raise 401, not exception if Authorization header not passed. Also minor fixes & Python exception-handling style tweak --- nova/objectstore/handler.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index b2ed3d482..e5c97c3bc 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -103,13 +103,16 @@ def get_argument(request, key, default_value): def get_context(request): try: # Authorization Header format: 'AWS :' - access, sep, secret = request.getHeader('Authorization').split(' ')[1].rpartition(':') + authorization_header = request.getHeader('Authorization') + if not authorization_header: + raise exception.NotAuthorized + access, sep, secret = authorization_header.split(' ')[1].rpartition(':') um = users.UserManager.instance() print 'um %s' % um (user, project) = um.authenticate(access, secret, {}, request.method, request.host, request.uri, False) # FIXME: check signature here! return api.APIRequestContext(None, user, project) - except exception.Error, ex: + except exception.Error as ex: logging.debug("Authentication Failure: %s" % ex) raise exception.NotAuthorized @@ -131,6 +134,7 @@ class S3(Resource): render_xml(request, {"ListAllMyBucketsResult": { "Buckets": {"Bucket": [b.metadata for b in buckets]}, }}) + request.finish() return server.NOT_DONE_YET class BucketResource(Resource): @@ -165,7 +169,7 @@ class BucketResource(Resource): logging.debug("Creating bucket %s" % (self.name)) try: print 'user is %s' % request.context - except Exception, e: + except Exception as e: logging.exception(e) logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context)) bucket.Bucket.create(self.name, request.context) @@ -239,7 +243,7 @@ class ImageResource(Resource): """ returns a json listing of all images that a user has permissions to see """ - images = [i for i in image.Image.all() if i.is_authorized(self.context)] + images = [i for i in image.Image.all() if i.is_authorized(request.context)] request.write(json.dumps([i.metadata for i in images])) return server.NOT_DONE_YET -- cgit From 8625275e14d40dd82d19d2273e14f82334c6b5ac Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 19 Jul 2010 18:26:19 -0700 Subject: I put the call to request.finish() in the wrong place. :-( --- nova/objectstore/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index e5c97c3bc..c670ee02f 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -134,7 +134,6 @@ class S3(Resource): render_xml(request, {"ListAllMyBucketsResult": { "Buckets": {"Bucket": [b.metadata for b in buckets]}, }}) - request.finish() return server.NOT_DONE_YET class BucketResource(Resource): @@ -246,6 +245,7 @@ class ImageResource(Resource): images = [i for i in image.Image.all() if i.is_authorized(request.context)] request.write(json.dumps([i.metadata for i in images])) + request.finish() return server.NOT_DONE_YET def render_PUT(self, request): -- cgit From fd25c2699867e16908aaadc3380236f84cc3cc5a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 17:05:02 -0500 Subject: remove spaces from export statements in scripts relating to certs --- CA/geninter.sh | 2 +- nova/cloudpipe/bootscript.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CA/geninter.sh b/CA/geninter.sh index 6c0528d1b..46f8f79ad 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -17,7 +17,7 @@ # under the License. # ARG is the id of the user -export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-intCA-$3 +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$3" mkdir INTER/$1 cd INTER/$1 cp ../../openssl.cnf.tmpl openssl.cnf diff --git a/nova/cloudpipe/bootscript.sh b/nova/cloudpipe/bootscript.sh index 43fc2ecab..82ec2012a 100755 --- a/nova/cloudpipe/bootscript.sh +++ b/nova/cloudpipe/bootscript.sh @@ -24,7 +24,7 @@ export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $1}'` export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $1}'` export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` -export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-vpn-$VPN_IP +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-vpn-$VPN_IP" DHCP_LOWER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 10 }'` DHCP_UPPER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 1 }'` -- cgit From 0506cce7d934ad093f5808606627aa19f43428ef Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 17:55:38 -0500 Subject: Fixed the broken reference to --- CA/geninter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CA/geninter.sh b/CA/geninter.sh index 46f8f79ad..7d6c280d5 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -17,7 +17,7 @@ # under the License. # ARG is the id of the user -export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$3" +export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$1" mkdir INTER/$1 cd INTER/$1 cp ../../openssl.cnf.tmpl openssl.cnf -- cgit From 8f9d5a66fdebe2bdd41bfbb7d20fbab531cd3b3f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 20:28:34 -0500 Subject: use a locally administered mac address so it isn't saved by udev --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index c5b935673..9ecceafe0 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -94,7 +94,7 @@ def generate_uid(topic, size=8): def generate_mac(): - mac = [0x00, 0x16, 0x3e, random.randint(0x00, 0x7f), + mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f), random.randint(0x00, 0xff), random.randint(0x00, 0xff) ] return ':'.join(map(lambda x: "%02x" % x, mac)) -- cgit From 1b6efa80e19a60d71a762683fa1edee02645355c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Jul 2010 22:28:23 -0500 Subject: fix for describe addresses showing everyone's public ips --- nova/endpoint/cloud.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 3b7b4804b..4fa9b5afd 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -453,21 +453,21 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - # TODO(vish): move authorization checking into network.py for address in self.network.host_objs: - #logging.debug(address_record) - address_rv = { - 'public_ip': address['address'], - 'instance_id' : address.get('instance_id', 'free') - } - if context.user.is_admin(): - address_rv['instance_id'] = "%s (%s, %s)" % ( - address['instance_id'], - address['user_id'], - address['project_id'], - ) + # TODO(vish): implement a by_project iterator for addresses + if (context.user.is_admin() or + address['project_id'] == self.project.id): + address_rv = { + 'public_ip': address['address'], + 'instance_id' : address.get('instance_id', 'free') + } + if context.user.is_admin(): + address_rv['instance_id'] = "%s (%s, %s)" % ( + address['instance_id'], + address['user_id'], + address['project_id'], + ) addresses.append(address_rv) - # logging.debug(addresses) return {'addressesSet': addresses} @rbac.allow('netadmin') -- cgit