From 0a5c5f8130fce42b1edcfb67c702e25f51aefa13 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sat, 12 Jun 2010 18:24:27 -0700 Subject: implement image serving in objectstore so nginx isn't required in development reviewed by yosh --- nova/objectstore/handler.py | 26 ++++++++++++++++++++++++++ nova/objectstore/image.py | 4 ++++ 2 files changed, 30 insertions(+) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index a7fff12fc..3f00bb0c4 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -71,6 +71,7 @@ class Application(web.Application): def __init__(self, user_manager): web.Application.__init__(self, [ (r"/", RootHandler), + (r"/_images/(.+)", ImageDownloadHandler), (r"/_images/", ImageHandler), (r"/([^/]+)/(.+)", ObjectHandler), (r"/([^/]+)/", BucketHandler), @@ -224,6 +225,31 @@ class ObjectHandler(BaseRequestHandler): self.finish() +class ImageDownloadHandler(BaseRequestHandler): + SUPPORTED_METHODS = ("GET", ) + + @catch_nova_exceptions + def get(self, image_id): + """ send the decrypted image file + + streaming content through python is slow and should only be used + in development mode. You should serve files via a web server + in production. + """ + + self.set_header("Content-Type", "application/octet-stream") + + READ_SIZE = 64*1024 + + img = image.Image(image_id) + with open(img.image_path, 'rb') as fp: + s = fp.read(READ_SIZE) + while s: + self.write(s) + s = fp.read(READ_SIZE) + + self.finish() + class ImageHandler(BaseRequestHandler): SUPPORTED_METHODS = ("POST", "PUT", "GET", "DELETE") diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 892ada00c..b8dae4077 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -47,6 +47,10 @@ class Image(object): not os.path.isdir(self.path): raise exception.NotFound + @property + def image_path(self): + return os.path.join(self.path, 'image') + def delete(self): for fn in ['info.json', 'image']: try: -- cgit From aabc316aa734107e82a6dd0317028f9a254f24bc Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 15 Jun 2010 10:42:34 -0700 Subject: first go at moving from tornado to twisted --- nova/objectstore/handler.py | 169 ++++++++++++++++++++++++++++++-------------- 1 file changed, 114 insertions(+), 55 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 3f00bb0c4..d9369afbf 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -33,18 +33,26 @@ S3 client with this module:: """ import datetime -import os -import urllib -import json import logging +import json import multiprocessing +import os +import re +import time +import urllib from nova import vendor -from tornado import escape, web + +from twisted.web import resource +from twisted.web import server +from twisted.internet import reactor + +from tornado import escape # FIXME(ja): move to non-tornado escape from nova import exception from nova import flags +from nova.auth import users from nova.endpoint import api from nova.objectstore import bucket from nova.objectstore import image @@ -53,54 +61,102 @@ from nova.objectstore import image FLAGS = flags.FLAGS -def catch_nova_exceptions(target): - # FIXME: find a way to wrap all handlers in the web.Application.__init__ ? - def wrapper(*args, **kwargs): - try: - return target(*args, **kwargs) - except exception.NotFound: - raise web.HTTPError(404) - except exception.NotAuthorized: - raise web.HTTPError(403) - - return wrapper +class Application(resource.Resource): + """Implementation of an S3-like storage server based on local files.""" + isLeaf = True -class Application(web.Application): - """Implementation of an S3-like storage server based on local files.""" - def __init__(self, user_manager): - web.Application.__init__(self, [ - (r"/", RootHandler), + def __init__(self): + # fixme(ja): optomize by compiling regexps? + self.handlers = [ (r"/_images/(.+)", ImageDownloadHandler), (r"/_images/", ImageHandler), (r"/([^/]+)/(.+)", ObjectHandler), (r"/([^/]+)/", BucketHandler), - ]) + (r"/", RootHandler), + ] self.buckets_path = os.path.abspath(FLAGS.buckets_path) self.images_path = os.path.abspath(FLAGS.images_path) if not os.path.exists(self.buckets_path): - raise Exception("buckets_path does not exist") + raise Exception("buckets_path %s does not exist" % self.buckets_path) if not os.path.exists(self.images_path): - raise Exception("images_path does not exist") - self.user_manager = user_manager + raise Exception("images_path %s does not exist" % self.images_path) + + def render_GET(self, request): + return self.route(request) + + def render_PUT(self, request): + return self.route(request) + + def render_POST(self, request): + return self.route(request) + + def render_DELETE(self, request): + return self.route(request) + + def route(self, request): + start_time = time.time() + + for regexp, handler in self.handlers: + match = re.search(regexp, request.path) + if match: + try: + print 'match: %s' % request.path + func = getattr(handler(request), request.method.lower()) + #print 'func: %s' % func + params = match.groups() + #print 'args: %s' % str(params) + response = func(*params) + #print 'resp: %s' % response + except exception.NotFound: + request.setResponseCode(404) + response = 'Not Found' + except exception.NotAuthorized: + request.setResponseCode(403) + response = 'Not Authorized' + except Exception, e: + request.setResponseCode(500) + response = 'Internal Error: %s' % e + break + + duration = (time.time() - start_time) * 1000 + logging.info("%d %s %s %0.1fms %s" % (request.code, request.method, request.uri, + duration, str(handler).split('.')[-1].split("'")[0])) + return response + + +class BaseRequestHandler(object): + SUPPORTED_METHODS = ("PUT", "GET", "DELETE", "HEAD") + def __init__(self, request): + self.request = request -class BaseRequestHandler(web.RequestHandler): - SUPPORTED_METHODS = ("PUT", "GET", "DELETE", "HEAD") + def set_header(self, name, value): + self.request.setHeader(name, value) + + def write(self, content): + self.request.write(content) + + def finish(self, content=None): + if content: + self.request.write(content) + self.request.finish() @property def context(self): if not hasattr(self, '_context'): try: # Authorization Header format: 'AWS :' - access, sep, secret = self.request.headers['Authorization'].split(' ')[1].rpartition(':') - (user, project) = self.application.user_manager.authenticate(access, secret, {}, self.request.method, self.request.host, self.request.path, False) + access, sep, secret = self.request.getHeader('Authorization').split(' ')[1].rpartition(':') + um = users.UserManager.instance() + print 'um %s' % um + (user, project) = um.authenticate(access, secret, {}, self.request.method, self.request.host, self.request.uri, False) # FIXME: check signature here! self._context = api.APIRequestContext(self, user, project) except exception.Error, ex: logging.debug("Authentication Failure: %s" % ex) - raise web.HTTPError(403) + raise exception.NotAuthorized return self._context def render_xml(self, value): @@ -138,6 +194,7 @@ class BaseRequestHandler(web.RequestHandler): class RootHandler(BaseRequestHandler): + def get(self): buckets = [b for b in bucket.Bucket.all() if b.is_authorized(self.context)] @@ -147,14 +204,13 @@ class RootHandler(BaseRequestHandler): class BucketHandler(BaseRequestHandler): - @catch_nova_exceptions def get(self, bucket_name): logging.debug("List keys for bucket %s" % (bucket_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized prefix = self.get_argument("prefix", u"") marker = self.get_argument("marker", u"") @@ -164,19 +220,21 @@ class BucketHandler(BaseRequestHandler): results = bucket_object.list_keys(prefix=prefix, marker=marker, max_keys=max_keys, terse=terse) self.render_xml({"ListBucketResult": results}) - @catch_nova_exceptions def put(self, bucket_name): logging.debug("Creating bucket %s" % (bucket_name)) + try: + print 'user is %s' % self.context + except Exception, e: + logging.exception(e) bucket.Bucket.create(bucket_name, self.context) self.finish() - @catch_nova_exceptions def delete(self, bucket_name): logging.debug("Deleting bucket %s" % (bucket_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized bucket_object.delete() self.set_status(204) @@ -184,14 +242,13 @@ class BucketHandler(BaseRequestHandler): class ObjectHandler(BaseRequestHandler): - @catch_nova_exceptions def get(self, bucket_name, object_name): logging.debug("Getting object: %s / %s" % (bucket_name, object_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized obj = bucket_object[urllib.unquote(object_name)] self.set_header("Content-Type", "application/unknown") @@ -199,26 +256,28 @@ class ObjectHandler(BaseRequestHandler): self.set_header("Etag", '"' + obj.md5 + '"') self.finish(obj.read()) - @catch_nova_exceptions def put(self, bucket_name, object_name): logging.debug("Putting object: %s / %s" % (bucket_name, object_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized key = urllib.unquote(object_name) - bucket_object[key] = self.request.body + print 'seeking' + self.request.content.seek(0, 0) + print 'writing' + bucket_object[key] = self.request.content.read() + print 'etag %s' % bucket_object[key].md5 self.set_header("Etag", '"' + bucket_object[key].md5 + '"') self.finish() - @catch_nova_exceptions def delete(self, bucket_name, object_name): logging.debug("Deleting object: %s / %s" % (bucket_name, object_name)) bucket_object = bucket.Bucket(bucket_name) if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized del bucket_object[urllib.unquote(object_name)] self.set_status(204) @@ -228,7 +287,6 @@ class ObjectHandler(BaseRequestHandler): class ImageDownloadHandler(BaseRequestHandler): SUPPORTED_METHODS = ("GET", ) - @catch_nova_exceptions def get(self, image_id): """ send the decrypted image file @@ -239,21 +297,21 @@ class ImageDownloadHandler(BaseRequestHandler): self.set_header("Content-Type", "application/octet-stream") - READ_SIZE = 64*1024 + READ_SIZE = 1024*1024 img = image.Image(image_id) with open(img.image_path, 'rb') as fp: - s = fp.read(READ_SIZE) - while s: - self.write(s) - s = fp.read(READ_SIZE) + chunk = fp.read(READ_SIZE) + while chunk: + self.write(chunk) + self.flush() + chunk = fp.read(READ_SIZE) self.finish() class ImageHandler(BaseRequestHandler): SUPPORTED_METHODS = ("POST", "PUT", "GET", "DELETE") - @catch_nova_exceptions def get(self): """ returns a json listing of all images that a user has permissions to see """ @@ -262,7 +320,6 @@ class ImageHandler(BaseRequestHandler): self.finish(json.dumps([i.metadata for i in images])) - @catch_nova_exceptions def put(self): """ create a new registered image """ @@ -272,20 +329,19 @@ class ImageHandler(BaseRequestHandler): image_path = os.path.join(FLAGS.images_path, image_id) if not image_path.startswith(FLAGS.images_path) or \ os.path.exists(image_path): - raise web.HTTPError(403) + raise exception.NotAuthorized bucket_object = bucket.Bucket(image_location.split("/")[0]) manifest = image_location[len(image_location.split('/')[0])+1:] if not bucket_object.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized p = multiprocessing.Process(target=image.Image.create,args= (image_id, image_location, self.context)) p.start() self.finish() - @catch_nova_exceptions def post(self): """ update image attributes: public/private """ @@ -295,21 +351,24 @@ class ImageHandler(BaseRequestHandler): image_object = image.Image(image_id) if not image.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized image_object.set_public(operation=='add') self.finish() - @catch_nova_exceptions def delete(self): """ delete a registered image """ image_id = self.get_argument("image_id", u"") image_object = image.Image(image_id) if not image.is_authorized(self.context): - raise web.HTTPError(403) + raise exception.NotAuthorized image_object.delete() self.set_status(204) + +factory = server.Site(Application()) +reactor.listenTCP(3333, factory) +reactor.run() -- cgit From b81b0f2ecf3ef9bcba71a581ccd0ed3729398fba Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sun, 20 Jun 2010 15:08:25 -0700 Subject: update spacing --- docs/getting.started.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/getting.started.rst b/docs/getting.started.rst index 9d7808a27..63e435906 100644 --- a/docs/getting.started.rst +++ b/docs/getting.started.rst @@ -1,12 +1,12 @@ .. Copyright [2010] [Anso Labs, LLC] - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,19 +67,19 @@ Installation # ON THE CLOUD CONTROLLER apt-get install -y rabbitmq-server dnsmasq nginx # build redis from 2.0.0-rc1 source - # setup ldap (slap.sh as root will remove ldap and reinstall it) - NOVA_PATH/nova/auth/slap.sh + # setup ldap (slap.sh as root will remove ldap and reinstall it) + NOVA_PATH/nova/auth/slap.sh /etc/init.d/rabbitmq-server start # ON VOLUME NODE: - apt-get install -y vblade-persist + apt-get install -y vblade-persist # ON THE COMPUTE NODE: apt-get install -y kpartx kvm # optional packages - apt-get install -y euca2ools - + apt-get install -y euca2ools + Configuration --------------- @@ -90,10 +90,10 @@ ON CLOUD CONTROLLER :: iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 - iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE + iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE -* Configure NginX proxy (/etc/nginx/sites-enabled/default) +* Configure NginX proxy (/etc/nginx/sites-enabled/default) :: @@ -137,8 +137,8 @@ Launch servers Launch nova components -* api_worker -* s3_worker -* node_worker -* storage_worker +* nova-api +* nova-compute +* nova-objectstore +* nova-volume -- cgit