From 4044051266d97ffe05fbe75b642759d2e604da4d Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 26 Jul 2010 23:19:51 -0400 Subject: Share my updates to the Rackspace API. --- bin/nova-rsapi | 13 +--- exercise_rsapi.py | 51 ---------------- nova/endpoint/rackspace.py | 149 +++++++++++++++++++++++++++++++++------------ 3 files changed, 110 insertions(+), 103 deletions(-) delete mode 100644 exercise_rsapi.py diff --git a/bin/nova-rsapi b/bin/nova-rsapi index c2f2c9d70..a529fc669 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -37,23 +37,12 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') def main(_argv): - user_manager = users.UserManager() - api_instance = rackspace.Api(user_manager) - conn = rpc.Connection.instance() - rpc_consumer = rpc.AdapterConsumer(connection=conn, - topic=FLAGS.cloud_topic, - proxy=api_instance) - -# TODO: fire rpc response listener (without attach to tornado) -# io_inst = ioloop.IOLoop.instance() -# _injected = consumer.attach_to_tornado(io_inst) - + api_instance = rackspace.Api() http_server = simple_server.WSGIServer(('0.0.0.0', FLAGS.cc_port), simple_server.WSGIRequestHandler) http_server.set_app(api_instance.handler) logging.debug('Started HTTP server on port %i' % FLAGS.cc_port) while True: http_server.handle_request() -# io_inst.start() if __name__ == '__main__': utils.default_flagfile() diff --git a/exercise_rsapi.py b/exercise_rsapi.py deleted file mode 100644 index 20589b9cb..000000000 --- a/exercise_rsapi.py +++ /dev/null @@ -1,51 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import cloudservers - -class IdFake: - def __init__(self, id): - self.id = id - -# to get your access key: -# from nova.auth import users -# users.UserManger.instance().get_users()[0].access -rscloud = cloudservers.CloudServers( - 'admin', - '6cca875e-5ab3-4c60-9852-abf5c5c60cc6' - ) -rscloud.client.AUTH_URL = 'http://localhost:8773/v1.0' - - -rv = rscloud.servers.list() -print "SERVERS: %s" % rv - -if len(rv) == 0: - server = rscloud.servers.create( - "test-server", - IdFake("ami-tiny"), - IdFake("m1.tiny") - ) - print "LAUNCH: %s" % server -else: - server = rv[0] - print "Server to kill: %s" % server - -raw_input("press enter key to kill the server") - -server.delete() diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 29a077b24..b561212f5 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -48,82 +48,128 @@ FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') -# TODO(todd): subclass Exception so we can bubble meaningful errors +class Unauthorized(Exception): + pass + +class NotFound(Exception): + pass class Api(object): - def __init__(self, rpc_mechanism): + def __init__(self): + """build endpoints here""" self.controllers = { "v1.0": RackspaceAuthenticationApi(), "servers": RackspaceCloudServerApi() } - self.rpc_mechanism = rpc_mechanism def handler(self, environ, responder): + """ + This is the entrypoint from wsgi. Read PEP 333 and wsgi.org for + more intormation. The key points are responder is a callback that + needs to run before you return, and takes two arguments, response + code string ("200 OK") and headers (["X-How-Cool-Am-I: Ultra-Suede"]) + and the return value is the body of the response. + """ environ['nova.context'] = self.build_context(environ) controller, path = wsgi.Util.route( environ['PATH_INFO'], self.controllers ) + logging.debug("Route %s to %s", str(path), str(controller)) if not controller: - # TODO(todd): Exception (404) - raise Exception("Missing Controller") - rv = controller.process(path, environ) - if type(rv) is tuple: - responder(rv[0], rv[1]) - rv = rv[2] - else: - responder("200 OK", []) - return rv + responder("404 Not Found", []) + return "" + try: + rv = controller.process(path, environ) + if type(rv) is tuple: + responder(rv[0], rv[1]) + rv = rv[2] + else: + responder("200 OK", []) + return rv + except Unauthorized: + responder("401 Unauthorized", []) + return "" + except NotFound: + responder("404 Not Found", []) + return "" + def build_context(self, env): rv = {} if env.has_key("HTTP_X_AUTH_TOKEN"): + # TODO(todd): once we make an actual unique token, this will change rv['user'] = users.UserManager.instance().get_user_from_access_key( - env['HTTP_X_AUTH_TOKEN'] - ) + env['HTTP_X_AUTH_TOKEN']) if rv['user']: rv['project'] = users.UserManager.instance().get_project( - rv['user'].name - ) + rv['user'].name) return rv class RackspaceApiEndpoint(object): def process(self, path, env): + """ + Main entrypoint for all controllers (what gets run by the wsgi handler). + Check authentication based on key, raise Unauthorized if invalid. + + Select the most appropriate action based on request type GET, POST, etc, + then pass it through to the implementing controller. Defalut to GET if + the implementing child doesn't respond to a particular type. + """ if not self.check_authentication(env): - # TODO(todd): Exception (Unauthorized) - raise Exception("Unable to authenticate") - - if len(path) == 0: + raise Unauthorized("Unable to authenticate") + + method = env['REQUEST_METHOD'].lower() + callback = getattr(self, method, None) + if not callback: + callback = getattr(self, "get") + logging.debug("%s processing %s with %s", self, method, callback) + return callback(path, env) + + def get(self, path, env): + """ + The default GET will look at the path and call an appropriate + action within this controller based on the the structure of the path. + + Given the following path lengths (with the first part stripped of by + router, as it is the controller name): + = 0 -> index + = 1 -> first component (/servers/details -> details) + >= 2 -> second path component (/servers/ID/ips/* -> ips) + + This should return + A String if 200 OK and no additional headers + (CODE, HEADERS, BODY) for custom response code and headers + """ + if len(path) == 0 and hasattr(self, "index"): + logging.debug("%s running index", self) return self.index(env) + if len(path) >= 2: + action = path[1] + else: + action = path.pop(0) - action = path.pop(0) + logging.debug("%s running action %s", self, action) if hasattr(self, action): method = getattr(self, action) return method(path, env) else: - # TODO(todd): Exception (404) - raise Exception("Missing method %s" % path[0]) + raise NotFound("Missing method %s" % path[0]) def check_authentication(self, env): - if hasattr(self, "process_without_authentication") \ - and getattr(self, "process_without_authentication"): - return True if not env['nova.context']['user']: return False return True -class RackspaceAuthenticationApi(RackspaceApiEndpoint): - - def __init__(self): - self.process_without_authentication = True +class RackspaceAuthenticationApi(object): # TODO(todd): make a actual session with a unique token # just pass the auth key back through for now - def index(self, env): + def index(self, _path, env): response = '204 No Content' headers = [ ('X-Server-Management-Url', 'http://%s' % env['HTTP_HOST']), @@ -141,20 +187,25 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): self.instdir = model.InstanceDirectory() self.network = network.PublicNetworkController() + def post(self, path, env): + if len(path) == 0: + return self.launch_server(env) + + def delete(self, path_parts, env): + if self.delete_server(path_parts[0]): + return ("202 Accepted", [], "") + else: + return ("404 Not Found", [], + "Did not find image, or it was not in a running state") + + def index(self, env): - if env['REQUEST_METHOD'] == 'GET': - return self.detail(env) - elif env['REQUEST_METHOD'] == 'POST': - return self.launch_server(env) + return self.detail(env) def detail(self, args, env): - value = { - "servers": - [] - } + value = {"servers": []} for inst in self.instdir.all: value["servers"].append(self.instance_details(inst)) - return json.dumps(value) ## @@ -227,3 +278,21 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): "args": {"instance_id": inst.instance_id} } ) + + def delete_server(self, instance_id): + owner_hostname = self.host_for_instance(instance_id) + # it isn't launched? + if not owner_hostname: + return None + rpc_transport = "%s:%s" % (FLAGS.compute_topic, owner_hostname) + rpc.cast(rpc_transport, + {"method": "reboot_instance", + "args": {"instance_id": instance_id}}) + return True + + def host_for_instance(self, instance_id): + instance = model.Instance.lookup(instance_id) + if not instance: + return None + return instance["node_name"] + -- cgit From 5326d7f76b48e93bd74d9539febe1f41bbf3f286 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 28 Jul 2010 22:41:49 -0700 Subject: Fix deprecation warning in AuthManager. __new__ isn't allowed to take args. --- nova/auth/manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2da53a736..2360c1a5c 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -24,7 +24,6 @@ import logging import os import shutil import string -import sys import tempfile import uuid import zipfile @@ -322,11 +321,10 @@ class AuthManager(object): need to be more accessible, such as vpn ips and ports. """ _instance=None - def __new__(cls, *args, **kwargs): + def __new__(cls): """Returns the AuthManager singleton""" if not cls._instance: - cls._instance = super(AuthManager, cls).__new__( - cls, *args, **kwargs) + cls._instance = super(AuthManager, cls).__new__(cls) return cls._instance def __init__(self, driver=None, *args, **kwargs): -- cgit From f8e7f79833b545a2812d0161f769271621fdf33c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 28 Jul 2010 23:19:07 -0700 Subject: oops retry and add extra exception check --- nova/auth/manager.py | 2 +- nova/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2360c1a5c..b690176bb 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -321,7 +321,7 @@ class AuthManager(object): need to be more accessible, such as vpn ips and ports. """ _instance=None - def __new__(cls): + def __new__(cls, *args, **kwargs): """Returns the AuthManager singleton""" if not cls._instance: cls._instance = super(AuthManager, cls).__new__(cls) diff --git a/nova/utils.py b/nova/utils.py index 0016b656e..0b23de7cd 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -41,7 +41,7 @@ def import_class(import_str): try: __import__(mod_str) return getattr(sys.modules[mod_str], class_str) - except (ImportError, AttributeError): + except (ImportError, ValueError, AttributeError): raise exception.NotFound('Class %s cannot be found' % class_str) def fetchfile(url, target): -- cgit From 9dfdbe11b89ae1f490257b6f687c2a5f5c90bb14 Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 29 Jul 2010 19:53:00 +0200 Subject: Add some useful features to our flags * No longer dies if there are unknown flags. * Allows you to declare that you will use a flag from another file * Allows you to import new flags at runtime and reparses the original arguments to fill them once they are accessed. --- nova/flags.py | 133 +++++++++++++++++++++++++++++++++++++++++-- nova/tests/declare_flags.py | 5 ++ nova/tests/flags_unittest.py | 94 ++++++++++++++++++++++++++++++ nova/tests/runtime_flags.py | 5 ++ run_tests.py | 1 + run_tests.sh | 2 +- 6 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 nova/tests/declare_flags.py create mode 100644 nova/tests/flags_unittest.py create mode 100644 nova/tests/runtime_flags.py diff --git a/nova/flags.py b/nova/flags.py index f35f5fa10..2ec7d9c9f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -21,16 +21,137 @@ Package-level global flags are defined here, the rest are defined where they're used. """ +import getopt import socket +import sys +import gflags -from gflags import * -# This keeps pylint from barfing on the imports -FLAGS = FLAGS -DEFINE_string = DEFINE_string -DEFINE_integer = DEFINE_integer -DEFINE_bool = DEFINE_bool +class FlagValues(gflags.FlagValues): + def __init__(self): + gflags.FlagValues.__init__(self) + self.__dict__['__dirty'] = [] + self.__dict__['__was_already_parsed'] = False + self.__dict__['__stored_argv'] = [] + + def __call__(self, argv): + # We're doing some hacky stuff here so that we don't have to copy + # out all the code of the original verbatim and then tweak a few lines. + # We're hijacking the output of getopt so we can still return the + # leftover args at the end + sneaky_unparsed_args = {"value": None} + original_argv = list(argv) + + if self.IsGnuGetOpt(): + orig_getopt = getattr(getopt, 'gnu_getopt') + orig_name = 'gnu_getopt' + else: + orig_getopt = getattr(getopt, 'getopt') + orig_name = 'getopt' + + def _sneaky(*args, **kw): + optlist, unparsed_args = orig_getopt(*args, **kw) + sneaky_unparsed_args['value'] = unparsed_args + return optlist, unparsed_args + + try: + setattr(getopt, orig_name, _sneaky) + args = gflags.FlagValues.__call__(self, argv) + except gflags.UnrecognizedFlagError: + # Undefined args were found, for now we don't care so just + # act like everything went well + # (these three lines are copied pretty much verbatim from the end + # of the __call__ function we are wrapping) + unparsed_args = sneaky_unparsed_args['value'] + if unparsed_args: + if self.IsGnuGetOpt(): + args = argv[:1] + unparsed + else: + args = argv[:1] + original_argv[-len(unparsed_args):] + else: + args = argv[:1] + finally: + setattr(getopt, orig_name, orig_getopt) + + # Store the arguments for later, we'll need them for new flags + # added at runtime + self.__dict__['__stored_argv'] = original_argv + self.__dict__['__was_already_parsed'] = True + self.ClearDirty() + return args + + def SetDirty(self, name): + """Mark a flag as dirty so that accessing it will case a reparse.""" + self.__dict__['__dirty'].append(name) + + def IsDirty(self, name): + return name in self.__dict__['__dirty'] + + def ClearDirty(self): + self.__dict__['__is_dirty'] = [] + + def WasAlreadyParsed(self): + return self.__dict__['__was_already_parsed'] + + def ParseNewFlags(self): + if '__stored_argv' not in self.__dict__: + return + new_flags = FlagValues() + for k in self.__dict__['__dirty']: + new_flags[k] = gflags.FlagValues.__getitem__(self, k) + + new_flags(self.__dict__['__stored_argv']) + for k in self.__dict__['__dirty']: + setattr(self, k, getattr(new_flags, k)) + self.ClearDirty() + + def __setitem__(self, name, flag): + gflags.FlagValues.__setitem__(self, name, flag) + if self.WasAlreadyParsed(): + self.SetDirty(name) + + def __getitem__(self, name): + if self.IsDirty(name): + self.ParseNewFlags() + return gflags.FlagValues.__getitem__(self, name) + + def __getattr__(self, name): + if self.IsDirty(name): + self.ParseNewFlags() + return gflags.FlagValues.__getattr__(self, name) + + +FLAGS = FlagValues() + + +def party_wrapper(func): + def _wrapped(*args, **kw): + kw.setdefault('flag_values', FLAGS) + func(*args, **kw) + _wrapped.func_name = func.func_name + return _wrapped + + +DEFINE_string = party_wrapper(gflags.DEFINE_string) +DEFINE_integer = party_wrapper(gflags.DEFINE_integer) +DEFINE_bool = party_wrapper(gflags.DEFINE_bool) +DEFINE_boolean = party_wrapper(gflags.DEFINE_boolean) +DEFINE_float = party_wrapper(gflags.DEFINE_float) +DEFINE_enum = party_wrapper(gflags.DEFINE_enum) +DEFINE_list = party_wrapper(gflags.DEFINE_list) +DEFINE_spaceseplist = party_wrapper(gflags.DEFINE_spaceseplist) +DEFINE_multistring = party_wrapper(gflags.DEFINE_multistring) +DEFINE_multi_int = party_wrapper(gflags.DEFINE_multi_int) + + +def DECLARE(name, module_string, flag_values=FLAGS): + if module_string not in sys.modules: + __import__(module_string, globals(), locals()) + if name not in flag_values: + raise gflags.UnrecognizedFlag( + "%s not defined by %s" % (name, module_string)) + # __GLOBAL FLAGS ONLY__ # Define any app-specific flags in their own files, docs at: diff --git a/nova/tests/declare_flags.py b/nova/tests/declare_flags.py new file mode 100644 index 000000000..f7c91f9dd --- /dev/null +++ b/nova/tests/declare_flags.py @@ -0,0 +1,5 @@ +from nova import flags + +FLAGS = flags.FLAGS + +flags.DEFINE_integer('answer', 42, 'test flag') diff --git a/nova/tests/flags_unittest.py b/nova/tests/flags_unittest.py new file mode 100644 index 000000000..44da94d04 --- /dev/null +++ b/nova/tests/flags_unittest.py @@ -0,0 +1,94 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +from twisted.internet import defer +from twisted.internet import reactor +from xml.etree import ElementTree + +from nova import exception +from nova import flags +from nova import process +from nova import test +from nova import utils + + +class FlagsTestCase(test.TrialTestCase): + def setUp(self): + super(FlagsTestCase, self).setUp() + self.FLAGS = flags.FlagValues() + self.global_FLAGS = flags.FLAGS + + def test_define(self): + self.assert_('string' not in self.FLAGS) + self.assert_('int' not in self.FLAGS) + self.assert_('false' not in self.FLAGS) + self.assert_('true' not in self.FLAGS) + + flags.DEFINE_string('string', 'default', 'desc', flag_values=self.FLAGS) + flags.DEFINE_integer('int', 1, 'desc', flag_values=self.FLAGS) + flags.DEFINE_bool('false', False, 'desc', flag_values=self.FLAGS) + flags.DEFINE_bool('true', True, 'desc', flag_values=self.FLAGS) + + self.assert_(self.FLAGS['string']) + self.assert_(self.FLAGS['int']) + self.assert_(self.FLAGS['false']) + self.assert_(self.FLAGS['true']) + self.assertEqual(self.FLAGS.string, 'default') + self.assertEqual(self.FLAGS.int, 1) + self.assertEqual(self.FLAGS.false, False) + self.assertEqual(self.FLAGS.true, True) + + argv = ['flags_test', + '--string', 'foo', + '--int', '2', + '--false', + '--notrue'] + + self.FLAGS(argv) + self.assertEqual(self.FLAGS.string, 'foo') + self.assertEqual(self.FLAGS.int, 2) + self.assertEqual(self.FLAGS.false, True) + self.assertEqual(self.FLAGS.true, False) + + def test_declare(self): + self.assert_('answer' not in self.global_FLAGS) + flags.DECLARE('answer', 'nova.tests.declare_flags') + self.assert_('answer' in self.global_FLAGS) + self.assertEqual(self.global_FLAGS.answer, 42) + + # Make sure we don't overwrite anything + self.global_FLAGS.answer = 256 + self.assertEqual(self.global_FLAGS.answer, 256) + flags.DECLARE('answer', 'nova.tests.declare_flags') + self.assertEqual(self.global_FLAGS.answer, 256) + + def test_runtime_and_unknown_flags(self): + self.assert_('runtime_answer' not in self.global_FLAGS) + + argv = ['flags_test', '--runtime_answer=60', 'extra_arg'] + args = self.global_FLAGS(argv) + self.assertEqual(len(args), 2) + self.assertEqual(args[1], 'extra_arg') + + self.assert_('runtime_answer' not in self.global_FLAGS) + + import nova.tests.runtime_flags + + self.assert_('runtime_answer' in self.global_FLAGS) + self.assertEqual(self.global_FLAGS.runtime_answer, 60) diff --git a/nova/tests/runtime_flags.py b/nova/tests/runtime_flags.py new file mode 100644 index 000000000..a2cc4738a --- /dev/null +++ b/nova/tests/runtime_flags.py @@ -0,0 +1,5 @@ +from nova import flags + +FLAGS = flags.FLAGS + +flags.DEFINE_integer('runtime_answer', 54, 'test flag') diff --git a/run_tests.py b/run_tests.py index 5a8966f02..14019b659 100644 --- a/run_tests.py +++ b/run_tests.py @@ -54,6 +54,7 @@ from nova.tests.auth_unittest import * from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * +from nova.tests.flags_unittest import * from nova.tests.model_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * diff --git a/run_tests.sh b/run_tests.sh index 1bf3d1a79..9b2de7aea 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,7 +4,7 @@ venv=.nova-venv with_venv=tools/with_venv.sh if [ -e ${venv} ]; then - ${with_venv} python run_tests.py + ${with_venv} python run_tests.py $@ else echo "You need to install the Nova virtualenv before you can run this." echo "" -- cgit From fe64d63240ee05e972731dbd97f76fcac3e1c5aa Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 29 Jul 2010 20:05:22 +0200 Subject: strip out some useless imports --- nova/tests/flags_unittest.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/nova/tests/flags_unittest.py b/nova/tests/flags_unittest.py index 44da94d04..d49d5dc43 100644 --- a/nova/tests/flags_unittest.py +++ b/nova/tests/flags_unittest.py @@ -16,16 +16,9 @@ # License for the specific language governing permissions and limitations # under the License. -import logging -from twisted.internet import defer -from twisted.internet import reactor -from xml.etree import ElementTree - from nova import exception from nova import flags -from nova import process from nova import test -from nova import utils class FlagsTestCase(test.TrialTestCase): -- cgit From 16e89bad15f5665a5f46c0bdcdfab1b7f3df4039 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 29 Jul 2010 17:58:44 -0700 Subject: flag for libvirt type --- nova/compute/libvirt.xml.template | 2 +- nova/virt/libvirt_conn.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/compute/libvirt.xml.template b/nova/compute/libvirt.xml.template index a763e8a4d..a17cd8fae 100644 --- a/nova/compute/libvirt.xml.template +++ b/nova/compute/libvirt.xml.template @@ -1,7 +1,7 @@ %(name)s - hvm + %(type)s %(basepath)s/kernel %(basepath)s/ramdisk root=/dev/vda1 console=ttyS0 diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c545e4190..e37444f63 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -48,6 +48,10 @@ flags.DEFINE_string('libvirt_xml_template', utils.abspath('compute/libvirt.xml.template'), 'Libvirt XML Template') +flags.DEFINE_string('libvirt_type', + 'hvm', + 'Libvirt virtualization type (hvm, qemu, etc)') + def get_connection(read_only): # These are loaded late so that there's no need to install these # libraries when not using libvirt. @@ -235,6 +239,7 @@ class LibvirtConnection(object): # TODO(termie): lazy lazy hack because xml is annoying xml_info['nova'] = json.dumps(instance.datamodel.copy()) + xml_info['type'] = FLAGS.libvirt_type libvirt_xml = libvirt_xml % xml_info logging.debug("Finished the toXML method") @@ -255,7 +260,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - + Returns a list of all block devices for this domain. """ domain = self._conn.lookupByName(instance_id) @@ -298,7 +303,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - + Returns a list of all network interfaces for this instance. """ domain = self._conn.lookupByName(instance_id) @@ -341,7 +346,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - """ + """ domain = self._conn.lookupByName(instance_id) return domain.blockStats(disk) @@ -350,6 +355,6 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - """ + """ domain = self._conn.lookupByName(instance_id) return domain.interfaceStats(interface) -- cgit From 80b79a923bc6fb331daaf6960e6353c700b89c41 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 11:19:03 -0700 Subject: use the right tag --- nova/compute/libvirt.xml.template | 5 ++--- nova/virt/libvirt_conn.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nova/compute/libvirt.xml.template b/nova/compute/libvirt.xml.template index a17cd8fae..307f9d03a 100644 --- a/nova/compute/libvirt.xml.template +++ b/nova/compute/libvirt.xml.template @@ -1,7 +1,7 @@ - + %(name)s - %(type)s + hvm %(basepath)s/kernel %(basepath)s/ramdisk root=/dev/vda1 console=ttyS0 @@ -12,7 +12,6 @@ %(memory_kb)s %(vcpus)s - /usr/bin/kvm diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e37444f63..74ab1f895 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -25,7 +25,6 @@ import json import logging import os.path import shutil -import sys from twisted.internet import defer from twisted.internet import task @@ -49,8 +48,8 @@ flags.DEFINE_string('libvirt_xml_template', 'Libvirt XML Template') flags.DEFINE_string('libvirt_type', - 'hvm', - 'Libvirt virtualization type (hvm, qemu, etc)') + 'kvm', + 'Libvirt domain type (kvm, qemu, etc)') def get_connection(read_only): # These are loaded late so that there's no need to install these -- cgit From 1934cbb0413f074213b1aeeda605d9b49055c581 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 15:19:41 -0700 Subject: Fixes access key passing in curl statement. --- nova/auth/manager.py | 4 ++++ nova/endpoint/images.py | 18 +++++++++++------- nova/virt/images.py | 14 +++++++++----- nova/virt/libvirt_conn.py | 16 ++++++++-------- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2da53a736..ca9f4fc86 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -419,6 +419,10 @@ class AuthManager(object): raise exception.NotAuthorized('Signature does not match') return (user, project) + def get_access_key(self, user, project): + """Get an access key that includes user and project""" + return "%s:%s" % (User.safe_id(user), Project.safe_id(project)) + def is_superuser(self, user): """Checks for superuser status, allowing user to bypass rbac diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index 32f7cc228..fe7cb5d11 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -27,6 +27,7 @@ import urllib from nova import flags from nova import utils +from nova.auth import manager FLAGS = flags.FLAGS @@ -75,13 +76,16 @@ def deregister(context, image_id): query_args=qs({'image_id': image_id})) def conn(context): - return boto.s3.connection.S3Connection ( - aws_access_key_id=str('%s:%s' % (context.user.access, context.project.name)), - aws_secret_access_key=str(context.user.secret), - is_secure=False, - calling_format=boto.s3.connection.OrdinaryCallingFormat(), - port=FLAGS.s3_port, - host=FLAGS.s3_host) + access = manager.AuthManager().get_access_key(context.user, + context.project) + secret = str(context.user.secret) + calling = boto.s3.connection.OrdinaryCallingFormat() + return boto.s3.connection.S3Connection(aws_access_key_id=access, + aws_secret_access_key=secret, + is_secure=False, + calling_format=calling, + port=FLAGS.s3_port, + host=FLAGS.s3_host) def qs(params): diff --git a/nova/virt/images.py b/nova/virt/images.py index 92210e242..872eb6d6a 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -27,6 +27,7 @@ import time from nova import flags from nova import process from nova.auth import signer +from nova.auth import manager FLAGS = flags.FLAGS @@ -34,14 +35,14 @@ flags.DEFINE_bool('use_s3', True, 'whether to get images from s3 or use local copy') -def fetch(image, path, user): +def fetch(image, path, user, project): if FLAGS.use_s3: f = _fetch_s3_image else: f = _fetch_local_image - return f(image, path, user) + return f(image, path, user, project) -def _fetch_s3_image(image, path, user): +def _fetch_s3_image(image, path, user, project): url = _image_url('%s/image' % image) # This should probably move somewhere else, like e.g. a download_as @@ -51,8 +52,11 @@ def _fetch_s3_image(image, path, user): headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) uri = '/' + url.partition('/')[2] - auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', uri) - headers['Authorization'] = 'AWS %s:%s' % (user.access, auth) + access = manager.AuthManager().get_access_key(user, project) + signature = signer.Signer(user.secret.encode()).s3_authorization(headers, + 'GET', + uri) + headers['Authorization'] = 'AWS %s:%s' % (access, signature) cmd = ['/usr/bin/curl', '--silent', url] for (k,v) in headers.iteritems(): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c545e4190..b3d514add 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -25,7 +25,6 @@ import json import logging import os.path import shutil -import sys from twisted.internet import defer from twisted.internet import task @@ -187,12 +186,13 @@ class LibvirtConnection(object): f.close() user = manager.AuthManager().get_user(data['user_id']) + project = manager.AuthManager().get_project(data['project_id']) if not os.path.exists(basepath('disk')): - yield images.fetch(data['image_id'], basepath('disk-raw'), user) + yield images.fetch(data['image_id'], basepath('disk-raw'), user, project) if not os.path.exists(basepath('kernel')): - yield images.fetch(data['kernel_id'], basepath('kernel'), user) + yield images.fetch(data['kernel_id'], basepath('kernel'), user, project) if not os.path.exists(basepath('ramdisk')): - yield images.fetch(data['ramdisk_id'], basepath('ramdisk'), user) + yield images.fetch(data['ramdisk_id'], basepath('ramdisk'), user, project) execute = lambda cmd, input=None: \ process.simple_execute(cmd=cmd, @@ -255,7 +255,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - + Returns a list of all block devices for this domain. """ domain = self._conn.lookupByName(instance_id) @@ -298,7 +298,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - + Returns a list of all network interfaces for this instance. """ domain = self._conn.lookupByName(instance_id) @@ -341,7 +341,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - """ + """ domain = self._conn.lookupByName(instance_id) return domain.blockStats(disk) @@ -350,6 +350,6 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - """ + """ domain = self._conn.lookupByName(instance_id) return domain.interfaceStats(interface) -- cgit From 490a97783b97c5753692099c4d7f609e29a8f74e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 15:36:11 -0700 Subject: use user.access instead of user.id --- nova/auth/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index ca9f4fc86..bf3a3556d 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -421,7 +421,9 @@ class AuthManager(object): def get_access_key(self, user, project): """Get an access key that includes user and project""" - return "%s:%s" % (User.safe_id(user), Project.safe_id(project)) + if not isinstance(user, User): + user = self.get_user(user) + return "%s:%s" % (user.access, Project.safe_id(project)) def is_superuser(self, user): """Checks for superuser status, allowing user to bypass rbac -- cgit From 04d6595d9b4c77f1fcaf01a7763caf11046ab164 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 16:15:09 -0700 Subject: another try on fix boto --- nova/auth/signer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/auth/signer.py b/nova/auth/signer.py index 3b9bc8f2c..634f22f0d 100644 --- a/nova/auth/signer.py +++ b/nova/auth/signer.py @@ -48,7 +48,8 @@ import hashlib import hmac import logging import urllib -import boto +import boto # NOTE(vish): for new boto +import boto.utils # NOTE(vish): for old boto from nova.exception import Error -- cgit From ed76ee9e823071c1c94db10907cc6a2bd725a999 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 18:32:12 -0700 Subject: Fixes nova volumes. The async commands yield properly. Simplified the call to create volume in cloud. Added some notes --- nova/endpoint/cloud.py | 11 +++++------ nova/volume/service.py | 43 ++++++++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 67fc04502..0ee278f84 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -295,17 +295,16 @@ class CloudController(object): return v @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it - res = rpc.call(FLAGS.volume_topic, {"method": "create_volume", + result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", "args" : {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) - def _format_result(result): - volume = self._get_volume(context, result['result']) - return {'volumeSet': [self.format_volume(context, volume)]} - res.addCallback(_format_result) - return res + # NOTE(vish): rpc returned value is in the result key in the dictionary + volume = self._get_volume(context, result['result']) + defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py diff --git a/nova/volume/service.py b/nova/volume/service.py index 87a47f40a..54496ad8d 100644 --- a/nova/volume/service.py +++ b/nova/volume/service.py @@ -104,6 +104,7 @@ class VolumeService(service.Service): pass @validate.rangetest(size=(0, 1000)) + @defer.inlineCallbacks def create_volume(self, size, user_id, project_id): """ Creates an exported volume (fake or real), @@ -111,11 +112,12 @@ class VolumeService(service.Service): Volume at this point has size, owner, and zone. """ logging.debug("Creating volume of size: %s" % (size)) - vol = self.volume_class.create(size, user_id, project_id) + vol = yield self.volume_class.create(size, user_id, project_id) datastore.Redis.instance().sadd('volumes', vol['volume_id']) datastore.Redis.instance().sadd('volumes:%s' % (FLAGS.storage_name), vol['volume_id']) - self._restart_exports() - return vol['volume_id'] + logging.debug("restarting exports") + yield self._restart_exports() + defer.returnValue(vol['volume_id']) def by_node(self, node_id): """ returns a list of volumes for a node """ @@ -128,6 +130,7 @@ class VolumeService(service.Service): for volume_id in datastore.Redis.instance().smembers('volumes'): yield self.volume_class(volume_id=volume_id) + @defer.inlineCallbacks def delete_volume(self, volume_id): logging.debug("Deleting volume with id of: %s" % (volume_id)) vol = get_volume(volume_id) @@ -135,19 +138,18 @@ class VolumeService(service.Service): raise exception.Error("Volume is still attached") if vol['node_name'] != FLAGS.storage_name: raise exception.Error("Volume is not local to this node") - vol.destroy() + yield vol.destroy() datastore.Redis.instance().srem('volumes', vol['volume_id']) datastore.Redis.instance().srem('volumes:%s' % (FLAGS.storage_name), vol['volume_id']) - return True + defer.returnValue(True) @defer.inlineCallbacks def _restart_exports(self): if FLAGS.fake_storage: return - yield process.simple_execute( - "sudo vblade-persist auto all") - yield process.simple_execute( - "sudo vblade-persist start all") + yield process.simple_execute("sudo vblade-persist auto all") + # NOTE(vish): this command sometimes sends output to stderr for warnings + yield process.simple_execute("sudo vblade-persist start all", error_ok=1) @defer.inlineCallbacks def _init_volume_group(self): @@ -173,6 +175,7 @@ class Volume(datastore.BasicModel): return {"volume_id": self.volume_id} @classmethod + @defer.inlineCallbacks def create(cls, size, user_id, project_id): volume_id = utils.generate_uid('vol') vol = cls(volume_id) @@ -188,13 +191,12 @@ class Volume(datastore.BasicModel): vol['attach_status'] = "detached" # attaching | attached | detaching | detached vol['delete_on_termination'] = 'False' vol.save() - vol.create_lv() - vol._setup_export() + yield vol._create_lv() + yield vol._setup_export() # TODO(joshua) - We need to trigger a fanout message for aoe-discover on all the nodes - # TODO(joshua vol['status'] = "available" vol.save() - return vol + defer.returnValue(vol) def start_attach(self, instance_id, mountpoint): """ """ @@ -223,16 +225,18 @@ class Volume(datastore.BasicModel): self['attach_status'] = "detached" self.save() + @defer.inlineCallbacks def destroy(self): try: - self._remove_export() - except: + yield self._remove_export() + except Exception as ex: + logging.debug("Ingnoring failure to remove export %s" % ex) pass - self._delete_lv() + yield self._delete_lv() super(Volume, self).destroy() @defer.inlineCallbacks - def create_lv(self): + def _create_lv(self): if str(self['size']) == '0': sizestr = '100M' else: @@ -248,13 +252,14 @@ class Volume(datastore.BasicModel): "sudo lvremove -f %s/%s" % (FLAGS.volume_group, self['volume_id'])) + @defer.inlineCallbacks def _setup_export(self): (shelf_id, blade_id) = get_next_aoe_numbers() self['aoe_device'] = "e%s.%s" % (shelf_id, blade_id) self['shelf_id'] = shelf_id self['blade_id'] = blade_id self.save() - self._exec_export() + yield self._exec_export() @defer.inlineCallbacks def _exec_export(self): @@ -277,7 +282,7 @@ class Volume(datastore.BasicModel): class FakeVolume(Volume): - def create_lv(self): + def _create_lv(self): pass def _exec_export(self): -- cgit From a3cc377f9dbe57195ef5f49f3f02a2178dc50cb1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 30 Jul 2010 19:33:07 -0700 Subject: Fix Tests --- nova/tests/volume_unittest.py | 20 ++++++++++---------- nova/volume/service.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index b536ac383..0f4f0e34d 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -42,15 +42,14 @@ class VolumeTestCase(test.TrialTestCase): vol_size = '0' user_id = 'fake' project_id = 'fake' - volume_id = self.volume.create_volume(vol_size, user_id, project_id) + volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) # TODO(termie): get_volume returns differently than create_volume self.assertEqual(volume_id, volume_service.get_volume(volume_id)['volume_id']) rv = self.volume.delete_volume(volume_id) - self.assertRaises(exception.Error, - volume_service.get_volume, - volume_id) + self.assertFailure(volume_service.get_volume(volume_id), + exception.Error) def test_too_big_volume(self): vol_size = '1001' @@ -68,13 +67,14 @@ class VolumeTestCase(test.TrialTestCase): total_slots = FLAGS.slots_per_shelf * num_shelves vols = [] for i in xrange(total_slots): - vid = self.volume.create_volume(vol_size, user_id, project_id) + vid = yield self.volume.create_volume(vol_size, user_id, project_id) vols.append(vid) - self.assertRaises(volume_service.NoMoreVolumes, - self.volume.create_volume, - vol_size, user_id, project_id) + self.assertFailure(self.volume.create_volume(vol_size, + user_id, + project_id), + volume_service.NoMoreVolumes) for id in vols: - self.volume.delete_volume(id) + yield self.volume.delete_volume(id) def test_run_attach_detach_volume(self): # Create one volume and one compute to test with @@ -83,7 +83,7 @@ class VolumeTestCase(test.TrialTestCase): user_id = "fake" project_id = 'fake' mountpoint = "/dev/sdf" - volume_id = self.volume.create_volume(vol_size, user_id, project_id) + volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) volume_obj = volume_service.get_volume(volume_id) volume_obj.start_attach(instance_id, mountpoint) diff --git a/nova/volume/service.py b/nova/volume/service.py index 54496ad8d..e12f675a7 100644 --- a/nova/volume/service.py +++ b/nova/volume/service.py @@ -103,8 +103,8 @@ class VolumeService(service.Service): except Exception, err: pass - @validate.rangetest(size=(0, 1000)) @defer.inlineCallbacks + @validate.rangetest(size=(0, 1000)) def create_volume(self, size, user_id, project_id): """ Creates an exported volume (fake or real), -- cgit From 83c4a429d29b7d69128d90504f6febc2efe1d3a3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 3 Aug 2010 01:29:31 -0700 Subject: Fixed instance model associations to host (node) and added association to ip --- nova/compute/model.py | 44 +++++++------- nova/datastore.py | 12 ++-- nova/tests/model_unittest.py | 135 ++++++++++++++++++++----------------------- 3 files changed, 95 insertions(+), 96 deletions(-) diff --git a/nova/compute/model.py b/nova/compute/model.py index 212830d3c..7dd130ca3 100644 --- a/nova/compute/model.py +++ b/nova/compute/model.py @@ -41,9 +41,6 @@ True """ import datetime -import logging -import time -import redis import uuid from nova import datastore @@ -72,19 +69,22 @@ class InstanceDirectory(object): for instance_id in datastore.Redis.instance().smembers('project:%s:instances' % project): yield Instance(instance_id) - def by_node(self, node_id): + @datastore.absorb_connection_error + def by_node(self, node): """returns a list of instances for a node""" + for instance_id in datastore.Redis.instance().smembers('node:%s:instances' % node): + yield Instance(instance_id) - for instance in self.all: - if instance['node_name'] == node_id: - yield instance - - def by_ip(self, ip_address): + def by_ip(self, ip): """returns an instance object that is using the IP""" - for instance in self.all: - if instance['private_dns_name'] == ip_address: - return instance - return None + # NOTE(vish): The ip association should be just a single value, but + # to maintain consistency it is using the standard + # association and the ugly method for retrieving + # the first item in the set below. + result = datastore.Redis.instance().smembers('ip:%s:instances' % ip) + if not result: + return None + return Instance(list(result)[0]) def by_volume(self, volume_id): """returns the instance a volume is attached to""" @@ -122,7 +122,8 @@ class Instance(datastore.BasicModel): 'instance_id': self.instance_id, 'node_name': 'unassigned', 'project_id': 'unassigned', - 'user_id': 'unassigned'} + 'user_id': 'unassigned', + 'private_dns_name': 'unassigned'} @property def identifier(self): @@ -148,19 +149,22 @@ class Instance(datastore.BasicModel): """Call into superclass to save object, then save associations""" # NOTE(todd): doesn't track migration between projects/nodes, # it just adds the first one - should_update_project = self.is_new_record() - should_update_node = self.is_new_record() + is_new = self.is_new_record() + node_set = (self.state['node_name'] != 'unassigned' and + self.initial_state['node_name'] == 'unassigned') success = super(Instance, self).save() - if success and should_update_project: + if success and is_new: self.associate_with("project", self.project) - if success and should_update_node: - self.associate_with("node", self['node_name']) + self.associate_with("ip", self.state['private_dns_name']) + if success and node_set: + self.associate_with("node", self.state['node_name']) return True def destroy(self): """Destroy associations, then destroy the object""" self.unassociate_with("project", self.project) - self.unassociate_with("node", self['node_name']) + self.unassociate_with("node", self.state['node_name']) + self.unassociate_with("ip", self.state['private_dns_name']) return super(Instance, self).destroy() class Host(datastore.BasicModel): diff --git a/nova/datastore.py b/nova/datastore.py index 9c2592334..51ef7a758 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -90,13 +90,15 @@ class BasicModel(object): @absorb_connection_error def __init__(self): - self.initial_state = {} - self.state = Redis.instance().hgetall(self.__redis_key) - if self.state: - self.initial_state = self.state + state = Redis.instance().hgetall(self.__redis_key) + if state: + self.initial_state = state + self.state = dict(self.initial_state) else: + self.initial_state = {} self.state = self.default_state() + def default_state(self): """You probably want to define this in your subclass""" return {} @@ -239,7 +241,7 @@ class BasicModel(object): for key, val in self.state.iteritems(): Redis.instance().hset(self.__redis_key, key, val) self.add_to_index() - self.initial_state = self.state + self.initial_state = dict(self.state) return True @absorb_connection_error diff --git a/nova/tests/model_unittest.py b/nova/tests/model_unittest.py index 6825cfe2a..dc2441c24 100644 --- a/nova/tests/model_unittest.py +++ b/nova/tests/model_unittest.py @@ -19,9 +19,7 @@ from datetime import datetime, timedelta import logging import time -from twisted.internet import defer -from nova import exception from nova import flags from nova import test from nova import utils @@ -49,9 +47,9 @@ class ModelTestCase(test.TrialTestCase): inst['user_id'] = 'fake' inst['project_id'] = 'fake' inst['instance_type'] = 'm1.tiny' - inst['node_name'] = FLAGS.node_name inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 + inst['private_dns_name'] = '10.0.0.1' inst.save() return inst @@ -71,118 +69,126 @@ class ModelTestCase(test.TrialTestCase): session_token.save() return session_token - @defer.inlineCallbacks def test_create_instance(self): """store with create_instace, then test that a load finds it""" - instance = yield self.create_instance() - old = yield model.Instance(instance.identifier) + instance = self.create_instance() + old = model.Instance(instance.identifier) self.assertFalse(old.is_new_record()) - @defer.inlineCallbacks def test_delete_instance(self): """create, then destroy, then make sure loads a new record""" - instance = yield self.create_instance() - yield instance.destroy() - newinst = yield model.Instance('i-test') + instance = self.create_instance() + instance.destroy() + newinst = model.Instance('i-test') self.assertTrue(newinst.is_new_record()) - @defer.inlineCallbacks def test_instance_added_to_set(self): - """create, then check that it is listed for the project""" - instance = yield self.create_instance() + """create, then check that it is listed in global set""" + instance = self.create_instance() found = False for x in model.InstanceDirectory().all: if x.identifier == 'i-test': found = True self.assert_(found) - @defer.inlineCallbacks def test_instance_associates_project(self): """create, then check that it is listed for the project""" - instance = yield self.create_instance() + instance = self.create_instance() found = False for x in model.InstanceDirectory().by_project(instance.project): if x.identifier == 'i-test': found = True self.assert_(found) - @defer.inlineCallbacks + def test_instance_associates_ip(self): + """create, then check that it is listed for the ip""" + instance = self.create_instance() + found = False + x = model.InstanceDirectory().by_ip(instance['private_dns_name']) + self.assertEqual(x.identifier, 'i-test') + + def test_instance_associates_node(self): + """create, then check that it is listed for the node_name""" + instance = self.create_instance() + found = False + for x in model.InstanceDirectory().by_node(FLAGS.node_name): + if x.identifier == 'i-test': + found = True + self.assertFalse(found) + instance['node_name'] = 'test_node' + instance.save() + for x in model.InstanceDirectory().by_node('test_node'): + if x.identifier == 'i-test': + found = True + self.assert_(found) + + def test_host_class_finds_hosts(self): - host = yield self.create_host() + host = self.create_host() self.assertEqual('testhost', model.Host.lookup('testhost').identifier) - @defer.inlineCallbacks def test_host_class_doesnt_find_missing_hosts(self): - rv = yield model.Host.lookup('woahnelly') + rv = model.Host.lookup('woahnelly') self.assertEqual(None, rv) - @defer.inlineCallbacks def test_create_host(self): """store with create_host, then test that a load finds it""" - host = yield self.create_host() - old = yield model.Host(host.identifier) + host = self.create_host() + old = model.Host(host.identifier) self.assertFalse(old.is_new_record()) - @defer.inlineCallbacks def test_delete_host(self): """create, then destroy, then make sure loads a new record""" - instance = yield self.create_host() - yield instance.destroy() - newinst = yield model.Host('testhost') + instance = self.create_host() + instance.destroy() + newinst = model.Host('testhost') self.assertTrue(newinst.is_new_record()) - @defer.inlineCallbacks def test_host_added_to_set(self): """create, then check that it is included in list""" - instance = yield self.create_host() + instance = self.create_host() found = False for x in model.Host.all(): if x.identifier == 'testhost': found = True self.assert_(found) - @defer.inlineCallbacks def test_create_daemon_two_args(self): """create a daemon with two arguments""" - d = yield self.create_daemon() + d = self.create_daemon() d = model.Daemon('testhost', 'nova-testdaemon') self.assertFalse(d.is_new_record()) - @defer.inlineCallbacks def test_create_daemon_single_arg(self): """Create a daemon using the combined host:bin format""" - d = yield model.Daemon("testhost:nova-testdaemon") + d = model.Daemon("testhost:nova-testdaemon") d.save() d = model.Daemon('testhost:nova-testdaemon') self.assertFalse(d.is_new_record()) - @defer.inlineCallbacks def test_equality_of_daemon_single_and_double_args(self): """Create a daemon using the combined host:bin arg, find with 2""" - d = yield model.Daemon("testhost:nova-testdaemon") + d = model.Daemon("testhost:nova-testdaemon") d.save() d = model.Daemon('testhost', 'nova-testdaemon') self.assertFalse(d.is_new_record()) - @defer.inlineCallbacks def test_equality_daemon_of_double_and_single_args(self): """Create a daemon using the combined host:bin arg, find with 2""" - d = yield self.create_daemon() + d = self.create_daemon() d = model.Daemon('testhost:nova-testdaemon') self.assertFalse(d.is_new_record()) - @defer.inlineCallbacks def test_delete_daemon(self): """create, then destroy, then make sure loads a new record""" - instance = yield self.create_daemon() - yield instance.destroy() - newinst = yield model.Daemon('testhost', 'nova-testdaemon') + instance = self.create_daemon() + instance.destroy() + newinst = model.Daemon('testhost', 'nova-testdaemon') self.assertTrue(newinst.is_new_record()) - @defer.inlineCallbacks def test_daemon_heartbeat(self): """Create a daemon, sleep, heartbeat, check for update""" - d = yield self.create_daemon() + d = self.create_daemon() ts = d['updated_at'] time.sleep(2) d.heartbeat() @@ -190,70 +196,62 @@ class ModelTestCase(test.TrialTestCase): ts2 = d2['updated_at'] self.assert_(ts2 > ts) - @defer.inlineCallbacks def test_daemon_added_to_set(self): """create, then check that it is included in list""" - instance = yield self.create_daemon() + instance = self.create_daemon() found = False for x in model.Daemon.all(): if x.identifier == 'testhost:nova-testdaemon': found = True self.assert_(found) - @defer.inlineCallbacks def test_daemon_associates_host(self): """create, then check that it is listed for the host""" - instance = yield self.create_daemon() + instance = self.create_daemon() found = False for x in model.Daemon.by_host('testhost'): if x.identifier == 'testhost:nova-testdaemon': found = True self.assertTrue(found) - @defer.inlineCallbacks def test_create_session_token(self): """create""" - d = yield self.create_session_token() + d = self.create_session_token() d = model.SessionToken(d.token) self.assertFalse(d.is_new_record()) - @defer.inlineCallbacks def test_delete_session_token(self): """create, then destroy, then make sure loads a new record""" - instance = yield self.create_session_token() - yield instance.destroy() - newinst = yield model.SessionToken(instance.token) + instance = self.create_session_token() + instance.destroy() + newinst = model.SessionToken(instance.token) self.assertTrue(newinst.is_new_record()) - @defer.inlineCallbacks def test_session_token_added_to_set(self): """create, then check that it is included in list""" - instance = yield self.create_session_token() + instance = self.create_session_token() found = False for x in model.SessionToken.all(): if x.identifier == instance.token: found = True self.assert_(found) - @defer.inlineCallbacks def test_session_token_associates_user(self): """create, then check that it is listed for the user""" - instance = yield self.create_session_token() + instance = self.create_session_token() found = False for x in model.SessionToken.associated_to('user', 'testuser'): if x.identifier == instance.identifier: found = True self.assertTrue(found) - @defer.inlineCallbacks def test_session_token_generation(self): - instance = yield model.SessionToken.generate('username', 'TokenType') + instance = model.SessionToken.generate('username', 'TokenType') self.assertFalse(instance.is_new_record()) - @defer.inlineCallbacks def test_find_generated_session_token(self): - instance = yield model.SessionToken.generate('username', 'TokenType') - found = yield model.SessionToken.lookup(instance.identifier) + instance = model.SessionToken.generate('username', 'TokenType') + found = model.SessionToken.lookup(instance.identifier) self.assert_(found) def test_update_session_token_expiry(self): @@ -264,34 +262,29 @@ class ModelTestCase(test.TrialTestCase): expiry = utils.parse_isotime(instance['expiry']) self.assert_(expiry > datetime.utcnow()) - @defer.inlineCallbacks def test_session_token_lookup_when_expired(self): - instance = yield model.SessionToken.generate("testuser") + instance = model.SessionToken.generate("testuser") instance['expiry'] = datetime.utcnow().strftime(utils.TIME_FORMAT) instance.save() inst = model.SessionToken.lookup(instance.identifier) self.assertFalse(inst) - @defer.inlineCallbacks def test_session_token_lookup_when_not_expired(self): - instance = yield model.SessionToken.generate("testuser") + instance = model.SessionToken.generate("testuser") inst = model.SessionToken.lookup(instance.identifier) self.assert_(inst) - @defer.inlineCallbacks def test_session_token_is_expired_when_expired(self): - instance = yield model.SessionToken.generate("testuser") + instance = model.SessionToken.generate("testuser") instance['expiry'] = datetime.utcnow().strftime(utils.TIME_FORMAT) self.assert_(instance.is_expired()) - @defer.inlineCallbacks def test_session_token_is_expired_when_not_expired(self): - instance = yield model.SessionToken.generate("testuser") + instance = model.SessionToken.generate("testuser") self.assertFalse(instance.is_expired()) - @defer.inlineCallbacks def test_session_token_ttl(self): - instance = yield model.SessionToken.generate("testuser") + instance = model.SessionToken.generate("testuser") now = datetime.utcnow() delta = timedelta(hours=1) instance['expiry'] = (now + delta).strftime(utils.TIME_FORMAT) -- cgit From e95aac3ac93dabd35eb86951fdc270e06d2b2622 Mon Sep 17 00:00:00 2001 From: andy Date: Tue, 3 Aug 2010 16:51:37 +0200 Subject: add copyright headers --- nova/tests/declare_flags.py | 18 ++++++++++++++++++ nova/tests/runtime_flags.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/nova/tests/declare_flags.py b/nova/tests/declare_flags.py index f7c91f9dd..51a55ec72 100644 --- a/nova/tests/declare_flags.py +++ b/nova/tests/declare_flags.py @@ -1,3 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from nova import flags FLAGS = flags.FLAGS diff --git a/nova/tests/runtime_flags.py b/nova/tests/runtime_flags.py index a2cc4738a..1eb501406 100644 --- a/nova/tests/runtime_flags.py +++ b/nova/tests/runtime_flags.py @@ -1,3 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from nova import flags FLAGS = flags.FLAGS -- cgit From 6dde6fdc10bdb6f75fabce1b0e7a6c4e031937ea Mon Sep 17 00:00:00 2001 From: andy Date: Tue, 3 Aug 2010 18:00:11 +0200 Subject: updated doc string and wrapper --- nova/flags.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 2ec7d9c9f..b3bdd088f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -29,6 +29,14 @@ import gflags class FlagValues(gflags.FlagValues): + """Extension of gflags.FlagValues that allows undefined and runtime flags. + + Unknown flags will be ignored when parsing the command line, but the + command line will be kept so that it can be replayed if new flags are + defined after the initial parsing. + + """ + def __init__(self): gflags.FlagValues.__init__(self) self.__dict__['__dirty'] = [] @@ -125,7 +133,7 @@ class FlagValues(gflags.FlagValues): FLAGS = FlagValues() -def party_wrapper(func): +def _wrapper(func): def _wrapped(*args, **kw): kw.setdefault('flag_values', FLAGS) func(*args, **kw) @@ -133,16 +141,16 @@ def party_wrapper(func): return _wrapped -DEFINE_string = party_wrapper(gflags.DEFINE_string) -DEFINE_integer = party_wrapper(gflags.DEFINE_integer) -DEFINE_bool = party_wrapper(gflags.DEFINE_bool) -DEFINE_boolean = party_wrapper(gflags.DEFINE_boolean) -DEFINE_float = party_wrapper(gflags.DEFINE_float) -DEFINE_enum = party_wrapper(gflags.DEFINE_enum) -DEFINE_list = party_wrapper(gflags.DEFINE_list) -DEFINE_spaceseplist = party_wrapper(gflags.DEFINE_spaceseplist) -DEFINE_multistring = party_wrapper(gflags.DEFINE_multistring) -DEFINE_multi_int = party_wrapper(gflags.DEFINE_multi_int) +DEFINE_string = _wrapper(gflags.DEFINE_string) +DEFINE_integer = _wrapper(gflags.DEFINE_integer) +DEFINE_bool = _wrapper(gflags.DEFINE_bool) +DEFINE_boolean = _wrapper(gflags.DEFINE_boolean) +DEFINE_float = _wrapper(gflags.DEFINE_float) +DEFINE_enum = _wrapper(gflags.DEFINE_enum) +DEFINE_list = _wrapper(gflags.DEFINE_list) +DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) +DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) +DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) def DECLARE(name, module_string, flag_values=FLAGS): -- cgit From ecf8608a84960496c6c8e350f99d53537e4581c8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 3 Aug 2010 14:31:47 -0700 Subject: Huge network refactor, Round I Made network into its own binary Made simple network a plugabble class Fixed unittests Moved various classes around Moved mac generation into network class --- bin/nova-dhcpbridge | 22 +- bin/nova-network | 11 +- nova/compute/exception.py | 40 --- nova/compute/linux_net.py | 181 ------------- nova/compute/network.py | 597 ----------------------------------------- nova/compute/service.py | 6 +- nova/endpoint/cloud.py | 129 +++++---- nova/network/exception.py | 40 +++ nova/network/linux_net.py | 181 +++++++++++++ nova/network/model.py | 549 +++++++++++++++++++++++++++++++++++++ nova/network/service.py | 135 +++++++++- nova/tests/network_unittest.py | 99 ++++--- 12 files changed, 1050 insertions(+), 940 deletions(-) delete mode 100644 nova/compute/exception.py delete mode 100644 nova/compute/linux_net.py delete mode 100644 nova/compute/network.py create mode 100644 nova/network/exception.py create mode 100644 nova/network/linux_net.py create mode 100644 nova/network/model.py diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 0db241b5e..b3e7d456a 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -35,32 +35,34 @@ sys.path.append(os.path.abspath(os.path.join(__file__, "../../"))) from nova import flags from nova import rpc from nova import utils -from nova.compute import linux_net -from nova.compute import network - +from nova.network import linux_net +from nova.network import model +from nova.network import service FLAGS = flags.FLAGS def add_lease(mac, ip, hostname, interface): if FLAGS.fake_rabbit: - network.lease_ip(ip) + service.VlanNetworkService().lease_ip(ip) else: - rpc.cast(FLAGS.cloud_topic, {"method": "lease_ip", - "args" : {"address": ip}}) + rpc.cast("%s.%s" (FLAGS.network_topic, FLAGS.node_name), + {"method": "lease_ip", + "args" : {"fixed_ip": ip}}) def old_lease(mac, ip, hostname, interface): logging.debug("Adopted old lease or got a change of mac/hostname") def del_lease(mac, ip, hostname, interface): if FLAGS.fake_rabbit: - network.release_ip(ip) + service.VlanNetworkService().release_ip(ip) else: - rpc.cast(FLAGS.cloud_topic, {"method": "release_ip", - "args" : {"address": ip}}) + rpc.cast("%s.%s" (FLAGS.network_topic, FLAGS.node_name), + {"method": "release_ip", + "args" : {"fixed_ip": ip}}) def init_leases(interface): - net = network.get_network_by_interface(interface) + net = model.get_network_by_interface(interface) res = "" for host_name in net.hosts: res += "%s\n" % linux_net.hostDHCP(net, host_name, net.hosts[host_name]) diff --git a/bin/nova-network b/bin/nova-network index 52d6cb70a..b2e2cf173 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -21,12 +21,19 @@ Twistd daemon for the nova network nodes. """ +from nova import flags from nova import twistd -from nova.network import service +from nova import utils +FLAGS = flags.FLAGS + +flags.DEFINE_string('network_service', + 'nova.network.service.VlanNetworkService', + 'Service Class for Networking') + if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.NetworkService.create() + application = utils.import_class(FLAGS.network_service).create() diff --git a/nova/compute/exception.py b/nova/compute/exception.py deleted file mode 100644 index 13e4f0a51..000000000 --- a/nova/compute/exception.py +++ /dev/null @@ -1,40 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exceptions for Compute Node errors, mostly network addressing. -""" - -from nova.exception import Error - - -class NoMoreAddresses(Error): - pass - -class AddressNotAllocated(Error): - pass - -class AddressAlreadyAssociated(Error): - pass - -class AddressNotAssociated(Error): - pass - -class NotValidNetworkSize(Error): - pass - diff --git a/nova/compute/linux_net.py b/nova/compute/linux_net.py deleted file mode 100644 index 4a4b4c8a8..000000000 --- a/nova/compute/linux_net.py +++ /dev/null @@ -1,181 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import signal -import os -import subprocess - -# todo(ja): does the definition of network_path belong here? - -from nova import utils - -from nova import flags -FLAGS=flags.FLAGS - -flags.DEFINE_string('dhcpbridge_flagfile', - '/etc/nova/nova-dhcpbridge.conf', - 'location of flagfile for dhcpbridge') - -def execute(cmd, addl_env=None): - if FLAGS.fake_network: - logging.debug("FAKE NET: %s" % cmd) - return "fake", 0 - else: - return utils.execute(cmd, addl_env=addl_env) - -def runthis(desc, cmd): - if FLAGS.fake_network: - return execute(cmd) - else: - return utils.runthis(desc,cmd) - -def Popen(cmd): - if FLAGS.fake_network: - execute(' '.join(cmd)) - else: - subprocess.Popen(cmd) - - -def device_exists(device): - (out, err) = execute("ifconfig %s" % device) - return not err - -def confirm_rule(cmd): - execute("sudo iptables --delete %s" % (cmd)) - execute("sudo iptables -I %s" % (cmd)) - -def remove_rule(cmd): - execute("sudo iptables --delete %s" % (cmd)) - -def bind_public_ip(ip, interface): - runthis("Binding IP to interface: %s", "sudo ip addr add %s dev %s" % (ip, interface)) - -def unbind_public_ip(ip, interface): - runthis("Binding IP to interface: %s", "sudo ip addr del %s dev %s" % (ip, interface)) - -def vlan_create(net): - """ create a vlan on on a bridge device unless vlan already exists """ - if not device_exists("vlan%s" % net['vlan']): - logging.debug("Starting VLAN inteface for %s network", (net['vlan'])) - execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") - execute("sudo vconfig add %s %s" % (FLAGS.bridge_dev, net['vlan'])) - execute("sudo ifconfig vlan%s up" % (net['vlan'])) - -def bridge_create(net): - """ create a bridge on a vlan unless it already exists """ - if not device_exists(net['bridge_name']): - logging.debug("Starting Bridge inteface for %s network", (net['vlan'])) - execute("sudo brctl addbr %s" % (net['bridge_name'])) - execute("sudo brctl setfd %s 0" % (net.bridge_name)) - # execute("sudo brctl setageing %s 10" % (net.bridge_name)) - execute("sudo brctl stp %s off" % (net['bridge_name'])) - execute("sudo brctl addif %s vlan%s" % (net['bridge_name'], net['vlan'])) - if net.bridge_gets_ip: - execute("sudo ifconfig %s %s broadcast %s netmask %s up" % \ - (net['bridge_name'], net.gateway, net.broadcast, net.netmask)) - confirm_rule("FORWARD --in-interface %s -j ACCEPT" % (net['bridge_name'])) - else: - execute("sudo ifconfig %s up" % net['bridge_name']) - -def dnsmasq_cmd(net): - cmd = ['sudo -E dnsmasq', - ' --strict-order', - ' --bind-interfaces', - ' --conf-file=', - ' --pid-file=%s' % dhcp_file(net['vlan'], 'pid'), - ' --listen-address=%s' % net.dhcp_listen_address, - ' --except-interface=lo', - ' --dhcp-range=%s,static,600s' % (net.dhcp_range_start), - ' --dhcp-hostsfile=%s' % dhcp_file(net['vlan'], 'conf'), - ' --dhcp-script=%s' % bin_file('nova-dhcpbridge'), - ' --leasefile-ro'] - return ''.join(cmd) - -def hostDHCP(network, host, mac): - idx = host.split(".")[-1] # Logically, the idx of instances they've launched in this net - return "%s,%s-%s-%s.novalocal,%s" % \ - (mac, network['user_id'], network['vlan'], idx, host) - -# todo(ja): if the system has restarted or pid numbers have wrapped -# then you cannot be certain that the pid refers to the -# dnsmasq. As well, sending a HUP only reloads the hostfile, -# so any configuration options (like dchp-range, vlan, ...) -# aren't reloaded -def start_dnsmasq(network): - """ (re)starts a dnsmasq server for a given network - - if a dnsmasq instance is already running then send a HUP - signal causing it to reload, otherwise spawn a new instance - """ - with open(dhcp_file(network['vlan'], 'conf'), 'w') as f: - for host_name in network.hosts: - f.write("%s\n" % hostDHCP(network, host_name, network.hosts[host_name])) - - pid = dnsmasq_pid_for(network) - - # if dnsmasq is already running, then tell it to reload - if pid: - # todo(ja): use "/proc/%d/cmdline" % (pid) to determine if pid refers - # correct dnsmasq process - try: - os.kill(pid, signal.SIGHUP) - except Exception, e: - logging.debug("Hupping dnsmasq threw %s", e) - - # otherwise delete the existing leases file and start dnsmasq - lease_file = dhcp_file(network['vlan'], 'leases') - if os.path.exists(lease_file): - os.unlink(lease_file) - - # FLAGFILE and DNSMASQ_INTERFACE in env - env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, - 'DNSMASQ_INTERFACE': network['bridge_name']} - execute(dnsmasq_cmd(network), addl_env=env) - -def stop_dnsmasq(network): - """ stops the dnsmasq instance for a given network """ - pid = dnsmasq_pid_for(network) - - if pid: - try: - os.kill(pid, signal.SIGTERM) - except Exception, e: - logging.debug("Killing dnsmasq threw %s", e) - -def dhcp_file(vlan, kind): - """ return path to a pid, leases or conf file for a vlan """ - - return os.path.abspath("%s/nova-%s.%s" % (FLAGS.networks_path, vlan, kind)) - -def bin_file(script): - return os.path.abspath(os.path.join(__file__, "../../../bin", script)) - -def dnsmasq_pid_for(network): - """ the pid for prior dnsmasq instance for a vlan, - returns None if no pid file exists - - if machine has rebooted pid might be incorrect (caller should check) - """ - - pid_file = dhcp_file(network['vlan'], 'pid') - - if os.path.exists(pid_file): - with open(pid_file, 'r') as f: - return int(f.read()) - diff --git a/nova/compute/network.py b/nova/compute/network.py deleted file mode 100644 index 62d892e58..000000000 --- a/nova/compute/network.py +++ /dev/null @@ -1,597 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Classes for network control, including VLANs, DHCP, and IP allocation. -""" - -import IPy -import logging -import os -import time - -from nova import datastore -from nova import exception -from nova import flags -from nova import utils -from nova.auth import manager -from nova.compute import exception as compute_exception -from nova.compute import linux_net - - -FLAGS = flags.FLAGS -flags.DEFINE_string('networks_path', utils.abspath('../networks'), - 'Location to keep network config files') -flags.DEFINE_integer('public_vlan', 1, 'VLAN for public IP addresses') -flags.DEFINE_string('public_interface', 'vlan1', - 'Interface for public IP addresses') -flags.DEFINE_string('bridge_dev', 'eth1', - 'network device for bridges') -flags.DEFINE_integer('vlan_start', 100, 'First VLAN for private networks') -flags.DEFINE_integer('vlan_end', 4093, 'Last VLAN for private networks') -flags.DEFINE_integer('network_size', 256, - 'Number of addresses in each private subnet') -flags.DEFINE_string('public_range', '4.4.4.0/24', 'Public IP address block') -flags.DEFINE_string('private_range', '10.0.0.0/8', 'Private IP address block') -flags.DEFINE_integer('cnt_vpn_clients', 5, - 'Number of addresses reserved for vpn clients') -flags.DEFINE_integer('cloudpipe_start_port', 12000, - 'Starting port for mapped CloudPipe external ports') - -flags.DEFINE_boolean('simple_network', False, - 'Use simple networking instead of vlans') -flags.DEFINE_string('simple_network_bridge', 'br100', - 'Bridge for simple network instances') -flags.DEFINE_list('simple_network_ips', ['192.168.0.2'], - 'Available ips for simple network') -flags.DEFINE_string('simple_network_template', - utils.abspath('compute/interfaces.template'), - 'Template file for simple network') -flags.DEFINE_string('simple_network_netmask', '255.255.255.0', - 'Netmask for simple network') -flags.DEFINE_string('simple_network_network', '192.168.0.0', - 'Network for simple network') -flags.DEFINE_string('simple_network_gateway', '192.168.0.1', - 'Broadcast for simple network') -flags.DEFINE_string('simple_network_broadcast', '192.168.0.255', - 'Broadcast for simple network') -flags.DEFINE_string('simple_network_dns', '8.8.4.4', - 'Dns for simple network') - -logging.getLogger().setLevel(logging.DEBUG) - - -class Vlan(datastore.BasicModel): - def __init__(self, project, vlan): - """ - Since we don't want to try and find a vlan by its identifier, - but by a project id, we don't call super-init. - """ - self.project_id = project - self.vlan_id = vlan - - @property - def identifier(self): - return "%s:%s" % (self.project_id, self.vlan_id) - - @classmethod - def create(cls, project, vlan): - instance = cls(project, vlan) - instance.save() - return instance - - @classmethod - @datastore.absorb_connection_error - def lookup(cls, project): - set_name = cls._redis_set_name(cls.__name__) - vlan = datastore.Redis.instance().hget(set_name, project) - if vlan: - return cls(project, vlan) - else: - return None - - @classmethod - @datastore.absorb_connection_error - def dict_by_project(cls): - """a hash of project:vlan""" - set_name = cls._redis_set_name(cls.__name__) - return datastore.Redis.instance().hgetall(set_name) - - @classmethod - @datastore.absorb_connection_error - def dict_by_vlan(cls): - """a hash of vlan:project""" - set_name = cls._redis_set_name(cls.__name__) - rv = {} - h = datastore.Redis.instance().hgetall(set_name) - for v in h.keys(): - rv[h[v]] = v - return rv - - @classmethod - @datastore.absorb_connection_error - def all(cls): - set_name = cls._redis_set_name(cls.__name__) - elements = datastore.Redis.instance().hgetall(set_name) - for project in elements: - yield cls(project, elements[project]) - - @datastore.absorb_connection_error - def save(self): - """ - Vlan saves state into a giant hash named "vlans", with keys of - project_id and value of vlan number. Therefore, we skip the - default way of saving into "vlan:ID" and adding to a set of "vlans". - """ - set_name = self._redis_set_name(self.__class__.__name__) - datastore.Redis.instance().hset(set_name, self.project_id, self.vlan_id) - - @datastore.absorb_connection_error - def destroy(self): - set_name = self._redis_set_name(self.__class__.__name__) - datastore.Redis.instance().hdel(set_name, self.project_id) - - def subnet(self): - vlan = int(self.vlan_id) - network = IPy.IP(FLAGS.private_range) - start = (vlan-FLAGS.vlan_start) * FLAGS.network_size - # minus one for the gateway. - return "%s-%s" % (network[start], - network[start + FLAGS.network_size - 1]) - -# CLEANUP: -# TODO(ja): Save the IPs at the top of each subnet for cloudpipe vpn clients -# TODO(ja): use singleton for usermanager instead of self.manager in vlanpool et al -# TODO(ja): does vlanpool "keeper" need to know the min/max - shouldn't FLAGS always win? -# TODO(joshua): Save the IPs at the top of each subnet for cloudpipe vpn clients - -class BaseNetwork(datastore.BasicModel): - override_type = 'network' - NUM_STATIC_IPS = 3 # Network, Gateway, and CloudPipe - - @property - def identifier(self): - return self.network_id - - def default_state(self): - return {'network_id': self.network_id, 'network_str': self.network_str} - - @classmethod - def create(cls, user_id, project_id, security_group, vlan, network_str): - network_id = "%s:%s" % (project_id, security_group) - net = cls(network_id, network_str) - net['user_id'] = user_id - net['project_id'] = project_id - net["vlan"] = vlan - net["bridge_name"] = "br%s" % vlan - net.save() - return net - - def __init__(self, network_id, network_str=None): - self.network_id = network_id - self.network_str = network_str - super(BaseNetwork, self).__init__() - self.save() - - @property - def network(self): - return IPy.IP(self['network_str']) - - @property - def netmask(self): - return self.network.netmask() - - @property - def gateway(self): - return self.network[1] - - @property - def broadcast(self): - return self.network.broadcast() - - @property - def bridge_name(self): - return "br%s" % (self["vlan"]) - - @property - def user(self): - return manager.AuthManager().get_user(self['user_id']) - - @property - def project(self): - return manager.AuthManager().get_project(self['project_id']) - - @property - def _hosts_key(self): - return "network:%s:hosts" % (self['network_str']) - - @property - def hosts(self): - return datastore.Redis.instance().hgetall(self._hosts_key) or {} - - def _add_host(self, _user_id, _project_id, host, target): - datastore.Redis.instance().hset(self._hosts_key, host, target) - - def _rem_host(self, host): - datastore.Redis.instance().hdel(self._hosts_key, host) - - @property - def assigned(self): - return datastore.Redis.instance().hkeys(self._hosts_key) - - @property - def available(self): - # the .2 address is always CloudPipe - # and the top are for vpn clients - for idx in range(self.num_static_ips, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): - address = str(self.network[idx]) - if not address in self.hosts.keys(): - yield str(address) - - @property - def num_static_ips(self): - return BaseNetwork.NUM_STATIC_IPS - - def allocate_ip(self, user_id, project_id, mac): - for address in self.available: - logging.debug("Allocating IP %s to %s" % (address, project_id)) - self._add_host(user_id, project_id, address, mac) - self.express(address=address) - return address - raise compute_exception.NoMoreAddresses("Project %s with network %s" % - (project_id, str(self.network))) - - def lease_ip(self, ip_str): - logging.debug("Leasing allocated IP %s" % (ip_str)) - - def release_ip(self, ip_str): - if not ip_str in self.assigned: - raise compute_exception.AddressNotAllocated() - self.deexpress(address=ip_str) - self._rem_host(ip_str) - - def deallocate_ip(self, ip_str): - # Do nothing for now, cleanup on ip release - pass - - def list_addresses(self): - for address in self.hosts: - yield address - - def express(self, address=None): pass - def deexpress(self, address=None): pass - - -class BridgedNetwork(BaseNetwork): - """ - Virtual Network that can express itself to create a vlan and - a bridge (with or without an IP address/netmask/gateway) - - properties: - bridge_name - string (example value: br42) - vlan - integer (example value: 42) - bridge_dev - string (example: eth0) - bridge_gets_ip - boolean used during bridge creation - - if bridge_gets_ip then network address for bridge uses the properties: - gateway - broadcast - netmask - """ - - bridge_gets_ip = False - override_type = 'network' - - @classmethod - def get_network_for_project(cls, user_id, project_id, security_group): - vlan = get_vlan_for_project(project_id) - network_str = vlan.subnet() - return cls.create(user_id, project_id, security_group, vlan.vlan_id, - network_str) - - def __init__(self, *args, **kwargs): - super(BridgedNetwork, self).__init__(*args, **kwargs) - self['bridge_dev'] = FLAGS.bridge_dev - self.save() - - def express(self, address=None): - super(BridgedNetwork, self).express(address=address) - linux_net.vlan_create(self) - linux_net.bridge_create(self) - -class DHCPNetwork(BridgedNetwork): - """ - properties: - dhcp_listen_address: the ip of the gateway / dhcp host - dhcp_range_start: the first ip to give out - dhcp_range_end: the last ip to give out - """ - bridge_gets_ip = True - override_type = 'network' - - def __init__(self, *args, **kwargs): - super(DHCPNetwork, self).__init__(*args, **kwargs) - # logging.debug("Initing DHCPNetwork object...") - self.dhcp_listen_address = self.network[1] - self.dhcp_range_start = self.network[3] - self.dhcp_range_end = self.network[-(1 + FLAGS.cnt_vpn_clients)] - try: - os.makedirs(FLAGS.networks_path) - # NOTE(todd): I guess this is a lazy way to not have to check if the - # directory exists, but shouldn't we be smarter about - # telling the difference between existing directory and - # permission denied? (Errno 17 vs 13, OSError) - except Exception, err: - pass - - def express(self, address=None): - super(DHCPNetwork, self).express(address=address) - if len(self.assigned) > 0: - logging.debug("Starting dnsmasq server for network with vlan %s", - self['vlan']) - linux_net.start_dnsmasq(self) - else: - logging.debug("Not launching dnsmasq: no hosts.") - self.express_cloudpipe() - - def allocate_vpn_ip(self, mac): - address = str(self.network[2]) - self._add_host(self['user_id'], self['project_id'], address, mac) - self.express(address=address) - return address - - def express_cloudpipe(self): - private_ip = self.network[2] - linux_net.confirm_rule("FORWARD -d %s -p udp --dport 1194 -j ACCEPT" - % (private_ip, )) - linux_net.confirm_rule("PREROUTING -t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (self.project.vpn_ip, self.project.vpn_port, private_ip)) - - def deexpress(self, address=None): - # if this is the last address, stop dns - super(DHCPNetwork, self).deexpress(address=address) - if len(self.assigned) == 0: - linux_net.stop_dnsmasq(self) - else: - linux_net.start_dnsmasq(self) - -class PublicAddress(datastore.BasicModel): - override_type = "address" - - def __init__(self, address): - self.address = address - super(PublicAddress, self).__init__() - - @property - def identifier(self): - return self.address - - def default_state(self): - return {'address': self.address} - - @classmethod - def create(cls, user_id, project_id, address): - addr = cls(address) - addr['user_id'] = user_id - addr['project_id'] = project_id - addr['instance_id'] = 'available' - addr['private_ip'] = 'available' - addr.save() - return addr - -DEFAULT_PORTS = [("tcp",80), ("tcp",22), ("udp",1194), ("tcp",443)] -class PublicNetworkController(BaseNetwork): - override_type = 'network' - - def __init__(self, *args, **kwargs): - network_id = "public:default" - super(PublicNetworkController, self).__init__(network_id, FLAGS.public_range) - self['user_id'] = "public" - self['project_id'] = "public" - self["create_time"] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) - self["vlan"] = FLAGS.public_vlan - self.save() - self.express() - - @property - def available(self): - for idx in range(2, len(self.network)-1): - address = str(self.network[idx]) - if not address in self.hosts.keys(): - yield address - - @property - def host_objs(self): - for address in self.assigned: - yield PublicAddress(address) - - def get_public_ip_for_instance(self, instance_id): - # FIXME: this should be a lookup - iteration won't scale - for address_record in self.host_objs: - if address_record.get('instance_id', 'available') == instance_id: - return address_record['address'] - - def get_host(self, host): - if host in self.assigned: - return PublicAddress(host) - return None - - def _add_host(self, user_id, project_id, host, _target): - datastore.Redis.instance().hset(self._hosts_key, host, project_id) - PublicAddress.create(user_id, project_id, host) - - def _rem_host(self, host): - PublicAddress(host).destroy() - datastore.Redis.instance().hdel(self._hosts_key, host) - - def associate_address(self, public_ip, private_ip, instance_id): - if not public_ip in self.assigned: - raise compute_exception.AddressNotAllocated() - # TODO(joshua): Keep an index going both ways - for addr in self.host_objs: - if addr.get('private_ip', None) == private_ip: - raise compute_exception.AddressAlreadyAssociated() - addr = self.get_host(public_ip) - if addr.get('private_ip', 'available') != 'available': - raise compute_exception.AddressAlreadyAssociated() - addr['private_ip'] = private_ip - addr['instance_id'] = instance_id - addr.save() - self.express(address=public_ip) - - def disassociate_address(self, public_ip): - if not public_ip in self.assigned: - raise compute_exception.AddressNotAllocated() - addr = self.get_host(public_ip) - if addr.get('private_ip', 'available') == 'available': - raise compute_exception.AddressNotAssociated() - self.deexpress(address=public_ip) - addr['private_ip'] = 'available' - addr['instance_id'] = 'available' - addr.save() - - def express(self, address=None): - addresses = self.host_objs - if address: - addresses = [self.get_host(address)] - for addr in addresses: - if addr.get('private_ip','available') == 'available': - continue - public_ip = addr['address'] - private_ip = addr['private_ip'] - linux_net.bind_public_ip(public_ip, FLAGS.public_interface) - linux_net.confirm_rule("PREROUTING -t nat -d %s -j DNAT --to %s" - % (public_ip, private_ip)) - linux_net.confirm_rule("POSTROUTING -t nat -s %s -j SNAT --to %s" - % (private_ip, public_ip)) - # TODO: Get these from the secgroup datastore entries - linux_net.confirm_rule("FORWARD -d %s -p icmp -j ACCEPT" - % (private_ip)) - for (protocol, port) in DEFAULT_PORTS: - linux_net.confirm_rule("FORWARD -d %s -p %s --dport %s -j ACCEPT" - % (private_ip, protocol, port)) - - def deexpress(self, address=None): - addr = self.get_host(address) - private_ip = addr['private_ip'] - linux_net.unbind_public_ip(address, FLAGS.public_interface) - linux_net.remove_rule("PREROUTING -t nat -d %s -j DNAT --to %s" - % (address, private_ip)) - linux_net.remove_rule("POSTROUTING -t nat -s %s -j SNAT --to %s" - % (private_ip, address)) - linux_net.remove_rule("FORWARD -d %s -p icmp -j ACCEPT" - % (private_ip)) - for (protocol, port) in DEFAULT_PORTS: - linux_net.remove_rule("FORWARD -d %s -p %s --dport %s -j ACCEPT" - % (private_ip, protocol, port)) - - -# FIXME(todd): does this present a race condition, or is there some piece of -# architecture that mitigates it (only one queue listener per net)? -def get_vlan_for_project(project_id): - """ - Allocate vlan IDs to individual users. - """ - vlan = Vlan.lookup(project_id) - if vlan: - return vlan - known_vlans = Vlan.dict_by_vlan() - for vnum in range(FLAGS.vlan_start, FLAGS.vlan_end): - vstr = str(vnum) - if not known_vlans.has_key(vstr): - return Vlan.create(project_id, vnum) - old_project_id = known_vlans[vstr] - if not manager.AuthManager().get_project(old_project_id): - vlan = Vlan.lookup(old_project_id) - if vlan: - # NOTE(todd): This doesn't check for vlan id match, because - # it seems to be assumed that vlan<=>project is - # always a 1:1 mapping. It could be made way - # sexier if it didn't fight against the way - # BasicModel worked and used associate_with - # to build connections to projects. - # NOTE(josh): This is here because we want to make sure we - # don't orphan any VLANs. It is basically - # garbage collection for after projects abandoned - # their reference. - vlan.destroy() - vlan.project_id = project_id - vlan.save() - return vlan - else: - return Vlan.create(project_id, vnum) - raise compute_exception.AddressNotAllocated("Out of VLANs") - -def get_network_by_interface(iface, security_group='default'): - vlan = iface.rpartition("br")[2] - return get_project_network(Vlan.dict_by_vlan().get(vlan), security_group) - -def get_network_by_address(address): - logging.debug("Get Network By Address: %s" % address) - for project in manager.AuthManager().get_projects(): - net = get_project_network(project.id) - if address in net.assigned: - logging.debug("Found %s in %s" % (address, project.id)) - return net - raise compute_exception.AddressNotAllocated() - -def allocate_simple_ip(): - redis = datastore.Redis.instance() - if not redis.exists('ips') and not len(redis.keys('instances:*')): - for address in FLAGS.simple_network_ips: - redis.sadd('ips', address) - address = redis.spop('ips') - if not address: - raise exception.NoMoreAddresses() - return address - -def deallocate_simple_ip(address): - datastore.Redis.instance().sadd('ips', address) - - -def allocate_vpn_ip(user_id, project_id, mac): - return get_project_network(project_id).allocate_vpn_ip(mac) - -def allocate_ip(user_id, project_id, mac): - return get_project_network(project_id).allocate_ip(user_id, project_id, mac) - -def deallocate_ip(address): - return get_network_by_address(address).deallocate_ip(address) - -def release_ip(address): - return get_network_by_address(address).release_ip(address) - -def lease_ip(address): - return get_network_by_address(address).lease_ip(address) - -def get_project_network(project_id, security_group='default'): - """ get a project's private network, allocating one if needed """ - # TODO(todd): It looks goofy to get a project from a UserManager. - # Refactor to still use the LDAP backend, but not User specific. - project = manager.AuthManager().get_project(project_id) - if not project: - raise exception.Error("Project %s doesn't exist, uhoh." % - project_id) - return DHCPNetwork.get_network_for_project(project.project_manager_id, - project.id, security_group) - - -def restart_nets(): - """ Ensure the network for each user is enabled""" - for project in manager.AuthManager().get_projects(): - get_project_network(project.id).express() diff --git a/nova/compute/service.py b/nova/compute/service.py index 9b162edc7..a22240b05 100644 --- a/nova/compute/service.py +++ b/nova/compute/service.py @@ -39,9 +39,9 @@ from nova import service from nova import utils from nova.compute import disk from nova.compute import model -from nova.compute import network from nova.compute import power_state from nova.compute.instance_types import INSTANCE_TYPES +from nova.network import model as net_model from nova.objectstore import image # for image_path flag from nova.virt import connection as virt_connection from nova.volume import service as volume_service @@ -117,10 +117,10 @@ class ComputeService(service.Service): """ launch a new instance with specified options """ logging.debug("Starting instance %s..." % (instance_id)) inst = self.instdir.get(instance_id) - if not FLAGS.simple_network: + if inst.get('network_type', 'dhcp') == 'dhcp': # TODO: Get the real security group of launch in here security_group = "default" - net = network.BridgedNetwork.get_network_for_project(inst['user_id'], + net = net_model.BridgedNetwork.get_network_for_project(inst['user_id'], inst['project_id'], security_group).express() inst['node_name'] = FLAGS.node_name diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 67fc04502..f605eec2c 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -36,11 +36,9 @@ from nova import utils from nova.auth import rbac from nova.auth import manager from nova.compute import model -from nova.compute import network from nova.compute.instance_types import INSTANCE_TYPES -from nova.compute import service as compute_service from nova.endpoint import images -from nova.volume import service as volume_service +from nova.volume import service FLAGS = flags.FLAGS @@ -64,7 +62,6 @@ class CloudController(object): """ def __init__(self): self.instdir = model.InstanceDirectory() - self.network = network.PublicNetworkController() self.setup() @property @@ -76,7 +73,7 @@ class CloudController(object): def volumes(self): """ returns a list of all volumes """ for volume_id in datastore.Redis.instance().smembers("volumes"): - volume = volume_service.get_volume(volume_id) + volume = service.get_volume(volume_id) yield volume def __str__(self): @@ -222,7 +219,7 @@ class CloudController(object): callback=_complete) return d - except users.UserError, e: + except manager.UserError as e: raise @rbac.allow('all') @@ -331,7 +328,7 @@ class CloudController(object): raise exception.NotFound('Instance %s could not be found' % instance_id) def _get_volume(self, context, volume_id): - volume = volume_service.get_volume(volume_id) + volume = service.get_volume(volume_id) if context.user.is_admin() or volume['project_id'] == context.project.id: return volume raise exception.NotFound('Volume %s could not be found' % volume_id) @@ -472,29 +469,34 @@ class CloudController(object): @rbac.allow('netadmin') def allocate_address(self, context, **kwargs): - address = self.network.allocate_ip( - context.user.id, context.project.id, 'public') - return defer.succeed({'addressSet': [{'publicIp' : address}]}) + alloc_result = rpc.call(self._get_network_host(context), + {"method": "allocate_elastic_ip"}) + public_ip = alloc_result['result'] + return defer.succeed({'addressSet': [{'publicIp' : public_ip}]}) @rbac.allow('netadmin') def release_address(self, context, public_ip, **kwargs): - self.network.deallocate_ip(public_ip) + # NOTE(vish): Should we make sure this works? + rpc.cast(self._get_network_host(context), + {"method": "deallocate_elastic_ip", + "args": {"elastic_ip": public_ip}}) return defer.succeed({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') - def associate_address(self, context, instance_id, **kwargs): + def associate_address(self, context, instance_id, public_ip, **kwargs): instance = self._get_instance(context, instance_id) - self.network.associate_address( - kwargs['public_ip'], - instance['private_dns_name'], - instance_id) + address = self._get_address(context, public_ip) + rpc.cast(self._get_network_host(context), + {"method": "associate_elastic_ip", + "args": {"elastic_ip": address['public_ip'], + "fixed_ip": instance['private_dns_name'], + "instance_id": instance['instance_id']}}) return defer.succeed({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') def disassociate_address(self, context, public_ip, **kwargs): address = self._get_address(context, public_ip) self.network.disassociate_address(public_ip) - # TODO - Strip the IP from the instance return defer.succeed({'disassociateResponse': ["Address disassociated."]}) def release_ip(self, context, private_ip, **kwargs): @@ -505,7 +507,13 @@ class CloudController(object): self.network.lease_ip(private_ip) return defer.succeed({'leaseResponse': ["Address leased."]}) + def get_network_host(self, context): + # FIXME(vish): this is temporary until we store net hosts for project + import socket + return socket.gethostname() + @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks def run_instances(self, context, **kwargs): # make sure user can access the image # vpn image is private so it doesn't show up on lists @@ -539,14 +547,25 @@ class CloudController(object): key_data = key_pair.public_key # TODO: Get the real security group of launch in here security_group = "default" - if FLAGS.simple_network: - bridge_name = FLAGS.simple_network_bridge - else: - net = network.BridgedNetwork.get_network_for_project( - context.user.id, context.project.id, security_group) - bridge_name = net['bridge_name'] + create_result = yield rpc.call(FLAGS.network_topic, + {"method": "create_network", + "args": {"user_id": context.user.id, + "project_id": context.project.id, + "security_group": security_group}}) + bridge_name = create_result['result'] + net_host = self._get_network_host(context) for num in range(int(kwargs['max_count'])): + vpn = False + if image_id == FLAGS.vpn_image_id: + vpn = True + allocate_result = yield rpc.call(net_host, + {"method": "allocate_fixed_ip", + "args": {"user_id": context.user.id, + "project_id": context.project.id, + "vpn": vpn}}) inst = self.instdir.new() + inst['mac_address'] = allocate_result['result']['mac_address'] + inst['private_dns_name'] = allocate_result['result']['ip_address'] inst['image_id'] = image_id inst['kernel_id'] = kernel_id inst['ramdisk_id'] = ramdisk_id @@ -558,24 +577,9 @@ class CloudController(object): inst['key_name'] = kwargs.get('key_name', '') inst['user_id'] = context.user.id inst['project_id'] = context.project.id - inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = num inst['bridge_name'] = bridge_name - if FLAGS.simple_network: - address = network.allocate_simple_ip() - else: - if inst['image_id'] == FLAGS.vpn_image_id: - address = network.allocate_vpn_ip( - inst['user_id'], - inst['project_id'], - mac=inst['mac_address']) - else: - address = network.allocate_ip( - inst['user_id'], - inst['project_id'], - mac=inst['mac_address']) - inst['private_dns_name'] = str(address) - # TODO: allocate expresses on the router node + inst.save() rpc.cast(FLAGS.compute_topic, {"method": "run_instance", @@ -583,8 +587,7 @@ class CloudController(object): logging.debug("Casting to node for %s's instance with IP of %s" % (context.user.name, inst['private_dns_name'])) # TODO: Make Network figure out the network name from ip. - return defer.succeed(self._format_instances( - context, reservation_id)) + defer.returnValue(self._format_instances(context, reservation_id)) @rbac.allow('projectmanager', 'sysadmin') def terminate_instances(self, context, instance_id, **kwargs): @@ -594,26 +597,34 @@ class CloudController(object): try: instance = self._get_instance(context, i) except exception.NotFound: - logging.warning("Instance %s was not found during terminate" % i) + logging.warning("Instance %s was not found during terminate" + % i) continue - try: - self.network.disassociate_address( - instance.get('public_dns_name', 'bork')) - except: - pass - if instance.get('private_dns_name', None): - logging.debug("Deallocating address %s" % instance.get('private_dns_name', None)) - if FLAGS.simple_network: - network.deallocate_simple_ip(instance.get('private_dns_name', None)) - else: - try: - self.network.deallocate_ip(instance.get('private_dns_name', None)) - except Exception, _err: - pass - if instance.get('node_name', 'unassigned') != 'unassigned': #It's also internal default + elastic_ip = instance.get('public_dns_name', None) + if elastic_ip: + logging.debug("Deallocating address %s" % elastic_ip) + # NOTE(vish): Right now we don't really care if the ip is + # disassociated. We may need to worry about + # checking this later. Perhaps in the scheduler? + rpc.cast(self._get_network_host(context), + {"method": "disassociate_elastic_ip", + "args": {"elastic_ip": elastic_ip}}) + + fixed_ip = instance.get('private_dns_name', None) + if fixed_ip: + logging.debug("Deallocating address %s" % fixed_ip) + # NOTE(vish): Right now we don't really care if the ip is + # actually removed. We may need to worry about + # checking this later. Perhaps in the scheduler? + rpc.cast(self._get_network_host(context), + {"method": "deallocate_fixed_ip", + "args": {"elastic_ip": elastic_ip}}) + + if instance.get('node_name', 'unassigned') != 'unassigned': + # NOTE(joshua?): It's also internal default rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "terminate_instance", - "args" : {"instance_id": i}}) + {"method": "terminate_instance", + "args": {"instance_id": i}}) else: instance.destroy() return defer.succeed(True) diff --git a/nova/network/exception.py b/nova/network/exception.py new file mode 100644 index 000000000..5722e9672 --- /dev/null +++ b/nova/network/exception.py @@ -0,0 +1,40 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Exceptions for network errors. +""" + +from nova.exception import Error + + +class NoMoreAddresses(Error): + pass + +class AddressNotAllocated(Error): + pass + +class AddressAlreadyAssociated(Error): + pass + +class AddressNotAssociated(Error): + pass + +class NotValidNetworkSize(Error): + pass + diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py new file mode 100644 index 000000000..4a4b4c8a8 --- /dev/null +++ b/nova/network/linux_net.py @@ -0,0 +1,181 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import signal +import os +import subprocess + +# todo(ja): does the definition of network_path belong here? + +from nova import utils + +from nova import flags +FLAGS=flags.FLAGS + +flags.DEFINE_string('dhcpbridge_flagfile', + '/etc/nova/nova-dhcpbridge.conf', + 'location of flagfile for dhcpbridge') + +def execute(cmd, addl_env=None): + if FLAGS.fake_network: + logging.debug("FAKE NET: %s" % cmd) + return "fake", 0 + else: + return utils.execute(cmd, addl_env=addl_env) + +def runthis(desc, cmd): + if FLAGS.fake_network: + return execute(cmd) + else: + return utils.runthis(desc,cmd) + +def Popen(cmd): + if FLAGS.fake_network: + execute(' '.join(cmd)) + else: + subprocess.Popen(cmd) + + +def device_exists(device): + (out, err) = execute("ifconfig %s" % device) + return not err + +def confirm_rule(cmd): + execute("sudo iptables --delete %s" % (cmd)) + execute("sudo iptables -I %s" % (cmd)) + +def remove_rule(cmd): + execute("sudo iptables --delete %s" % (cmd)) + +def bind_public_ip(ip, interface): + runthis("Binding IP to interface: %s", "sudo ip addr add %s dev %s" % (ip, interface)) + +def unbind_public_ip(ip, interface): + runthis("Binding IP to interface: %s", "sudo ip addr del %s dev %s" % (ip, interface)) + +def vlan_create(net): + """ create a vlan on on a bridge device unless vlan already exists """ + if not device_exists("vlan%s" % net['vlan']): + logging.debug("Starting VLAN inteface for %s network", (net['vlan'])) + execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") + execute("sudo vconfig add %s %s" % (FLAGS.bridge_dev, net['vlan'])) + execute("sudo ifconfig vlan%s up" % (net['vlan'])) + +def bridge_create(net): + """ create a bridge on a vlan unless it already exists """ + if not device_exists(net['bridge_name']): + logging.debug("Starting Bridge inteface for %s network", (net['vlan'])) + execute("sudo brctl addbr %s" % (net['bridge_name'])) + execute("sudo brctl setfd %s 0" % (net.bridge_name)) + # execute("sudo brctl setageing %s 10" % (net.bridge_name)) + execute("sudo brctl stp %s off" % (net['bridge_name'])) + execute("sudo brctl addif %s vlan%s" % (net['bridge_name'], net['vlan'])) + if net.bridge_gets_ip: + execute("sudo ifconfig %s %s broadcast %s netmask %s up" % \ + (net['bridge_name'], net.gateway, net.broadcast, net.netmask)) + confirm_rule("FORWARD --in-interface %s -j ACCEPT" % (net['bridge_name'])) + else: + execute("sudo ifconfig %s up" % net['bridge_name']) + +def dnsmasq_cmd(net): + cmd = ['sudo -E dnsmasq', + ' --strict-order', + ' --bind-interfaces', + ' --conf-file=', + ' --pid-file=%s' % dhcp_file(net['vlan'], 'pid'), + ' --listen-address=%s' % net.dhcp_listen_address, + ' --except-interface=lo', + ' --dhcp-range=%s,static,600s' % (net.dhcp_range_start), + ' --dhcp-hostsfile=%s' % dhcp_file(net['vlan'], 'conf'), + ' --dhcp-script=%s' % bin_file('nova-dhcpbridge'), + ' --leasefile-ro'] + return ''.join(cmd) + +def hostDHCP(network, host, mac): + idx = host.split(".")[-1] # Logically, the idx of instances they've launched in this net + return "%s,%s-%s-%s.novalocal,%s" % \ + (mac, network['user_id'], network['vlan'], idx, host) + +# todo(ja): if the system has restarted or pid numbers have wrapped +# then you cannot be certain that the pid refers to the +# dnsmasq. As well, sending a HUP only reloads the hostfile, +# so any configuration options (like dchp-range, vlan, ...) +# aren't reloaded +def start_dnsmasq(network): + """ (re)starts a dnsmasq server for a given network + + if a dnsmasq instance is already running then send a HUP + signal causing it to reload, otherwise spawn a new instance + """ + with open(dhcp_file(network['vlan'], 'conf'), 'w') as f: + for host_name in network.hosts: + f.write("%s\n" % hostDHCP(network, host_name, network.hosts[host_name])) + + pid = dnsmasq_pid_for(network) + + # if dnsmasq is already running, then tell it to reload + if pid: + # todo(ja): use "/proc/%d/cmdline" % (pid) to determine if pid refers + # correct dnsmasq process + try: + os.kill(pid, signal.SIGHUP) + except Exception, e: + logging.debug("Hupping dnsmasq threw %s", e) + + # otherwise delete the existing leases file and start dnsmasq + lease_file = dhcp_file(network['vlan'], 'leases') + if os.path.exists(lease_file): + os.unlink(lease_file) + + # FLAGFILE and DNSMASQ_INTERFACE in env + env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, + 'DNSMASQ_INTERFACE': network['bridge_name']} + execute(dnsmasq_cmd(network), addl_env=env) + +def stop_dnsmasq(network): + """ stops the dnsmasq instance for a given network """ + pid = dnsmasq_pid_for(network) + + if pid: + try: + os.kill(pid, signal.SIGTERM) + except Exception, e: + logging.debug("Killing dnsmasq threw %s", e) + +def dhcp_file(vlan, kind): + """ return path to a pid, leases or conf file for a vlan """ + + return os.path.abspath("%s/nova-%s.%s" % (FLAGS.networks_path, vlan, kind)) + +def bin_file(script): + return os.path.abspath(os.path.join(__file__, "../../../bin", script)) + +def dnsmasq_pid_for(network): + """ the pid for prior dnsmasq instance for a vlan, + returns None if no pid file exists + + if machine has rebooted pid might be incorrect (caller should check) + """ + + pid_file = dhcp_file(network['vlan'], 'pid') + + if os.path.exists(pid_file): + with open(pid_file, 'r') as f: + return int(f.read()) + diff --git a/nova/network/model.py b/nova/network/model.py new file mode 100644 index 000000000..5346549b8 --- /dev/null +++ b/nova/network/model.py @@ -0,0 +1,549 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Model Classes for network control, including VLANs, DHCP, and IP allocation. +""" + +import IPy +import logging +import os +import time + +from nova import datastore +from nova import exception as nova_exception +from nova import flags +from nova import utils +from nova.auth import manager +from nova.network import exception +from nova.network import linux_net + + +FLAGS = flags.FLAGS +flags.DEFINE_string('networks_path', utils.abspath('../networks'), + 'Location to keep network config files') +flags.DEFINE_integer('public_vlan', 1, 'VLAN for public IP addresses') +flags.DEFINE_string('public_interface', 'vlan1', + 'Interface for public IP addresses') +flags.DEFINE_string('bridge_dev', 'eth1', + 'network device for bridges') +flags.DEFINE_integer('vlan_start', 100, 'First VLAN for private networks') +flags.DEFINE_integer('vlan_end', 4093, 'Last VLAN for private networks') +flags.DEFINE_integer('network_size', 256, + 'Number of addresses in each private subnet') +flags.DEFINE_string('public_range', '4.4.4.0/24', 'Public IP address block') +flags.DEFINE_string('private_range', '10.0.0.0/8', 'Private IP address block') +flags.DEFINE_integer('cnt_vpn_clients', 5, + 'Number of addresses reserved for vpn clients') +flags.DEFINE_integer('cloudpipe_start_port', 12000, + 'Starting port for mapped CloudPipe external ports') + +logging.getLogger().setLevel(logging.DEBUG) + + +class Vlan(datastore.BasicModel): + def __init__(self, project, vlan): + """ + Since we don't want to try and find a vlan by its identifier, + but by a project id, we don't call super-init. + """ + self.project_id = project + self.vlan_id = vlan + + @property + def identifier(self): + return "%s:%s" % (self.project_id, self.vlan_id) + + @classmethod + def create(cls, project, vlan): + instance = cls(project, vlan) + instance.save() + return instance + + @classmethod + @datastore.absorb_connection_error + def lookup(cls, project): + set_name = cls._redis_set_name(cls.__name__) + vlan = datastore.Redis.instance().hget(set_name, project) + if vlan: + return cls(project, vlan) + else: + return None + + @classmethod + @datastore.absorb_connection_error + def dict_by_project(cls): + """a hash of project:vlan""" + set_name = cls._redis_set_name(cls.__name__) + return datastore.Redis.instance().hgetall(set_name) + + @classmethod + @datastore.absorb_connection_error + def dict_by_vlan(cls): + """a hash of vlan:project""" + set_name = cls._redis_set_name(cls.__name__) + rv = {} + h = datastore.Redis.instance().hgetall(set_name) + for v in h.keys(): + rv[h[v]] = v + return rv + + @classmethod + @datastore.absorb_connection_error + def all(cls): + set_name = cls._redis_set_name(cls.__name__) + elements = datastore.Redis.instance().hgetall(set_name) + for project in elements: + yield cls(project, elements[project]) + + @datastore.absorb_connection_error + def save(self): + """ + Vlan saves state into a giant hash named "vlans", with keys of + project_id and value of vlan number. Therefore, we skip the + default way of saving into "vlan:ID" and adding to a set of "vlans". + """ + set_name = self._redis_set_name(self.__class__.__name__) + datastore.Redis.instance().hset(set_name, self.project_id, self.vlan_id) + + @datastore.absorb_connection_error + def destroy(self): + set_name = self._redis_set_name(self.__class__.__name__) + datastore.Redis.instance().hdel(set_name, self.project_id) + + def subnet(self): + vlan = int(self.vlan_id) + network = IPy.IP(FLAGS.private_range) + start = (vlan-FLAGS.vlan_start) * FLAGS.network_size + # minus one for the gateway. + return "%s-%s" % (network[start], + network[start + FLAGS.network_size - 1]) + +# CLEANUP: +# TODO(ja): Save the IPs at the top of each subnet for cloudpipe vpn clients +# TODO(ja): does vlanpool "keeper" need to know the min/max - shouldn't FLAGS always win? +# TODO(joshua): Save the IPs at the top of each subnet for cloudpipe vpn clients + +class BaseNetwork(datastore.BasicModel): + override_type = 'network' + NUM_STATIC_IPS = 3 # Network, Gateway, and CloudPipe + + @property + def identifier(self): + return self.network_id + + def default_state(self): + return {'network_id': self.network_id, 'network_str': self.network_str} + + @classmethod + def create(cls, user_id, project_id, security_group, vlan, network_str): + network_id = "%s:%s" % (project_id, security_group) + net = cls(network_id, network_str) + net['user_id'] = user_id + net['project_id'] = project_id + net["vlan"] = vlan + net["bridge_name"] = "br%s" % vlan + net.save() + return net + + def __init__(self, network_id, network_str=None): + self.network_id = network_id + self.network_str = network_str + super(BaseNetwork, self).__init__() + self.save() + + @property + def network(self): + return IPy.IP(self['network_str']) + + @property + def netmask(self): + return self.network.netmask() + + @property + def gateway(self): + return self.network[1] + + @property + def broadcast(self): + return self.network.broadcast() + + @property + def bridge_name(self): + return "br%s" % (self["vlan"]) + + @property + def user(self): + return manager.AuthManager().get_user(self['user_id']) + + @property + def project(self): + return manager.AuthManager().get_project(self['project_id']) + + @property + def _hosts_key(self): + return "network:%s:hosts" % (self['network_str']) + + @property + def hosts(self): + return datastore.Redis.instance().hgetall(self._hosts_key) or {} + + def _add_host(self, _user_id, _project_id, host, target): + datastore.Redis.instance().hset(self._hosts_key, host, target) + + def _rem_host(self, host): + datastore.Redis.instance().hdel(self._hosts_key, host) + + @property + def assigned(self): + return datastore.Redis.instance().hkeys(self._hosts_key) + + @property + def available(self): + # the .2 address is always CloudPipe + # and the top are for vpn clients + for idx in range(self.num_static_ips, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): + address = str(self.network[idx]) + if not address in self.hosts.keys(): + yield address + + @property + def num_static_ips(self): + return BaseNetwork.NUM_STATIC_IPS + + def allocate_ip(self, user_id, project_id, mac): + for address in self.available: + logging.debug("Allocating IP %s to %s" % (address, project_id)) + self._add_host(user_id, project_id, address, mac) + self.express(address=address) + return address + raise exception.NoMoreAddresses("Project %s with network %s" % + (project_id, str(self.network))) + + def lease_ip(self, ip_str): + logging.debug("Leasing allocated IP %s" % (ip_str)) + + def release_ip(self, ip_str): + if not ip_str in self.assigned: + raise exception.AddressNotAllocated() + self.deexpress(address=ip_str) + self._rem_host(ip_str) + + def deallocate_ip(self, ip_str): + # Do nothing for now, cleanup on ip release + pass + + def list_addresses(self): + for address in self.hosts: + yield address + + def express(self, address=None): pass + def deexpress(self, address=None): pass + + +class BridgedNetwork(BaseNetwork): + """ + Virtual Network that can express itself to create a vlan and + a bridge (with or without an IP address/netmask/gateway) + + properties: + bridge_name - string (example value: br42) + vlan - integer (example value: 42) + bridge_dev - string (example: eth0) + bridge_gets_ip - boolean used during bridge creation + + if bridge_gets_ip then network address for bridge uses the properties: + gateway + broadcast + netmask + """ + + bridge_gets_ip = False + override_type = 'network' + + @classmethod + def get_network_for_project(cls, user_id, project_id, security_group): + vlan = get_vlan_for_project(project_id) + network_str = vlan.subnet() + return cls.create(user_id, project_id, security_group, vlan.vlan_id, + network_str) + + def __init__(self, *args, **kwargs): + super(BridgedNetwork, self).__init__(*args, **kwargs) + self['bridge_dev'] = FLAGS.bridge_dev + self.save() + + def express(self, address=None): + super(BridgedNetwork, self).express(address=address) + linux_net.vlan_create(self) + linux_net.bridge_create(self) + +class DHCPNetwork(BridgedNetwork): + """ + properties: + dhcp_listen_address: the ip of the gateway / dhcp host + dhcp_range_start: the first ip to give out + dhcp_range_end: the last ip to give out + """ + bridge_gets_ip = True + override_type = 'network' + + def __init__(self, *args, **kwargs): + super(DHCPNetwork, self).__init__(*args, **kwargs) + # logging.debug("Initing DHCPNetwork object...") + self.dhcp_listen_address = self.network[1] + self.dhcp_range_start = self.network[3] + self.dhcp_range_end = self.network[-(1 + FLAGS.cnt_vpn_clients)] + try: + os.makedirs(FLAGS.networks_path) + # NOTE(todd): I guess this is a lazy way to not have to check if the + # directory exists, but shouldn't we be smarter about + # telling the difference between existing directory and + # permission denied? (Errno 17 vs 13, OSError) + except Exception, err: + pass + + def express(self, address=None): + super(DHCPNetwork, self).express(address=address) + if len(self.assigned) > 0: + logging.debug("Starting dnsmasq server for network with vlan %s", + self['vlan']) + linux_net.start_dnsmasq(self) + else: + logging.debug("Not launching dnsmasq: no hosts.") + self.express_cloudpipe() + + def allocate_vpn_ip(self, user_id, project_id, mac): + address = str(self.network[2]) + self._add_host(user_id, project_id, address, mac) + self.express(address=address) + return address + + def express_cloudpipe(self): + private_ip = str(self.network[2]) + linux_net.confirm_rule("FORWARD -d %s -p udp --dport 1194 -j ACCEPT" + % (private_ip, )) + linux_net.confirm_rule("PREROUTING -t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" + % (self.project.vpn_ip, self.project.vpn_port, private_ip)) + + def deexpress(self, address=None): + # if this is the last address, stop dns + super(DHCPNetwork, self).deexpress(address=address) + if len(self.assigned) == 0: + linux_net.stop_dnsmasq(self) + else: + linux_net.start_dnsmasq(self) + +class PublicAddress(datastore.BasicModel): + override_type = "address" + + def __init__(self, address): + self.address = address + super(PublicAddress, self).__init__() + + @property + def identifier(self): + return self.address + + def default_state(self): + return {'address': self.address} + + @classmethod + def create(cls, user_id, project_id, address): + addr = cls(address) + addr['user_id'] = user_id + addr['project_id'] = project_id + addr['instance_id'] = 'available' + addr['private_ip'] = 'available' + addr.save() + return addr + +DEFAULT_PORTS = [("tcp",80), ("tcp",22), ("udp",1194), ("tcp",443)] +class PublicNetworkController(BaseNetwork): + override_type = 'network' + + def __init__(self, *args, **kwargs): + network_id = "public:default" + super(PublicNetworkController, self).__init__(network_id, FLAGS.public_range) + self['user_id'] = "public" + self['project_id'] = "public" + self["create_time"] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + self["vlan"] = FLAGS.public_vlan + self.save() + self.express() + + @property + def available(self): + for idx in range(2, len(self.network)-1): + address = str(self.network[idx]) + if not address in self.hosts.keys(): + yield address + + @property + def host_objs(self): + for address in self.assigned: + yield PublicAddress(address) + + def get_public_ip_for_instance(self, instance_id): + # FIXME: this should be a lookup - iteration won't scale + for address_record in self.host_objs: + if address_record.get('instance_id', 'available') == instance_id: + return address_record['address'] + + def get_host(self, host): + if host in self.assigned: + return PublicAddress(host) + return None + + def _add_host(self, user_id, project_id, host, _target): + datastore.Redis.instance().hset(self._hosts_key, host, project_id) + PublicAddress.create(user_id, project_id, host) + + def _rem_host(self, host): + PublicAddress(host).destroy() + datastore.Redis.instance().hdel(self._hosts_key, host) + + def associate_address(self, public_ip, private_ip, instance_id): + if not public_ip in self.assigned: + raise exception.AddressNotAllocated() + # TODO(joshua): Keep an index going both ways + for addr in self.host_objs: + if addr.get('private_ip', None) == private_ip: + raise exception.AddressAlreadyAssociated() + addr = self.get_host(public_ip) + if addr.get('private_ip', 'available') != 'available': + raise exception.AddressAlreadyAssociated() + addr['private_ip'] = private_ip + addr['instance_id'] = instance_id + addr.save() + self.express(address=public_ip) + + def disassociate_address(self, public_ip): + if not public_ip in self.assigned: + raise exception.AddressNotAllocated() + addr = self.get_host(public_ip) + if addr.get('private_ip', 'available') == 'available': + raise exception.AddressNotAssociated() + self.deexpress(address=public_ip) + addr['private_ip'] = 'available' + addr['instance_id'] = 'available' + addr.save() + + def express(self, address=None): + addresses = self.host_objs + if address: + addresses = [self.get_host(address)] + for addr in addresses: + if addr.get('private_ip','available') == 'available': + continue + public_ip = addr['address'] + private_ip = addr['private_ip'] + linux_net.bind_public_ip(public_ip, FLAGS.public_interface) + linux_net.confirm_rule("PREROUTING -t nat -d %s -j DNAT --to %s" + % (public_ip, private_ip)) + linux_net.confirm_rule("POSTROUTING -t nat -s %s -j SNAT --to %s" + % (private_ip, public_ip)) + # TODO: Get these from the secgroup datastore entries + linux_net.confirm_rule("FORWARD -d %s -p icmp -j ACCEPT" + % (private_ip)) + for (protocol, port) in DEFAULT_PORTS: + linux_net.confirm_rule("FORWARD -d %s -p %s --dport %s -j ACCEPT" + % (private_ip, protocol, port)) + + def deexpress(self, address=None): + addr = self.get_host(address) + private_ip = addr['private_ip'] + linux_net.unbind_public_ip(address, FLAGS.public_interface) + linux_net.remove_rule("PREROUTING -t nat -d %s -j DNAT --to %s" + % (address, private_ip)) + linux_net.remove_rule("POSTROUTING -t nat -s %s -j SNAT --to %s" + % (private_ip, address)) + linux_net.remove_rule("FORWARD -d %s -p icmp -j ACCEPT" + % (private_ip)) + for (protocol, port) in DEFAULT_PORTS: + linux_net.remove_rule("FORWARD -d %s -p %s --dport %s -j ACCEPT" + % (private_ip, protocol, port)) + + +# FIXME(todd): does this present a race condition, or is there some piece of +# architecture that mitigates it (only one queue listener per net)? +def get_vlan_for_project(project_id): + """ + Allocate vlan IDs to individual users. + """ + vlan = Vlan.lookup(project_id) + if vlan: + return vlan + known_vlans = Vlan.dict_by_vlan() + for vnum in range(FLAGS.vlan_start, FLAGS.vlan_end): + vstr = str(vnum) + if not known_vlans.has_key(vstr): + return Vlan.create(project_id, vnum) + old_project_id = known_vlans[vstr] + if not manager.AuthManager().get_project(old_project_id): + vlan = Vlan.lookup(old_project_id) + if vlan: + # NOTE(todd): This doesn't check for vlan id match, because + # it seems to be assumed that vlan<=>project is + # always a 1:1 mapping. It could be made way + # sexier if it didn't fight against the way + # BasicModel worked and used associate_with + # to build connections to projects. + # NOTE(josh): This is here because we want to make sure we + # don't orphan any VLANs. It is basically + # garbage collection for after projects abandoned + # their reference. + vlan.destroy() + vlan.project_id = project_id + vlan.save() + return vlan + else: + return Vlan.create(project_id, vnum) + raise exception.AddressNotAllocated("Out of VLANs") + +def get_project_network(project_id, security_group='default'): + """ get a project's private network, allocating one if needed """ + project = manager.AuthManager().get_project(project_id) + if not project: + raise nova_exception.NotFound("Project %s doesn't exist." % project_id) + manager_id = project.project_manager_id + return DHCPNetwork.get_network_for_project(manager_id, + project.id, + security_group) + + +def get_network_by_address(address): + # TODO(vish): This is completely the wrong way to do this, but + # I'm getting the network binary working before I + # tackle doing this the right way. + logging.debug("Get Network By Address: %s" % address) + for project in manager.AuthManager().get_projects(): + net = get_project_network(project.id) + if address in net.assigned: + logging.debug("Found %s in %s" % (address, project.id)) + return net + raise exception.AddressNotAllocated() + + +def get_network_by_interface(iface, security_group='default'): + vlan = iface.rpartition("br")[2] + project_id = Vlan.dict_by_vlan().get(vlan) + return get_project_network(project_id, security_group) + + + diff --git a/nova/network/service.py b/nova/network/service.py index 9d87e05e6..97976a752 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -22,14 +22,141 @@ Network Nodes are responsible for allocating ips and setting up network import logging +from nova import datastore +from nova import exception as nova_exception from nova import flags from nova import service - +from nova import utils +from nova.auth import manager +from nova.network import exception +from nova.network import model FLAGS = flags.FLAGS -class NetworkService(service.Service): +flags.DEFINE_string('flat_network_bridge', 'br100', + 'Bridge for simple network instances') +flags.DEFINE_list('flat_network_ips', + ['192.168.0.2','192.168.0.3','192.168.0.4'], + 'Available ips for simple network') +flags.DEFINE_string('flat_network_network', '192.168.0.0', + 'Network for simple network') +flags.DEFINE_string('flat_network_netmask', '255.255.255.0', + 'Netmask for simple network') +flags.DEFINE_string('flat_network_gateway', '192.168.0.1', + 'Broadcast for simple network') +flags.DEFINE_string('flat_network_broadcast', '192.168.0.255', + 'Broadcast for simple network') +flags.DEFINE_string('flat_network_dns', '8.8.4.4', + 'Dns for simple network') + + +class BaseNetworkService(service.Service): + """Implements common network service functionality + + This class must be subclassed. + """ + def __init__(self, *args, **kwargs): + self.network = model.PublicNetworkController() + + def create_network(self, user_id, project_id, security_group='default', + *args, **kwargs): + """Subclass implements creating network and returns network data""" + raise NotImplementedError() + + def allocate_fixed_ip(self, user_id, project_id, *args, **kwargs): + """Subclass implements getting fixed ip from the pool""" + raise NotImplementedError() + + def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): + """Subclass implements return of ip to the pool""" + raise NotImplementedError() + + def allocate_elastic_ip(self): + """Gets a elastic ip from the pool""" + return self.network.allocate_ip() + + def associate_elastic_ip(self, elastic_ip, fixed_ip, instance_id): + """Associates an elastic ip to a fixed ip""" + self.network.associate_address(elastic_ip, fixed_ip, instance_id) + + def disassociate_elastic_ip(self, elastic_ip): + """Disassociates a elastic ip""" + self.network.disassociate_address(elastic_ip) + + def deallocate_elastic_ip(self, elastic_ip): + """Returns a elastic ip to the pool""" + self.network.deallocate_ip(elastic_ip) + + +class FlatNetworkService(BaseNetworkService): + def create_network(self, user_id, project_id, security_group='default', + *args, **kwargs): + """Creates network and returns bridge + + Flat network service simply returns a common bridge regardless of + project. + """ + return {'network_type': 'injected', + 'bridge_name': FLAGS.flat_network_bridge, + 'network_network': FLAGS.flat_network_network, + 'network_netmask': FLAGS.flat_network_netmask, + 'network_gateway': FLAGS.flat_network_gateway, + 'network_broadcast': FLAGS.flat_network_broadcast, + 'network_dns': FLAGS.flat_network_dns} + + def allocate_fixed_ip(self, user_id, project_id, *args, **kwargs): + """Gets a fixed ip from the pool + + Flat network just grabs the next available ip from the pool + """ + redis = datastore.Redis.instance() + if not redis.exists('ips') and not len(redis.keys('instances:*')): + for fixed_ip in FLAGS.flat_network_ips: + redis.sadd('ips', fixed_ip) + fixed_ip = redis.spop('ips') + if not fixed_ip: + raise exception.NoMoreAddresses() + return {'mac': utils.generate_mac(), + 'ip' : str(fixed_ip)} + + def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): + """Returns an ip to the pool""" + datastore.Redis.instance().sadd('ips', fixed_ip) + +class VlanNetworkService(BaseNetworkService): """Allocates ips and sets up networks""" + def create_network(self, user_id, project_id, security_group='default', + *args, **kwargs): + """Creates network and returns bridge""" + net = model.get_project_network(project_id, security_group) + return {'network_type': 'dhcp', + 'bridge_name': net['bridge_name']} + + def allocate_fixed_ip(self, user_id, project_id, vpn=False, *args, **kwargs): + """Gets a fixed ip from the pool """ + mac = utils.generate_mac() + net = model.get_project_network(project_id) + if vpn: + fixed_ip = net.allocate_vpn_ip(user_id, project_id, mac) + else: + fixed_ip = net.allocate_ip(user_id, project_id, mac) + return {'mac': mac, + 'ip' : fixed_ip} + + def deallocate_fixed_ip(self, fixed_ip, + *args, **kwargs): + """Returns an ip to the pool""" + model.get_network_by_address(fixed_ip).deallocate_ip(fixed_ip) + + def lease_ip(self, address): + return self. __get_network_by_address(address).lease_ip(address) + + def release_ip(self, address): + return model.get_network_by_address(address).release_ip(address) + + def restart_nets(self): + """ Ensure the network for each user is enabled""" + for project in manager.AuthManager().get_projects(): + model.get_project_network(project.id).express() + - def __init__(self): - logging.debug("Network node working") diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index f24eefb0d..4b2f6c649 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -24,8 +24,9 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager -from nova.compute import network -from nova.compute.exception import NoMoreAddresses +from nova.network import model +from nova.network import service +from nova.network.exception import NoMoreAddresses FLAGS = flags.FLAGS @@ -52,7 +53,8 @@ class NetworkTestCase(test.TrialTestCase): self.projects.append(self.manager.create_project(name, 'netuser', name)) - self.network = network.PublicNetworkController() + self.network = model.PublicNetworkController() + self.service = service.VlanNetworkService() def tearDown(self): super(NetworkTestCase, self).tearDown() @@ -66,16 +68,17 @@ class NetworkTestCase(test.TrialTestCase): self.assertTrue(IPy.IP(address) in pubnet) self.assertTrue(IPy.IP(address) in self.network.network) - def test_allocate_deallocate_ip(self): - address = network.allocate_ip( - self.user.id, self.projects[0].id, utils.generate_mac()) + def test_allocate_deallocate_fixed_ip(self): + result = self.service.allocate_fixed_ip( + self.user.id, self.projects[0].id) + address = result['ip'] + mac = result['mac'] logging.debug("Was allocated %s" % (address)) - net = network.get_project_network(self.projects[0].id, "default") + net = model.get_project_network(self.projects[0].id, "default") self.assertEqual(True, is_in_project(address, self.projects[0].id)) - mac = utils.generate_mac() hostname = "test-host" self.dnsmasq.issue_ip(mac, address, hostname, net.bridge_name) - rv = network.deallocate_ip(address) + rv = self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released self.assertEqual(True, is_in_project(address, self.projects[0].id)) @@ -84,15 +87,18 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(False, is_in_project(address, self.projects[0].id)) def test_range_allocation(self): - mac = utils.generate_mac() - secondmac = utils.generate_mac() hostname = "test-host" - address = network.allocate_ip( - self.user.id, self.projects[0].id, mac) - secondaddress = network.allocate_ip( - self.user, self.projects[1].id, secondmac) - net = network.get_project_network(self.projects[0].id, "default") - secondnet = network.get_project_network(self.projects[1].id, "default") + result = self.service.allocate_fixed_ip( + self.user.id, self.projects[0].id) + mac = result['mac'] + address = result['ip'] + result = self.service.allocate_fixed_ip( + self.user, self.projects[1].id) + secondmac = result['mac'] + secondaddress = result['ip'] + + net = model.get_project_network(self.projects[0].id, "default") + secondnet = model.get_project_network(self.projects[1].id, "default") self.assertEqual(True, is_in_project(address, self.projects[0].id)) self.assertEqual(True, is_in_project(secondaddress, self.projects[1].id)) @@ -103,46 +109,50 @@ class NetworkTestCase(test.TrialTestCase): self.dnsmasq.issue_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) - rv = network.deallocate_ip(address) + rv = self.service.deallocate_fixed_ip(address) self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) self.assertEqual(False, is_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second self.assertEqual(True, is_in_project(secondaddress, self.projects[1].id)) - rv = network.deallocate_ip(secondaddress) + rv = self.service.deallocate_fixed_ip(secondaddress) self.dnsmasq.release_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) self.assertEqual(False, is_in_project(secondaddress, self.projects[1].id)) def test_subnet_edge(self): - secondaddress = network.allocate_ip(self.user.id, self.projects[0].id, - utils.generate_mac()) + result = self.service.allocate_fixed_ip(self.user.id, + self.projects[0].id) + firstaddress = result['ip'] hostname = "toomany-hosts" for i in range(1,5): project_id = self.projects[i].id - mac = utils.generate_mac() - mac2 = utils.generate_mac() - mac3 = utils.generate_mac() - address = network.allocate_ip( - self.user, project_id, mac) - address2 = network.allocate_ip( - self.user, project_id, mac2) - address3 = network.allocate_ip( - self.user, project_id, mac3) + result = self.service.allocate_fixed_ip( + self.user, project_id) + mac = result['mac'] + address = result['ip'] + result = self.service.allocate_fixed_ip( + self.user, project_id) + mac2 = result['mac'] + address2 = result['ip'] + result = self.service.allocate_fixed_ip( + self.user, project_id) + mac3 = result['mac'] + address3 = result['ip'] self.assertEqual(False, is_in_project(address, self.projects[0].id)) self.assertEqual(False, is_in_project(address2, self.projects[0].id)) self.assertEqual(False, is_in_project(address3, self.projects[0].id)) - rv = network.deallocate_ip(address) - rv = network.deallocate_ip(address2) - rv = network.deallocate_ip(address3) - net = network.get_project_network(project_id, "default") + rv = self.service.deallocate_fixed_ip(address) + rv = self.service.deallocate_fixed_ip(address2) + rv = self.service.deallocate_fixed_ip(address3) + net = model.get_project_network(project_id, "default") self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) self.dnsmasq.release_ip(mac2, address2, hostname, net.bridge_name) self.dnsmasq.release_ip(mac3, address3, hostname, net.bridge_name) - net = network.get_project_network(self.projects[0].id, "default") - rv = network.deallocate_ip(secondaddress) - self.dnsmasq.release_ip(mac, secondaddress, hostname, net.bridge_name) + net = model.get_project_network(self.projects[0].id, "default") + rv = self.service.deallocate_fixed_ip(firstaddress) + self.dnsmasq.release_ip(mac, firstaddress, hostname, net.bridge_name) def test_release_before_deallocate(self): pass @@ -169,7 +179,7 @@ class NetworkTestCase(test.TrialTestCase): NUM_RESERVED_VPN_IPS) usable addresses """ - net = network.get_project_network(self.projects[0].id, "default") + net = model.get_project_network(self.projects[0].id, "default") # Determine expected number of available IP addresses num_static_ips = net.num_static_ips @@ -183,22 +193,23 @@ class NetworkTestCase(test.TrialTestCase): macs = {} addresses = {} for i in range(0, (num_available_ips - 1)): - macs[i] = utils.generate_mac() - addresses[i] = network.allocate_ip(self.user.id, self.projects[0].id, macs[i]) + result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) + macs[i] = result['mac'] + addresses[i] = result['ip'] self.dnsmasq.issue_ip(macs[i], addresses[i], hostname, net.bridge_name) - self.assertRaises(NoMoreAddresses, network.allocate_ip, self.user.id, self.projects[0].id, utils.generate_mac()) + self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, self.user.id, self.projects[0].id) for i in range(0, (num_available_ips - 1)): - rv = network.deallocate_ip(addresses[i]) + rv = self.service.deallocate_fixed_ip(addresses[i]) self.dnsmasq.release_ip(macs[i], addresses[i], hostname, net.bridge_name) def is_in_project(address, project_id): - return address in network.get_project_network(project_id).list_addresses() + return address in model.get_project_network(project_id).list_addresses() def _get_project_addresses(project_id): project_addresses = [] - for addr in network.get_project_network(project_id).list_addresses(): + for addr in model.get_project_network(project_id).list_addresses(): project_addresses.append(addr) return project_addresses -- cgit From 6187529119ab51a6df7e30ef5190757ee0feca5e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 3 Aug 2010 15:04:38 -0700 Subject: vblade commands randomly toss stuff into stderr, ignore it --- nova/volume/service.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/nova/volume/service.py b/nova/volume/service.py index e12f675a7..9dd63e88f 100644 --- a/nova/volume/service.py +++ b/nova/volume/service.py @@ -227,11 +227,7 @@ class Volume(datastore.BasicModel): @defer.inlineCallbacks def destroy(self): - try: - yield self._remove_export() - except Exception as ex: - logging.debug("Ingnoring failure to remove export %s" % ex) - pass + yield self._remove_export() yield self._delete_lv() super(Volume, self).destroy() @@ -250,7 +246,7 @@ class Volume(datastore.BasicModel): def _delete_lv(self): yield process.simple_execute( "sudo lvremove -f %s/%s" % (FLAGS.volume_group, - self['volume_id'])) + self['volume_id']), error_ok=1) @defer.inlineCallbacks def _setup_export(self): @@ -275,10 +271,10 @@ class Volume(datastore.BasicModel): def _remove_export(self): yield process.simple_execute( "sudo vblade-persist stop %s %s" % (self['shelf_id'], - self['blade_id'])) + self['blade_id']), error_ok=1) yield process.simple_execute( "sudo vblade-persist destroy %s %s" % (self['shelf_id'], - self['blade_id'])) + self['blade_id']), error_ok=1) class FakeVolume(Volume): -- cgit From 576dade1d53814416977522637bea9e3c32e5483 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 3 Aug 2010 15:13:07 -0700 Subject: change network_service flag to network_type and don't take full class name --- bin/nova-network | 10 ++++++---- nova/network/service.py | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/nova-network b/bin/nova-network index b2e2cf173..1d620b525 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -25,15 +25,17 @@ from nova import flags from nova import twistd from nova import utils +from nova.network import service FLAGS = flags.FLAGS -flags.DEFINE_string('network_service', - 'nova.network.service.VlanNetworkService', - 'Service Class for Networking') if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = utils.import_class(FLAGS.network_service).create() + t = FLAGS.network_type + if t == 'flat': + application = service.FlatNetworkService.create() + elif t == 'vlan': + application = service.VlanNetworkService.create() diff --git a/nova/network/service.py b/nova/network/service.py index 97976a752..72f7db126 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -33,6 +33,9 @@ from nova.network import model FLAGS = flags.FLAGS +flags.DEFINE_string('network_type', + 'flat', + 'Service Class for Networking') flags.DEFINE_string('flat_network_bridge', 'br100', 'Bridge for simple network instances') flags.DEFINE_list('flat_network_ips', -- cgit From 8261e26f78f061de5f5e98f8066da33f9b4e3a23 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 3 Aug 2010 15:48:49 -0700 Subject: use get to retrieve node_name from initial_state --- nova/compute/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/compute/model.py b/nova/compute/model.py index 7dd130ca3..266a93b9a 100644 --- a/nova/compute/model.py +++ b/nova/compute/model.py @@ -151,7 +151,8 @@ class Instance(datastore.BasicModel): # it just adds the first one is_new = self.is_new_record() node_set = (self.state['node_name'] != 'unassigned' and - self.initial_state['node_name'] == 'unassigned') + self.initial_state.get('node_name', 'unassigned') + == 'unassigned') success = super(Instance, self).save() if success and is_new: self.associate_with("project", self.project) -- cgit From 13ec179c99012ed3d579e19094c0039ccb630796 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 3 Aug 2010 17:27:29 -0700 Subject: created assocaition between project and host, modified commands to get host async, simplified calls to network --- nova/network/service.py | 72 +++++++++++++++++++++++------------------- nova/tests/network_unittest.py | 30 +++++++++--------- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/nova/network/service.py b/nova/network/service.py index 72f7db126..45c2867ca 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -23,7 +23,6 @@ Network Nodes are responsible for allocating ips and setting up network import logging from nova import datastore -from nova import exception as nova_exception from nova import flags from nova import service from nova import utils @@ -52,6 +51,13 @@ flags.DEFINE_string('flat_network_broadcast', '192.168.0.255', flags.DEFINE_string('flat_network_dns', '8.8.4.4', 'Dns for simple network') +def get_host_for_project(project_id): + redis = datastore.Redis.instance() + host = redis.get(__host_key(project_id)) + +def __host_key(project_id): + return "network_host:%s" % project_id + class BaseNetworkService(service.Service): """Implements common network service functionality @@ -61,12 +67,18 @@ class BaseNetworkService(service.Service): def __init__(self, *args, **kwargs): self.network = model.PublicNetworkController() - def create_network(self, user_id, project_id, security_group='default', - *args, **kwargs): - """Subclass implements creating network and returns network data""" - raise NotImplementedError() + def get_network_host(self, user_id, project_id, *args, **kwargs): + """Safely becomes the host of the projects network""" + redis = datastore.Redis.instance() + key = __host_key(project_id) + if redis.setnx(key, FLAGS.node_name): + return FLAGS.node_name + else: + return redis.get(key) - def allocate_fixed_ip(self, user_id, project_id, *args, **kwargs): + def allocate_fixed_ip(self, user_id, project_id, + security_group='default', + *args, **kwargs): """Subclass implements getting fixed ip from the pool""" raise NotImplementedError() @@ -92,22 +104,11 @@ class BaseNetworkService(service.Service): class FlatNetworkService(BaseNetworkService): - def create_network(self, user_id, project_id, security_group='default', - *args, **kwargs): - """Creates network and returns bridge + """Basic network where no vlans are used""" - Flat network service simply returns a common bridge regardless of - project. - """ - return {'network_type': 'injected', - 'bridge_name': FLAGS.flat_network_bridge, - 'network_network': FLAGS.flat_network_network, - 'network_netmask': FLAGS.flat_network_netmask, - 'network_gateway': FLAGS.flat_network_gateway, - 'network_broadcast': FLAGS.flat_network_broadcast, - 'network_dns': FLAGS.flat_network_dns} - - def allocate_fixed_ip(self, user_id, project_id, *args, **kwargs): + def allocate_fixed_ip(self, user_id, project_id, + security_group='default', + *args, **kwargs): """Gets a fixed ip from the pool Flat network just grabs the next available ip from the pool @@ -119,23 +120,26 @@ class FlatNetworkService(BaseNetworkService): fixed_ip = redis.spop('ips') if not fixed_ip: raise exception.NoMoreAddresses() - return {'mac': utils.generate_mac(), - 'ip' : str(fixed_ip)} + return {'network_type': 'injected', + 'mac_address': utils.generate_mac(), + 'private_dns_name': str(fixed_ip), + 'bridge_name': FLAGS.flat_network_bridge, + 'network_network': FLAGS.flat_network_network, + 'network_netmask': FLAGS.flat_network_netmask, + 'network_gateway': FLAGS.flat_network_gateway, + 'network_broadcast': FLAGS.flat_network_broadcast, + 'network_dns': FLAGS.flat_network_dns} def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): """Returns an ip to the pool""" datastore.Redis.instance().sadd('ips', fixed_ip) class VlanNetworkService(BaseNetworkService): - """Allocates ips and sets up networks""" - def create_network(self, user_id, project_id, security_group='default', - *args, **kwargs): - """Creates network and returns bridge""" - net = model.get_project_network(project_id, security_group) - return {'network_type': 'dhcp', - 'bridge_name': net['bridge_name']} + """Vlan network with dhcp""" - def allocate_fixed_ip(self, user_id, project_id, vpn=False, *args, **kwargs): + def allocate_fixed_ip(self, user_id, project_id, + security_group='default', + vpn=False, *args, **kwargs): """Gets a fixed ip from the pool """ mac = utils.generate_mac() net = model.get_project_network(project_id) @@ -143,8 +147,10 @@ class VlanNetworkService(BaseNetworkService): fixed_ip = net.allocate_vpn_ip(user_id, project_id, mac) else: fixed_ip = net.allocate_ip(user_id, project_id, mac) - return {'mac': mac, - 'ip' : fixed_ip} + return {'network_type': 'dhcp', + 'bridge_name': net['bridge_name'], + 'mac_address': mac, + 'private_dns_name' : fixed_ip} def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 4b2f6c649..a9695f818 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -71,8 +71,8 @@ class NetworkTestCase(test.TrialTestCase): def test_allocate_deallocate_fixed_ip(self): result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) - address = result['ip'] - mac = result['mac'] + address = result['private_dns_name'] + mac = result['mac_address'] logging.debug("Was allocated %s" % (address)) net = model.get_project_network(self.projects[0].id, "default") self.assertEqual(True, is_in_project(address, self.projects[0].id)) @@ -90,12 +90,12 @@ class NetworkTestCase(test.TrialTestCase): hostname = "test-host" result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) - mac = result['mac'] - address = result['ip'] + mac = result['mac_address'] + address = result['private_dns_name'] result = self.service.allocate_fixed_ip( self.user, self.projects[1].id) - secondmac = result['mac'] - secondaddress = result['ip'] + secondmac = result['mac_address'] + secondaddress = result['private_dns_name'] net = model.get_project_network(self.projects[0].id, "default") secondnet = model.get_project_network(self.projects[1].id, "default") @@ -124,22 +124,22 @@ class NetworkTestCase(test.TrialTestCase): def test_subnet_edge(self): result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) - firstaddress = result['ip'] + firstaddress = result['private_dns_name'] hostname = "toomany-hosts" for i in range(1,5): project_id = self.projects[i].id result = self.service.allocate_fixed_ip( self.user, project_id) - mac = result['mac'] - address = result['ip'] + mac = result['mac_address'] + address = result['private_dns_name'] result = self.service.allocate_fixed_ip( self.user, project_id) - mac2 = result['mac'] - address2 = result['ip'] + mac2 = result['mac_address'] + address2 = result['private_dns_name'] result = self.service.allocate_fixed_ip( self.user, project_id) - mac3 = result['mac'] - address3 = result['ip'] + mac3 = result['mac_address'] + address3 = result['private_dns_name'] self.assertEqual(False, is_in_project(address, self.projects[0].id)) self.assertEqual(False, is_in_project(address2, self.projects[0].id)) self.assertEqual(False, is_in_project(address3, self.projects[0].id)) @@ -194,8 +194,8 @@ class NetworkTestCase(test.TrialTestCase): addresses = {} for i in range(0, (num_available_ips - 1)): result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) - macs[i] = result['mac'] - addresses[i] = result['ip'] + macs[i] = result['mac_address'] + addresses[i] = result['private_dns_name'] self.dnsmasq.issue_ip(macs[i], addresses[i], hostname, net.bridge_name) self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, self.user.id, self.projects[0].id) -- cgit From d0ca78ea900d71492212ac531ec75616b02300b0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 11:12:58 -0700 Subject: it helps to save files BEFORE committing --- nova/endpoint/cloud.py | 60 ++++++++++++++++++++++++++--------------------- nova/virt/libvirt_conn.py | 25 +++++++++++--------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index f605eec2c..32cbfdc10 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -38,6 +38,7 @@ from nova.auth import manager from nova.compute import model from nova.compute.instance_types import INSTANCE_TYPES from nova.endpoint import images +from nova.network import service as network_service from nova.volume import service @@ -468,25 +469,31 @@ class CloudController(object): return {'addressesSet': addresses} @rbac.allow('netadmin') + @defer.inlineCallbacks def allocate_address(self, context, **kwargs): - alloc_result = rpc.call(self._get_network_host(context), + network_host = yield self._get_network_host(context) + alloc_result = rpc.call(network_host, {"method": "allocate_elastic_ip"}) public_ip = alloc_result['result'] return defer.succeed({'addressSet': [{'publicIp' : public_ip}]}) @rbac.allow('netadmin') + @defer.inlineCallbacks def release_address(self, context, public_ip, **kwargs): # NOTE(vish): Should we make sure this works? - rpc.cast(self._get_network_host(context), + network_host = yield self._get_network_host(context) + rpc.cast(network_host, {"method": "deallocate_elastic_ip", "args": {"elastic_ip": public_ip}}) return defer.succeed({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') + @defer.inlineCallbacks def associate_address(self, context, instance_id, public_ip, **kwargs): instance = self._get_instance(context, instance_id) address = self._get_address(context, public_ip) - rpc.cast(self._get_network_host(context), + network_host = yield self._get_network_host(context) + rpc.cast(network_host, {"method": "associate_elastic_ip", "args": {"elastic_ip": address['public_ip'], "fixed_ip": instance['private_dns_name'], @@ -494,23 +501,26 @@ class CloudController(object): return defer.succeed({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') + @defer.inlineCallbacks def disassociate_address(self, context, public_ip, **kwargs): address = self._get_address(context, public_ip) - self.network.disassociate_address(public_ip) + network_host = yield self._get_network_host(context) + rpc.cast(network_host, + {"method": "associate_elastic_ip", + "args": {"elastic_ip": address['public_ip']}}) return defer.succeed({'disassociateResponse': ["Address disassociated."]}) - def release_ip(self, context, private_ip, **kwargs): - self.network.release_ip(private_ip) - return defer.succeed({'releaseResponse': ["Address released."]}) - - def lease_ip(self, context, private_ip, **kwargs): - self.network.lease_ip(private_ip) - return defer.succeed({'leaseResponse': ["Address leased."]}) - - def get_network_host(self, context): - # FIXME(vish): this is temporary until we store net hosts for project - import socket - return socket.gethostname() + @defer.inlineCallbacks + def _get_network_host(self, context): + """Retrieves the network host for a project""" + host = network_service.get_host_for_project(context.project.id) + if not host: + result = yield rpc.call(FLAGS.network_topic, + {"method": "get_network_host", + "args": {"user_id": context.user.id, + "project_id": context.project.id}}) + host = result['result'] + defer.returnValue(host) @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks @@ -545,27 +555,21 @@ class CloudController(object): raise exception.ApiError('Key Pair %s not found' % kwargs['key_name']) key_data = key_pair.public_key + network_host = yield self._get_network_host(context) # TODO: Get the real security group of launch in here security_group = "default" - create_result = yield rpc.call(FLAGS.network_topic, - {"method": "create_network", - "args": {"user_id": context.user.id, - "project_id": context.project.id, - "security_group": security_group}}) - bridge_name = create_result['result'] - net_host = self._get_network_host(context) for num in range(int(kwargs['max_count'])): vpn = False if image_id == FLAGS.vpn_image_id: vpn = True - allocate_result = yield rpc.call(net_host, + allocate_result = yield rpc.call(network_host, {"method": "allocate_fixed_ip", "args": {"user_id": context.user.id, "project_id": context.project.id, + "security_group": security_group, "vpn": vpn}}) + allocate_data = allocate_result['result'] inst = self.instdir.new() - inst['mac_address'] = allocate_result['result']['mac_address'] - inst['private_dns_name'] = allocate_result['result']['ip_address'] inst['image_id'] = image_id inst['kernel_id'] = kernel_id inst['ramdisk_id'] = ramdisk_id @@ -578,7 +582,9 @@ class CloudController(object): inst['user_id'] = context.user.id inst['project_id'] = context.project.id inst['ami_launch_index'] = num - inst['bridge_name'] = bridge_name + inst['security_group'] = security_group + for (key, value) in allocate_data: + inst[key] = value inst.save() rpc.cast(FLAGS.compute_topic, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c545e4190..8133daf0b 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -47,6 +47,9 @@ FLAGS = flags.FLAGS flags.DEFINE_string('libvirt_xml_template', utils.abspath('compute/libvirt.xml.template'), 'Libvirt XML Template') +flags.DEFINE_string('injected_network_template', + utils.abspath('compute/interfaces.template'), + 'Template file for injected network') def get_connection(read_only): # These are loaded late so that there's no need to install these @@ -201,14 +204,14 @@ class LibvirtConnection(object): key = data['key_data'] net = None - if FLAGS.simple_network: - with open(FLAGS.simple_network_template) as f: + if data.get('network_type', None) == 'injected': + with open(FLAGS.injected_network_template) as f: net = f.read() % {'address': data['private_dns_name'], - 'network': FLAGS.simple_network_network, - 'netmask': FLAGS.simple_network_netmask, - 'gateway': FLAGS.simple_network_gateway, - 'broadcast': FLAGS.simple_network_broadcast, - 'dns': FLAGS.simple_network_dns} + 'network': data['network_network'], + 'netmask': data['network_netmask'], + 'gateway': data['network_gateway'], + 'broadcast': data['network_broadcast'], + 'dns': data['network_dns']} if key or net: logging.info('Injecting data into image %s', data['image_id']) yield disk.inject_data(basepath('disk-raw'), key, net, execute=execute) @@ -255,7 +258,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - + Returns a list of all block devices for this domain. """ domain = self._conn.lookupByName(instance_id) @@ -298,7 +301,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - + Returns a list of all network interfaces for this instance. """ domain = self._conn.lookupByName(instance_id) @@ -341,7 +344,7 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - """ + """ domain = self._conn.lookupByName(instance_id) return domain.blockStats(disk) @@ -350,6 +353,6 @@ class LibvirtConnection(object): """ Note that this function takes an instance ID, not an Instance, so that it can be called by monitor. - """ + """ domain = self._conn.lookupByName(instance_id) return domain.interfaceStats(interface) -- cgit From b2e220c976b7689a2c5d924395c57012c6b99212 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 11:18:46 -0700 Subject: inline commands use returnValue --- nova/endpoint/cloud.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 32cbfdc10..3db999fba 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -475,7 +475,7 @@ class CloudController(object): alloc_result = rpc.call(network_host, {"method": "allocate_elastic_ip"}) public_ip = alloc_result['result'] - return defer.succeed({'addressSet': [{'publicIp' : public_ip}]}) + defer.returnValue({'addressSet': [{'publicIp' : public_ip}]}) @rbac.allow('netadmin') @defer.inlineCallbacks @@ -485,7 +485,7 @@ class CloudController(object): rpc.cast(network_host, {"method": "deallocate_elastic_ip", "args": {"elastic_ip": public_ip}}) - return defer.succeed({'releaseResponse': ["Address released."]}) + defer.returnValue({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') @defer.inlineCallbacks @@ -498,7 +498,7 @@ class CloudController(object): "args": {"elastic_ip": address['public_ip'], "fixed_ip": instance['private_dns_name'], "instance_id": instance['instance_id']}}) - return defer.succeed({'associateResponse': ["Address associated."]}) + defer.returnValue({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') @defer.inlineCallbacks @@ -508,7 +508,7 @@ class CloudController(object): rpc.cast(network_host, {"method": "associate_elastic_ip", "args": {"elastic_ip": address['public_ip']}}) - return defer.succeed({'disassociateResponse': ["Address disassociated."]}) + defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks def _get_network_host(self, context): @@ -596,8 +596,10 @@ class CloudController(object): defer.returnValue(self._format_instances(context, reservation_id)) @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") + network_host = yield self._get_network_host(context) for i in instance_id: logging.debug("Going to try and terminate %s" % i) try: @@ -612,7 +614,7 @@ class CloudController(object): # NOTE(vish): Right now we don't really care if the ip is # disassociated. We may need to worry about # checking this later. Perhaps in the scheduler? - rpc.cast(self._get_network_host(context), + rpc.cast(network_host, {"method": "disassociate_elastic_ip", "args": {"elastic_ip": elastic_ip}}) @@ -622,7 +624,7 @@ class CloudController(object): # NOTE(vish): Right now we don't really care if the ip is # actually removed. We may need to worry about # checking this later. Perhaps in the scheduler? - rpc.cast(self._get_network_host(context), + rpc.cast(network_host, {"method": "deallocate_fixed_ip", "args": {"elastic_ip": elastic_ip}}) @@ -633,7 +635,7 @@ class CloudController(object): "args": {"instance_id": i}}) else: instance.destroy() - return defer.succeed(True) + defer.returnValue(True) @rbac.allow('projectmanager', 'sysadmin') def reboot_instances(self, context, instance_id, **kwargs): -- cgit From 311daf14758d8a04c5f73fa4e2911e469a716c1f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 11:44:25 -0700 Subject: don't __ module methods --- nova/network/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/network/service.py b/nova/network/service.py index 45c2867ca..b3645d2a3 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -53,9 +53,9 @@ flags.DEFINE_string('flat_network_dns', '8.8.4.4', def get_host_for_project(project_id): redis = datastore.Redis.instance() - host = redis.get(__host_key(project_id)) + host = redis.get(_host_key(project_id)) -def __host_key(project_id): +def _host_key(project_id): return "network_host:%s" % project_id @@ -70,7 +70,7 @@ class BaseNetworkService(service.Service): def get_network_host(self, user_id, project_id, *args, **kwargs): """Safely becomes the host of the projects network""" redis = datastore.Redis.instance() - key = __host_key(project_id) + key = _host_key(project_id) if redis.setnx(key, FLAGS.node_name): return FLAGS.node_name else: -- cgit From e816e7923582d7ac11b7f7a554eec815ea61496e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 12:02:23 -0700 Subject: use deferreds in network --- nova/network/service.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nova/network/service.py b/nova/network/service.py index b3645d2a3..e998c5575 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -22,6 +22,8 @@ Network Nodes are responsible for allocating ips and setting up network import logging +from twisted.internet import defer + from nova import datastore from nova import flags from nova import service @@ -72,9 +74,9 @@ class BaseNetworkService(service.Service): redis = datastore.Redis.instance() key = _host_key(project_id) if redis.setnx(key, FLAGS.node_name): - return FLAGS.node_name + return defer.succeed(FLAGS.node_name) else: - return redis.get(key) + return defer.succeed(redis.get(key)) def allocate_fixed_ip(self, user_id, project_id, security_group='default', @@ -120,7 +122,7 @@ class FlatNetworkService(BaseNetworkService): fixed_ip = redis.spop('ips') if not fixed_ip: raise exception.NoMoreAddresses() - return {'network_type': 'injected', + return defer.succeed({'network_type': 'injected', 'mac_address': utils.generate_mac(), 'private_dns_name': str(fixed_ip), 'bridge_name': FLAGS.flat_network_bridge, @@ -128,7 +130,7 @@ class FlatNetworkService(BaseNetworkService): 'network_netmask': FLAGS.flat_network_netmask, 'network_gateway': FLAGS.flat_network_gateway, 'network_broadcast': FLAGS.flat_network_broadcast, - 'network_dns': FLAGS.flat_network_dns} + 'network_dns': FLAGS.flat_network_dns}) def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): """Returns an ip to the pool""" @@ -147,18 +149,18 @@ class VlanNetworkService(BaseNetworkService): fixed_ip = net.allocate_vpn_ip(user_id, project_id, mac) else: fixed_ip = net.allocate_ip(user_id, project_id, mac) - return {'network_type': 'dhcp', + return defer.succeed({'network_type': 'dhcp', 'bridge_name': net['bridge_name'], 'mac_address': mac, - 'private_dns_name' : fixed_ip} + 'private_dns_name' : fixed_ip}) def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): """Returns an ip to the pool""" - model.get_network_by_address(fixed_ip).deallocate_ip(fixed_ip) + return model.get_network_by_address(fixed_ip).deallocate_ip(fixed_ip) def lease_ip(self, address): - return self. __get_network_by_address(address).lease_ip(address) + return model.get_network_by_address(address).lease_ip(address) def release_ip(self, address): return model.get_network_by_address(address).release_ip(address) -- cgit From c821709a48eb22db4db182f25f1e405039294d2c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 12:18:52 -0700 Subject: method should return network topic instead of network host --- nova/endpoint/cloud.py | 32 ++++++++++++++++---------------- nova/network/service.py | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 3db999fba..33be0a612 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -471,8 +471,8 @@ class CloudController(object): @rbac.allow('netadmin') @defer.inlineCallbacks def allocate_address(self, context, **kwargs): - network_host = yield self._get_network_host(context) - alloc_result = rpc.call(network_host, + network_topic = yield self._get_network_topic(context) + alloc_result = rpc.call(network_topic, {"method": "allocate_elastic_ip"}) public_ip = alloc_result['result'] defer.returnValue({'addressSet': [{'publicIp' : public_ip}]}) @@ -481,8 +481,8 @@ class CloudController(object): @defer.inlineCallbacks def release_address(self, context, public_ip, **kwargs): # NOTE(vish): Should we make sure this works? - network_host = yield self._get_network_host(context) - rpc.cast(network_host, + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, {"method": "deallocate_elastic_ip", "args": {"elastic_ip": public_ip}}) defer.returnValue({'releaseResponse': ["Address released."]}) @@ -492,8 +492,8 @@ class CloudController(object): def associate_address(self, context, instance_id, public_ip, **kwargs): instance = self._get_instance(context, instance_id) address = self._get_address(context, public_ip) - network_host = yield self._get_network_host(context) - rpc.cast(network_host, + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, {"method": "associate_elastic_ip", "args": {"elastic_ip": address['public_ip'], "fixed_ip": instance['private_dns_name'], @@ -504,23 +504,23 @@ class CloudController(object): @defer.inlineCallbacks def disassociate_address(self, context, public_ip, **kwargs): address = self._get_address(context, public_ip) - network_host = yield self._get_network_host(context) - rpc.cast(network_host, + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, {"method": "associate_elastic_ip", "args": {"elastic_ip": address['public_ip']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks - def _get_network_host(self, context): + def _get_network_topic(self, context): """Retrieves the network host for a project""" host = network_service.get_host_for_project(context.project.id) if not host: result = yield rpc.call(FLAGS.network_topic, - {"method": "get_network_host", + {"method": "get_network_topic", "args": {"user_id": context.user.id, "project_id": context.project.id}}) host = result['result'] - defer.returnValue(host) + defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks @@ -555,14 +555,14 @@ class CloudController(object): raise exception.ApiError('Key Pair %s not found' % kwargs['key_name']) key_data = key_pair.public_key - network_host = yield self._get_network_host(context) + network_topic = yield self._get_network_topic(context) # TODO: Get the real security group of launch in here security_group = "default" for num in range(int(kwargs['max_count'])): vpn = False if image_id == FLAGS.vpn_image_id: vpn = True - allocate_result = yield rpc.call(network_host, + allocate_result = yield rpc.call(network_topic, {"method": "allocate_fixed_ip", "args": {"user_id": context.user.id, "project_id": context.project.id, @@ -599,7 +599,7 @@ class CloudController(object): @defer.inlineCallbacks def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") - network_host = yield self._get_network_host(context) + network_topic = yield self._get_network_topic(context) for i in instance_id: logging.debug("Going to try and terminate %s" % i) try: @@ -614,7 +614,7 @@ class CloudController(object): # NOTE(vish): Right now we don't really care if the ip is # disassociated. We may need to worry about # checking this later. Perhaps in the scheduler? - rpc.cast(network_host, + rpc.cast(network_topic, {"method": "disassociate_elastic_ip", "args": {"elastic_ip": elastic_ip}}) @@ -624,7 +624,7 @@ class CloudController(object): # NOTE(vish): Right now we don't really care if the ip is # actually removed. We may need to worry about # checking this later. Perhaps in the scheduler? - rpc.cast(network_host, + rpc.cast(network_topic, {"method": "deallocate_fixed_ip", "args": {"elastic_ip": elastic_ip}}) diff --git a/nova/network/service.py b/nova/network/service.py index e998c5575..1f36abc2b 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -55,7 +55,7 @@ flags.DEFINE_string('flat_network_dns', '8.8.4.4', def get_host_for_project(project_id): redis = datastore.Redis.instance() - host = redis.get(_host_key(project_id)) + return redis.get(_host_key(project_id)) def _host_key(project_id): return "network_host:%s" % project_id -- cgit From 148f319759fc9f566e0e9020ceb8ea00081ff8c8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 12:48:16 -0700 Subject: fixes in get public address and extra references to self.network --- nova/endpoint/cloud.py | 18 ++++++++++-------- nova/network/model.py | 12 ++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 33be0a612..957b25b9b 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -39,6 +39,7 @@ from nova.compute import model from nova.compute.instance_types import INSTANCE_TYPES from nova.endpoint import images from nova.network import service as network_service +from nova.network import model as network_model from nova.volume import service @@ -307,7 +308,7 @@ class CloudController(object): def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py - address = self.network.get_host(public_ip) + address = self.network_model.PublicAddress.lookup(public_ip) if address and (context.user.is_admin() or address['project_id'] == context.project.id): return address raise exception.NotFound("Address at ip %s not found" % public_ip) @@ -416,7 +417,7 @@ class CloudController(object): 'code': instance.get('state', 0), 'name': instance.get('state_description', 'pending') } - i['public_dns_name'] = self.network.get_public_ip_for_instance( + i['public_dns_name'] = self.network_model.get_public_ip_for_instance( i['instance_id']) i['private_dns_name'] = instance.get('private_dns_name', None) if not i['public_dns_name']: @@ -451,7 +452,7 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - for address in self.network.host_objs: + for address in self.network_model.PublicAddress.all(): # TODO(vish): implement a by_project iterator for addresses if (context.user.is_admin() or address['project_id'] == self.project.id): @@ -520,7 +521,7 @@ class CloudController(object): "args": {"user_id": context.user.id, "project_id": context.project.id}}) host = result['result'] - defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) + defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks @@ -608,9 +609,10 @@ class CloudController(object): logging.warning("Instance %s was not found during terminate" % i) continue - elastic_ip = instance.get('public_dns_name', None) - if elastic_ip: - logging.debug("Deallocating address %s" % elastic_ip) + address = self.network_model.get_public_ip_for_instance(i) + if address: + elastic_ip = address['public_ip'] + logging.debug("Disassociating address %s" % elastic_ip) # NOTE(vish): Right now we don't really care if the ip is # disassociated. We may need to worry about # checking this later. Perhaps in the scheduler? @@ -626,7 +628,7 @@ class CloudController(object): # checking this later. Perhaps in the scheduler? rpc.cast(network_topic, {"method": "deallocate_fixed_ip", - "args": {"elastic_ip": elastic_ip}}) + "args": {"fixed_ip": fixed_ip}}) if instance.get('node_name', 'unassigned') != 'unassigned': # NOTE(joshua?): It's also internal default diff --git a/nova/network/model.py b/nova/network/model.py index 5346549b8..7f0fded4c 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -399,12 +399,6 @@ class PublicNetworkController(BaseNetwork): for address in self.assigned: yield PublicAddress(address) - def get_public_ip_for_instance(self, instance_id): - # FIXME: this should be a lookup - iteration won't scale - for address_record in self.host_objs: - if address_record.get('instance_id', 'available') == instance_id: - return address_record['address'] - def get_host(self, host): if host in self.assigned: return PublicAddress(host) @@ -547,3 +541,9 @@ def get_network_by_interface(iface, security_group='default'): +def get_public_ip_for_instance(self, instance_id): + # FIXME: this should be a lookup - iteration won't scale + for address_record in PublicAddress.all(): + if address_record.get('instance_id', 'available') == instance_id: + return address_record['address'] + -- cgit From bfe90c9c26a0c477386f3143c1e9f0563b6a1a97 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 12:51:05 -0700 Subject: reference to self.project instead of context.project + self.network_model instead of network_model --- nova/endpoint/cloud.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 957b25b9b..338a52214 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -308,7 +308,7 @@ class CloudController(object): def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py - address = self.network_model.PublicAddress.lookup(public_ip) + address = network_model.PublicAddress.lookup(public_ip) if address and (context.user.is_admin() or address['project_id'] == context.project.id): return address raise exception.NotFound("Address at ip %s not found" % public_ip) @@ -417,7 +417,7 @@ class CloudController(object): 'code': instance.get('state', 0), 'name': instance.get('state_description', 'pending') } - i['public_dns_name'] = self.network_model.get_public_ip_for_instance( + i['public_dns_name'] = network_model.get_public_ip_for_instance( i['instance_id']) i['private_dns_name'] = instance.get('private_dns_name', None) if not i['public_dns_name']: @@ -452,10 +452,10 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - for address in self.network_model.PublicAddress.all(): + for address in network_model.PublicAddress.all(): # TODO(vish): implement a by_project iterator for addresses if (context.user.is_admin() or - address['project_id'] == self.project.id): + address['project_id'] == context.project.id): address_rv = { 'public_ip': address['address'], 'instance_id' : address.get('instance_id', 'free') @@ -609,7 +609,7 @@ class CloudController(object): logging.warning("Instance %s was not found during terminate" % i) continue - address = self.network_model.get_public_ip_for_instance(i) + address = network_model.get_public_ip_for_instance(i) if address: elastic_ip = address['public_ip'] logging.debug("Disassociating address %s" % elastic_ip) -- cgit From 4f6d71411ca23a4f92654f000e24fe008f0a00da Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 12:54:21 -0700 Subject: use iteritems --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 338a52214..789663899 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -584,7 +584,7 @@ class CloudController(object): inst['project_id'] = context.project.id inst['ami_launch_index'] = num inst['security_group'] = security_group - for (key, value) in allocate_data: + for (key, value) in allocate_data.iteritems(): inst[key] = value inst.save() -- cgit From a0eb1b9cc2e33c1a90501daeb3776738689e328f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 14:08:23 -0700 Subject: fix extra reference, method passing to network, various errors in elastic_ips --- nova/endpoint/cloud.py | 12 +++++++----- nova/network/model.py | 6 +++++- nova/network/service.py | 7 +++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 789663899..1695e4b05 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -473,8 +473,10 @@ class CloudController(object): @defer.inlineCallbacks def allocate_address(self, context, **kwargs): network_topic = yield self._get_network_topic(context) - alloc_result = rpc.call(network_topic, - {"method": "allocate_elastic_ip"}) + alloc_result = yield rpc.call(network_topic, + {"method": "allocate_elastic_ip", + "args": {"user_id": context.user.id, + "project_id": context.project.id}}) public_ip = alloc_result['result'] defer.returnValue({'addressSet': [{'publicIp' : public_ip}]}) @@ -496,7 +498,7 @@ class CloudController(object): network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, {"method": "associate_elastic_ip", - "args": {"elastic_ip": address['public_ip'], + "args": {"elastic_ip": address['address'], "fixed_ip": instance['private_dns_name'], "instance_id": instance['instance_id']}}) defer.returnValue({'associateResponse': ["Address associated."]}) @@ -507,8 +509,8 @@ class CloudController(object): address = self._get_address(context, public_ip) network_topic = yield self._get_network_topic(context) rpc.cast(network_topic, - {"method": "associate_elastic_ip", - "args": {"elastic_ip": address['public_ip']}}) + {"method": "disassociate_elastic_ip", + "args": {"elastic_ip": address['address']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) @defer.inlineCallbacks diff --git a/nova/network/model.py b/nova/network/model.py index 7f0fded4c..a1eaf5753 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -412,6 +412,10 @@ class PublicNetworkController(BaseNetwork): PublicAddress(host).destroy() datastore.Redis.instance().hdel(self._hosts_key, host) + def deallocate_ip(self, ip_str): + # NOTE(vish): cleanup is now done on release by the parent class + self.release_ip(ip_str) + def associate_address(self, public_ip, private_ip, instance_id): if not public_ip in self.assigned: raise exception.AddressNotAllocated() @@ -541,7 +545,7 @@ def get_network_by_interface(iface, security_group='default'): -def get_public_ip_for_instance(self, instance_id): +def get_public_ip_for_instance(instance_id): # FIXME: this should be a lookup - iteration won't scale for address_record in PublicAddress.all(): if address_record.get('instance_id', 'available') == instance_id: diff --git a/nova/network/service.py b/nova/network/service.py index 1f36abc2b..bf4aa0073 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -88,9 +88,12 @@ class BaseNetworkService(service.Service): """Subclass implements return of ip to the pool""" raise NotImplementedError() - def allocate_elastic_ip(self): + def allocate_elastic_ip(self, user_id, project_id): """Gets a elastic ip from the pool""" - return self.network.allocate_ip() + # NOTE(vish): Replicating earlier decision to use 'public' as + # mac address name, although this should probably + # be done inside of the PublicNetworkController + return self.network.allocate_ip(user_id, project_id, 'public') def associate_elastic_ip(self, elastic_ip, fixed_ip, instance_id): """Associates an elastic ip to a fixed ip""" -- cgit From 9a038d2b81163d3e658e4fb3be4f8c14aa3b5fab Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 15:44:23 -0700 Subject: fixed tests, moved compute network config call, added notes, made inject option into a boolean --- bin/nova-network | 7 +--- nova/compute/service.py | 19 +++++++---- nova/network/service.py | 75 ++++++++++++++++++++++++++++++++---------- nova/tests/network_unittest.py | 18 +++++----- nova/virt/libvirt_conn.py | 2 +- 5 files changed, 81 insertions(+), 40 deletions(-) diff --git a/bin/nova-network b/bin/nova-network index 1d620b525..ba9063f56 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -23,7 +23,6 @@ from nova import flags from nova import twistd -from nova import utils from nova.network import service @@ -34,8 +33,4 @@ if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - t = FLAGS.network_type - if t == 'flat': - application = service.FlatNetworkService.create() - elif t == 'vlan': - application = service.VlanNetworkService.create() + application = service.type_to_class(FLAGS.network_type).create() diff --git a/nova/compute/service.py b/nova/compute/service.py index a22240b05..820116453 100644 --- a/nova/compute/service.py +++ b/nova/compute/service.py @@ -41,7 +41,7 @@ from nova.compute import disk from nova.compute import model from nova.compute import power_state from nova.compute.instance_types import INSTANCE_TYPES -from nova.network import model as net_model +from nova.network import service as network_service from nova.objectstore import image # for image_path flag from nova.virt import connection as virt_connection from nova.volume import service as volume_service @@ -117,12 +117,17 @@ class ComputeService(service.Service): """ launch a new instance with specified options """ logging.debug("Starting instance %s..." % (instance_id)) inst = self.instdir.get(instance_id) - if inst.get('network_type', 'dhcp') == 'dhcp': - # TODO: Get the real security group of launch in here - security_group = "default" - net = net_model.BridgedNetwork.get_network_for_project(inst['user_id'], - inst['project_id'], - security_group).express() + # TODO: Get the real security group of launch in here + security_group = "default" + # NOTE(vish): passing network type allows us to express the + # network without making a call to network to find + # out which type of network to setup + network_service.setup_compute_network( + inst.get('network_type', 'vlan'), + inst['user_id'], + inst['project_id'], + security_group) + inst['node_name'] = FLAGS.node_name inst.save() # TODO(vish) check to make sure the availability zone matches diff --git a/nova/network/service.py b/nova/network/service.py index bf4aa0073..57b8bbb78 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -20,8 +20,6 @@ Network Nodes are responsible for allocating ips and setting up network """ -import logging - from twisted.internet import defer from nova import datastore @@ -29,6 +27,7 @@ from nova import flags from nova import service from nova import utils from nova.auth import manager +from nova.exception import NotFound from nova.network import exception from nova.network import model @@ -53,10 +52,24 @@ flags.DEFINE_string('flat_network_broadcast', '192.168.0.255', flags.DEFINE_string('flat_network_dns', '8.8.4.4', 'Dns for simple network') +def type_to_class(network_type): + if network_type == 'flat': + return FlatNetworkService + elif network_type == 'vlan': + return VlanNetworkService + raise NotFound("Couldn't find %s network type" % network_type) + + +def setup_compute_network(network_type, user_id, project_id, security_group): + srv = type_to_class(network_type) + srv.setup_compute_network(network_type, user_id, project_id, security_group) + + def get_host_for_project(project_id): redis = datastore.Redis.instance() return redis.get(_host_key(project_id)) + def _host_key(project_id): return "network_host:%s" % project_id @@ -88,6 +101,12 @@ class BaseNetworkService(service.Service): """Subclass implements return of ip to the pool""" raise NotImplementedError() + @classmethod + def setup_compute_network(self, user_id, project_id, security_group, + *args, **kwargs): + """Sets up matching network for compute hosts""" + raise NotImplementedError() + def allocate_elastic_ip(self, user_id, project_id): """Gets a elastic ip from the pool""" # NOTE(vish): Replicating earlier decision to use 'public' as @@ -111,6 +130,11 @@ class BaseNetworkService(service.Service): class FlatNetworkService(BaseNetworkService): """Basic network where no vlans are used""" + @classmethod + def setup_compute_network(self, fixed_ip, *args, **kwargs): + """Network is created manually""" + pass + def allocate_fixed_ip(self, user_id, project_id, security_group='default', *args, **kwargs): @@ -118,6 +142,9 @@ class FlatNetworkService(BaseNetworkService): Flat network just grabs the next available ip from the pool """ + # NOTE(vish): Some automation could be done here. For example, + # creating the flat_network_bridge and setting up + # a gateway. This is all done manually atm redis = datastore.Redis.instance() if not redis.exists('ips') and not len(redis.keys('instances:*')): for fixed_ip in FLAGS.flat_network_ips: @@ -125,15 +152,16 @@ class FlatNetworkService(BaseNetworkService): fixed_ip = redis.spop('ips') if not fixed_ip: raise exception.NoMoreAddresses() - return defer.succeed({'network_type': 'injected', - 'mac_address': utils.generate_mac(), - 'private_dns_name': str(fixed_ip), - 'bridge_name': FLAGS.flat_network_bridge, - 'network_network': FLAGS.flat_network_network, - 'network_netmask': FLAGS.flat_network_netmask, - 'network_gateway': FLAGS.flat_network_gateway, - 'network_broadcast': FLAGS.flat_network_broadcast, - 'network_dns': FLAGS.flat_network_dns}) + return defer.succeed({'inject_network': True, + 'network_type': FLAGS.network_type, + 'mac_address': utils.generate_mac(), + 'private_dns_name': str(fixed_ip), + 'bridge_name': FLAGS.flat_network_bridge, + 'network_network': FLAGS.flat_network_network, + 'network_netmask': FLAGS.flat_network_netmask, + 'network_gateway': FLAGS.flat_network_gateway, + 'network_broadcast': FLAGS.flat_network_broadcast, + 'network_dns': FLAGS.flat_network_dns}) def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): """Returns an ip to the pool""" @@ -141,7 +169,10 @@ class FlatNetworkService(BaseNetworkService): class VlanNetworkService(BaseNetworkService): """Vlan network with dhcp""" - + # NOTE(vish): A lot of the interactions with network/model.py can be + # simplified and improved. Also there it may be useful + # to support vlans separately from dhcp, instead of having + # both of them together in this class. def allocate_fixed_ip(self, user_id, project_id, security_group='default', vpn=False, *args, **kwargs): @@ -152,10 +183,10 @@ class VlanNetworkService(BaseNetworkService): fixed_ip = net.allocate_vpn_ip(user_id, project_id, mac) else: fixed_ip = net.allocate_ip(user_id, project_id, mac) - return defer.succeed({'network_type': 'dhcp', - 'bridge_name': net['bridge_name'], - 'mac_address': mac, - 'private_dns_name' : fixed_ip}) + return defer.succeed({'network_type': FLAGS.network_type, + 'bridge_name': net['bridge_name'], + 'mac_address': mac, + 'private_dns_name' : fixed_ip}) def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): @@ -173,4 +204,14 @@ class VlanNetworkService(BaseNetworkService): for project in manager.AuthManager().get_projects(): model.get_project_network(project.id).express() - + @classmethod + def setup_compute_network(self, user_id, project_id, security_group, + *args, **kwargs): + """Sets up matching network for compute hosts""" + # NOTE(vish): Use BridgedNetwork instead of DHCPNetwork because + # we don't want to run dnsmasq on the client machines + net = model.BridgedNetwork.get_network_for_project( + user_id, + project_id, + security_group) + net.express() diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index a9695f818..42cae327f 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -69,7 +69,7 @@ class NetworkTestCase(test.TrialTestCase): self.assertTrue(IPy.IP(address) in self.network.network) def test_allocate_deallocate_fixed_ip(self): - result = self.service.allocate_fixed_ip( + result = yield self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) address = result['private_dns_name'] mac = result['mac_address'] @@ -88,11 +88,11 @@ class NetworkTestCase(test.TrialTestCase): def test_range_allocation(self): hostname = "test-host" - result = self.service.allocate_fixed_ip( + result = yield self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) mac = result['mac_address'] address = result['private_dns_name'] - result = self.service.allocate_fixed_ip( + result = yield self.service.allocate_fixed_ip( self.user, self.projects[1].id) secondmac = result['mac_address'] secondaddress = result['private_dns_name'] @@ -122,21 +122,21 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(False, is_in_project(secondaddress, self.projects[1].id)) def test_subnet_edge(self): - result = self.service.allocate_fixed_ip(self.user.id, + result = yield self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) firstaddress = result['private_dns_name'] hostname = "toomany-hosts" for i in range(1,5): project_id = self.projects[i].id - result = self.service.allocate_fixed_ip( + result = yield self.service.allocate_fixed_ip( self.user, project_id) mac = result['mac_address'] address = result['private_dns_name'] - result = self.service.allocate_fixed_ip( + result = yield self.service.allocate_fixed_ip( self.user, project_id) mac2 = result['mac_address'] address2 = result['private_dns_name'] - result = self.service.allocate_fixed_ip( + result = yield self.service.allocate_fixed_ip( self.user, project_id) mac3 = result['mac_address'] address3 = result['private_dns_name'] @@ -193,12 +193,12 @@ class NetworkTestCase(test.TrialTestCase): macs = {} addresses = {} for i in range(0, (num_available_ips - 1)): - result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) + result = yield self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) macs[i] = result['mac_address'] addresses[i] = result['private_dns_name'] self.dnsmasq.issue_ip(macs[i], addresses[i], hostname, net.bridge_name) - self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, self.user.id, self.projects[0].id) + self.assertFailure(self.service.allocate_fixed_ip(self.user.id, self.projects[0].id), NoMoreAddresses) for i in range(0, (num_available_ips - 1)): rv = self.service.deallocate_fixed_ip(addresses[i]) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 8133daf0b..7c3f7a6e1 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -204,7 +204,7 @@ class LibvirtConnection(object): key = data['key_data'] net = None - if data.get('network_type', None) == 'injected': + if data.get('inject_network', False): with open(FLAGS.injected_network_template) as f: net = f.read() % {'address': data['private_dns_name'], 'network': data['network_network'], -- cgit From de456585b67f3eb46bcae5869af4ac83c6d95908 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 15:53:24 -0700 Subject: fix error on terminate instance relating to elastic ip --- nova/endpoint/cloud.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 1695e4b05..349007efa 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -611,9 +611,8 @@ class CloudController(object): logging.warning("Instance %s was not found during terminate" % i) continue - address = network_model.get_public_ip_for_instance(i) - if address: - elastic_ip = address['public_ip'] + elastic_ip = network_model.get_public_ip_for_instance(i) + if elastic_ip: logging.debug("Disassociating address %s" % elastic_ip) # NOTE(vish): Right now we don't really care if the ip is # disassociated. We may need to worry about -- cgit From aa84936cb63cd1913a1640944a9353d018ace13f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 16:51:48 -0700 Subject: fix rpc command line call, remove useless deferreds --- nova/network/service.py | 34 ++++++++++++++++------------------ nova/rpc.py | 4 ++-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/nova/network/service.py b/nova/network/service.py index 57b8bbb78..873e31630 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -20,8 +20,6 @@ Network Nodes are responsible for allocating ips and setting up network """ -from twisted.internet import defer - from nova import datastore from nova import flags from nova import service @@ -87,9 +85,9 @@ class BaseNetworkService(service.Service): redis = datastore.Redis.instance() key = _host_key(project_id) if redis.setnx(key, FLAGS.node_name): - return defer.succeed(FLAGS.node_name) + return FLAGS.node_name else: - return defer.succeed(redis.get(key)) + return redis.get(key) def allocate_fixed_ip(self, user_id, project_id, security_group='default', @@ -152,16 +150,16 @@ class FlatNetworkService(BaseNetworkService): fixed_ip = redis.spop('ips') if not fixed_ip: raise exception.NoMoreAddresses() - return defer.succeed({'inject_network': True, - 'network_type': FLAGS.network_type, - 'mac_address': utils.generate_mac(), - 'private_dns_name': str(fixed_ip), - 'bridge_name': FLAGS.flat_network_bridge, - 'network_network': FLAGS.flat_network_network, - 'network_netmask': FLAGS.flat_network_netmask, - 'network_gateway': FLAGS.flat_network_gateway, - 'network_broadcast': FLAGS.flat_network_broadcast, - 'network_dns': FLAGS.flat_network_dns}) + return {'inject_network': True, + 'network_type': FLAGS.network_type, + 'mac_address': utils.generate_mac(), + 'private_dns_name': str(fixed_ip), + 'bridge_name': FLAGS.flat_network_bridge, + 'network_network': FLAGS.flat_network_network, + 'network_netmask': FLAGS.flat_network_netmask, + 'network_gateway': FLAGS.flat_network_gateway, + 'network_broadcast': FLAGS.flat_network_broadcast, + 'network_dns': FLAGS.flat_network_dns} def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): """Returns an ip to the pool""" @@ -183,10 +181,10 @@ class VlanNetworkService(BaseNetworkService): fixed_ip = net.allocate_vpn_ip(user_id, project_id, mac) else: fixed_ip = net.allocate_ip(user_id, project_id, mac) - return defer.succeed({'network_type': FLAGS.network_type, - 'bridge_name': net['bridge_name'], - 'mac_address': mac, - 'private_dns_name' : fixed_ip}) + return {'network_type': FLAGS.network_type, + 'bridge_name': net['bridge_name'], + 'mac_address': mac, + 'private_dns_name' : fixed_ip} def deallocate_fixed_ip(self, fixed_ip, *args, **kwargs): diff --git a/nova/rpc.py b/nova/rpc.py index ebf140d92..2a550c3ae 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -238,12 +238,12 @@ def send_message(topic, message, wait=True): exchange=msg_id, auto_delete=True, exchange_type="direct", - routing_key=msg_id, - durable=False) + routing_key=msg_id) consumer.register_callback(generic_response) publisher = messaging.Publisher(connection=Connection.instance(), exchange=FLAGS.control_exchange, + durable=False, exchange_type="topic", routing_key=topic) publisher.send(message) -- cgit From 6b70951e5b7cb8cabe5d6eb50fce7ae0a6e55d52 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 17:40:16 -0700 Subject: renamed Vpn to NetworkData, moved the creation of data to inside network --- nova/auth/manager.py | 100 +++--------------------------------------------- nova/network/model.py | 90 +++++++++++++++++++++++++++++++++++++++++++ nova/network/service.py | 20 ++++++++-- 3 files changed, 112 insertions(+), 98 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 2da53a736..dacdeb383 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -37,6 +37,7 @@ from nova import objectstore # for flags from nova import utils from nova.auth import ldapdriver # for flags from nova.auth import signer +from nova.network import model FLAGS = flags.FLAGS @@ -51,7 +52,6 @@ flags.DEFINE_list('global_roles', ['cloudadmin', 'itsec'], 'Roles that apply to all projects') -flags.DEFINE_bool('use_vpn', True, 'Support per-project vpns') flags.DEFINE_string('credentials_template', utils.abspath('auth/novarc.template'), 'Template for creating users rc file') @@ -65,19 +65,11 @@ flags.DEFINE_string('credential_cert_file', 'cert.pem', flags.DEFINE_string('credential_rc_file', 'novarc', 'Filename of rc in credentials zip') -flags.DEFINE_integer('vpn_start_port', 1000, - 'Start port for the cloudpipe VPN servers') -flags.DEFINE_integer('vpn_end_port', 2000, - 'End port for the cloudpipe VPN servers') - flags.DEFINE_string('credential_cert_subject', '/C=US/ST=California/L=MountainView/O=AnsoLabs/' 'OU=NovaDev/CN=%s-%s', 'Subject for certificate for users') -flags.DEFINE_string('vpn_ip', '127.0.0.1', - 'Public IP for the cloudpipe VPN servers') - flags.DEFINE_string('auth_driver', 'nova.auth.ldapdriver.FakeLdapDriver', 'Driver that auth manager uses') @@ -229,86 +221,6 @@ class Project(AuthBase): self.member_ids) -class NoMorePorts(exception.Error): - pass - - -class Vpn(datastore.BasicModel): - """Manages vpn ips and ports for projects""" - def __init__(self, project_id): - self.project_id = project_id - super(Vpn, self).__init__() - - @property - def identifier(self): - """Identifier used for key in redis""" - return self.project_id - - @classmethod - def create(cls, project_id): - """Creates a vpn for project - - This method finds a free ip and port and stores the associated - values in the datastore. - """ - # TODO(vish): get list of vpn ips from redis - port = cls.find_free_port_for_ip(FLAGS.vpn_ip) - vpn = cls(project_id) - # save ip for project - vpn['project'] = project_id - vpn['ip'] = FLAGS.vpn_ip - vpn['port'] = port - vpn.save() - return vpn - - @classmethod - def find_free_port_for_ip(cls, ip): - """Finds a free port for a given ip from the redis set""" - # TODO(vish): these redis commands should be generalized and - # placed into a base class. Conceptually, it is - # similar to an association, but we are just - # storing a set of values instead of keys that - # should be turned into objects. - redis = datastore.Redis.instance() - key = 'ip:%s:ports' % ip - # TODO(vish): these ports should be allocated through an admin - # command instead of a flag - if (not redis.exists(key) and - not redis.exists(cls._redis_association_name('ip', ip))): - for i in range(FLAGS.vpn_start_port, FLAGS.vpn_end_port + 1): - redis.sadd(key, i) - - port = redis.spop(key) - if not port: - raise NoMorePorts() - return port - - @classmethod - def num_ports_for_ip(cls, ip): - """Calculates the number of free ports for a given ip""" - return datastore.Redis.instance().scard('ip:%s:ports' % ip) - - @property - def ip(self): - """The ip assigned to the project""" - return self['ip'] - - @property - def port(self): - """The port assigned to the project""" - return int(self['port']) - - def save(self): - """Saves the association to the given ip""" - self.associate_with('ip', self.ip) - super(Vpn, self).save() - - def destroy(self): - """Cleans up datastore and adds port back to pool""" - self.unassociate_with('ip', self.ip) - datastore.Redis.instance().sadd('ip:%s:ports' % self.ip, self.port) - super(Vpn, self).destroy() - class AuthManager(object): """Manager Singleton for dealing with Users, Projects, and Keypairs @@ -581,8 +493,6 @@ class AuthManager(object): description, member_users) if project_dict: - if FLAGS.use_vpn: - Vpn.create(project_dict['id']) return Project(**project_dict) def add_to_project(self, user, project): @@ -619,10 +529,10 @@ class AuthManager(object): @return: A tuple containing (ip, port) or None, None if vpn has not been allocated for user. """ - vpn = Vpn.lookup(Project.safe_id(project)) - if not vpn: - return None, None - return (vpn.ip, vpn.port) + network_data = model.NetworkData.lookup(Project.safe_id(project)) + if not network_data: + raise exception.NotFound('project network data has not been set') + return (network_data.ip, network_data.port) def delete_project(self, project): """Deletes a project""" diff --git a/nova/network/model.py b/nova/network/model.py index a1eaf5753..b065d174e 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -53,6 +53,13 @@ flags.DEFINE_integer('cnt_vpn_clients', 5, flags.DEFINE_integer('cloudpipe_start_port', 12000, 'Starting port for mapped CloudPipe external ports') +flags.DEFINE_string('vpn_ip', utils.get_my_ip(), + 'Public IP for the cloudpipe VPN servers') +flags.DEFINE_integer('vpn_start_port', 1000, + 'Start port for the cloudpipe VPN servers') +flags.DEFINE_integer('vpn_end_port', 2000, + 'End port for the cloudpipe VPN servers') + logging.getLogger().setLevel(logging.DEBUG) @@ -373,6 +380,89 @@ class PublicAddress(datastore.BasicModel): addr.save() return addr + +class NoMorePorts(exception.Error): + pass + + +class NetworkData(datastore.BasicModel): + """Manages network host, and vpn ip and port for projects""" + def __init__(self, project_id): + self.project_id = project_id + super(NetworkData, self).__init__() + + @property + def identifier(self): + """Identifier used for key in redis""" + return self.project_id + + @classmethod + def create(cls, project_id): + """Creates a vpn for project + + This method finds a free ip and port and stores the associated + values in the datastore. + """ + # TODO(vish): will we ever need multiiple ips per host? + port = cls.find_free_port_for_ip(FLAGS.vpn_ip) + network_data = cls(project_id) + # save ip for project + network_data['host'] = FLAGS.node_name + network_data['project'] = project_id + network_data['ip'] = FLAGS.vpn_ip + network_data['port'] = port + network_data.save() + return network_data + + @classmethod + def find_free_port_for_ip(cls, ip): + """Finds a free port for a given ip from the redis set""" + # TODO(vish): these redis commands should be generalized and + # placed into a base class. Conceptually, it is + # similar to an association, but we are just + # storing a set of values instead of keys that + # should be turned into objects. + redis = datastore.Redis.instance() + key = 'ip:%s:ports' % ip + # TODO(vish): these ports should be allocated through an admin + # command instead of a flag + if (not redis.exists(key) and + not redis.exists(cls._redis_association_name('ip', ip))): + for i in range(FLAGS.vpn_start_port, FLAGS.vpn_end_port + 1): + redis.sadd(key, i) + + port = redis.spop(key) + if not port: + raise NoMorePorts() + return port + + @classmethod + def num_ports_for_ip(cls, ip): + """Calculates the number of free ports for a given ip""" + return datastore.Redis.instance().scard('ip:%s:ports' % ip) + + @property + def ip(self): + """The ip assigned to the project""" + return self['ip'] + + @property + def port(self): + """The port assigned to the project""" + return int(self['port']) + + def save(self): + """Saves the association to the given ip""" + self.associate_with('ip', self.ip) + super(NetworkData, self).save() + + def destroy(self): + """Cleans up datastore and adds port back to pool""" + self.unassociate_with('ip', self.ip) + datastore.Redis.instance().sadd('ip:%s:ports' % self.ip, self.port) + super(NetworkData, self).destroy() + + DEFAULT_PORTS = [("tcp",80), ("tcp",22), ("udp",1194), ("tcp",443)] class PublicNetworkController(BaseNetwork): override_type = 'network' diff --git a/nova/network/service.py b/nova/network/service.py index 873e31630..243f9a720 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -80,11 +80,14 @@ class BaseNetworkService(service.Service): def __init__(self, *args, **kwargs): self.network = model.PublicNetworkController() - def get_network_host(self, user_id, project_id, *args, **kwargs): + def set_network_host(self, user_id, project_id, *args, **kwargs): """Safely becomes the host of the projects network""" redis = datastore.Redis.instance() key = _host_key(project_id) if redis.setnx(key, FLAGS.node_name): + self._on_set_network_host(user_id, project_id, + security_group='default', + *args, **kwargs) return FLAGS.node_name else: return redis.get(key) @@ -99,6 +102,11 @@ class BaseNetworkService(service.Service): """Subclass implements return of ip to the pool""" raise NotImplementedError() + def _on_set_network_host(self, user_id, project_id, + *args, **kwargs): + """Called when this host becomes the host for a project""" + pass + @classmethod def setup_compute_network(self, user_id, project_id, security_group, *args, **kwargs): @@ -129,7 +137,8 @@ class FlatNetworkService(BaseNetworkService): """Basic network where no vlans are used""" @classmethod - def setup_compute_network(self, fixed_ip, *args, **kwargs): + def setup_compute_network(self, user_id, project_id, security_group, + *args, **kwargs): """Network is created manually""" pass @@ -198,10 +207,15 @@ class VlanNetworkService(BaseNetworkService): return model.get_network_by_address(address).release_ip(address) def restart_nets(self): - """ Ensure the network for each user is enabled""" + """Ensure the network for each user is enabled""" for project in manager.AuthManager().get_projects(): model.get_project_network(project.id).express() + def _on_set_network_host(self, user_id, project_id, + *args, **kwargs): + """Called when this host becomes the host for a project""" + model.NetworkData.create(project_id) + @classmethod def setup_compute_network(self, user_id, project_id, security_group, *args, **kwargs): -- cgit From c1e0abd5be3ac8473aaf255f77fb2357b5771ea9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 18:00:13 -0700 Subject: fixed circular reference and tests --- nova/auth/manager.py | 4 +- nova/network/model.py | 89 ------------------------------- nova/network/networkdata.py | 116 +++++++++++++++++++++++++++++++++++++++++ nova/network/service.py | 5 +- nova/tests/auth_unittest.py | 14 ----- nova/tests/network_unittest.py | 15 ++++++ 6 files changed, 136 insertions(+), 107 deletions(-) create mode 100644 nova/network/networkdata.py diff --git a/nova/auth/manager.py b/nova/auth/manager.py index dacdeb383..463cfdf4a 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -37,7 +37,7 @@ from nova import objectstore # for flags from nova import utils from nova.auth import ldapdriver # for flags from nova.auth import signer -from nova.network import model +from nova.network import networkdata FLAGS = flags.FLAGS @@ -529,7 +529,7 @@ class AuthManager(object): @return: A tuple containing (ip, port) or None, None if vpn has not been allocated for user. """ - network_data = model.NetworkData.lookup(Project.safe_id(project)) + network_data = networkdata.NetworkData.lookup(Project.safe_id(project)) if not network_data: raise exception.NotFound('project network data has not been set') return (network_data.ip, network_data.port) diff --git a/nova/network/model.py b/nova/network/model.py index b065d174e..daac035e4 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -53,13 +53,6 @@ flags.DEFINE_integer('cnt_vpn_clients', 5, flags.DEFINE_integer('cloudpipe_start_port', 12000, 'Starting port for mapped CloudPipe external ports') -flags.DEFINE_string('vpn_ip', utils.get_my_ip(), - 'Public IP for the cloudpipe VPN servers') -flags.DEFINE_integer('vpn_start_port', 1000, - 'Start port for the cloudpipe VPN servers') -flags.DEFINE_integer('vpn_end_port', 2000, - 'End port for the cloudpipe VPN servers') - logging.getLogger().setLevel(logging.DEBUG) @@ -381,88 +374,6 @@ class PublicAddress(datastore.BasicModel): return addr -class NoMorePorts(exception.Error): - pass - - -class NetworkData(datastore.BasicModel): - """Manages network host, and vpn ip and port for projects""" - def __init__(self, project_id): - self.project_id = project_id - super(NetworkData, self).__init__() - - @property - def identifier(self): - """Identifier used for key in redis""" - return self.project_id - - @classmethod - def create(cls, project_id): - """Creates a vpn for project - - This method finds a free ip and port and stores the associated - values in the datastore. - """ - # TODO(vish): will we ever need multiiple ips per host? - port = cls.find_free_port_for_ip(FLAGS.vpn_ip) - network_data = cls(project_id) - # save ip for project - network_data['host'] = FLAGS.node_name - network_data['project'] = project_id - network_data['ip'] = FLAGS.vpn_ip - network_data['port'] = port - network_data.save() - return network_data - - @classmethod - def find_free_port_for_ip(cls, ip): - """Finds a free port for a given ip from the redis set""" - # TODO(vish): these redis commands should be generalized and - # placed into a base class. Conceptually, it is - # similar to an association, but we are just - # storing a set of values instead of keys that - # should be turned into objects. - redis = datastore.Redis.instance() - key = 'ip:%s:ports' % ip - # TODO(vish): these ports should be allocated through an admin - # command instead of a flag - if (not redis.exists(key) and - not redis.exists(cls._redis_association_name('ip', ip))): - for i in range(FLAGS.vpn_start_port, FLAGS.vpn_end_port + 1): - redis.sadd(key, i) - - port = redis.spop(key) - if not port: - raise NoMorePorts() - return port - - @classmethod - def num_ports_for_ip(cls, ip): - """Calculates the number of free ports for a given ip""" - return datastore.Redis.instance().scard('ip:%s:ports' % ip) - - @property - def ip(self): - """The ip assigned to the project""" - return self['ip'] - - @property - def port(self): - """The port assigned to the project""" - return int(self['port']) - - def save(self): - """Saves the association to the given ip""" - self.associate_with('ip', self.ip) - super(NetworkData, self).save() - - def destroy(self): - """Cleans up datastore and adds port back to pool""" - self.unassociate_with('ip', self.ip) - datastore.Redis.instance().sadd('ip:%s:ports' % self.ip, self.port) - super(NetworkData, self).destroy() - - DEFAULT_PORTS = [("tcp",80), ("tcp",22), ("udp",1194), ("tcp",443)] class PublicNetworkController(BaseNetwork): override_type = 'network' diff --git a/nova/network/networkdata.py b/nova/network/networkdata.py new file mode 100644 index 000000000..cec84287c --- /dev/null +++ b/nova/network/networkdata.py @@ -0,0 +1,116 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Network Data for projects""" + +from nova import datastore +from nova import exception +from nova import flags +from nova import utils + +FLAGS = flags.FLAGS + + +flags.DEFINE_string('vpn_ip', utils.get_my_ip(), + 'Public IP for the cloudpipe VPN servers') +flags.DEFINE_integer('vpn_start_port', 1000, + 'Start port for the cloudpipe VPN servers') +flags.DEFINE_integer('vpn_end_port', 2000, + 'End port for the cloudpipe VPN servers') + +class NoMorePorts(exception.Error): + pass + + +class NetworkData(datastore.BasicModel): + """Manages network host, and vpn ip and port for projects""" + def __init__(self, project_id): + self.project_id = project_id + super(NetworkData, self).__init__() + + @property + def identifier(self): + """Identifier used for key in redis""" + return self.project_id + + @classmethod + def create(cls, project_id): + """Creates a vpn for project + + This method finds a free ip and port and stores the associated + values in the datastore. + """ + # TODO(vish): will we ever need multiiple ips per host? + port = cls.find_free_port_for_ip(FLAGS.vpn_ip) + network_data = cls(project_id) + # save ip for project + network_data['host'] = FLAGS.node_name + network_data['project'] = project_id + network_data['ip'] = FLAGS.vpn_ip + network_data['port'] = port + network_data.save() + return network_data + + @classmethod + def find_free_port_for_ip(cls, ip): + """Finds a free port for a given ip from the redis set""" + # TODO(vish): these redis commands should be generalized and + # placed into a base class. Conceptually, it is + # similar to an association, but we are just + # storing a set of values instead of keys that + # should be turned into objects. + redis = datastore.Redis.instance() + key = 'ip:%s:ports' % ip + # TODO(vish): these ports should be allocated through an admin + # command instead of a flag + if (not redis.exists(key) and + not redis.exists(cls._redis_association_name('ip', ip))): + for i in range(FLAGS.vpn_start_port, FLAGS.vpn_end_port + 1): + redis.sadd(key, i) + + port = redis.spop(key) + if not port: + raise NoMorePorts() + return port + + @classmethod + def num_ports_for_ip(cls, ip): + """Calculates the number of free ports for a given ip""" + return datastore.Redis.instance().scard('ip:%s:ports' % ip) + + @property + def ip(self): + """The ip assigned to the project""" + return self['ip'] + + @property + def port(self): + """The port assigned to the project""" + return int(self['port']) + + def save(self): + """Saves the association to the given ip""" + self.associate_with('ip', self.ip) + super(NetworkData, self).save() + + def destroy(self): + """Cleans up datastore and adds port back to pool""" + self.unassociate_with('ip', self.ip) + datastore.Redis.instance().sadd('ip:%s:ports' % self.ip, self.port) + super(NetworkData, self).destroy() + diff --git a/nova/network/service.py b/nova/network/service.py index 243f9a720..afc20c0d5 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -28,6 +28,7 @@ from nova.auth import manager from nova.exception import NotFound from nova.network import exception from nova.network import model +from nova.network import networkdata FLAGS = flags.FLAGS @@ -81,7 +82,7 @@ class BaseNetworkService(service.Service): self.network = model.PublicNetworkController() def set_network_host(self, user_id, project_id, *args, **kwargs): - """Safely becomes the host of the projects network""" + """Safely sets the host of the projects network""" redis = datastore.Redis.instance() key = _host_key(project_id) if redis.setnx(key, FLAGS.node_name): @@ -214,7 +215,7 @@ class VlanNetworkService(BaseNetworkService): def _on_set_network_host(self, user_id, project_id, *args, **kwargs): """Called when this host becomes the host for a project""" - model.NetworkData.create(project_id) + networkdata.NetworkData.create(project_id) @classmethod def setup_compute_network(self, user_id, project_id, security_group, diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 2167c2385..3bd0a432c 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -179,20 +179,6 @@ class AuthTestCase(test.BaseTestCase): self.manager.remove_role('test1', 'sysadmin') self.assertFalse(project.has_role('test1', 'sysadmin')) - def test_212_vpn_ip_and_port_looks_valid(self): - project = self.manager.get_project('testproj') - self.assert_(project.vpn_ip) - self.assert_(project.vpn_port >= FLAGS.vpn_start_port) - self.assert_(project.vpn_port <= FLAGS.vpn_end_port) - - def test_213_too_many_vpns(self): - vpns = [] - for i in xrange(manager.Vpn.num_ports_for_ip(FLAGS.vpn_ip)): - vpns.append(manager.Vpn.create("vpnuser%s" % i)) - self.assertRaises(manager.NoMorePorts, manager.Vpn.create, "boom") - for vpn in vpns: - vpn.destroy() - def test_214_can_retrieve_project_by_user(self): project = self.manager.create_project('testproj2', 'test2', 'Another test project', ['test2']) self.assert_(len(self.manager.get_projects()) > 1) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 42cae327f..49147d4ec 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -25,6 +25,7 @@ from nova import test from nova import utils from nova.auth import manager from nova.network import model +from nova.network import networkdata from nova.network import service from nova.network.exception import NoMoreAddresses @@ -154,6 +155,20 @@ class NetworkTestCase(test.TrialTestCase): rv = self.service.deallocate_fixed_ip(firstaddress) self.dnsmasq.release_ip(mac, firstaddress, hostname, net.bridge_name) + def test_212_vpn_ip_and_port_looks_valid(self): + networkdata.NetworkData.create(self.projects[0].id) + self.assert_(self.projects[0].vpn_ip) + self.assert_(self.projects[0].vpn_port >= FLAGS.vpn_start_port) + self.assert_(self.projects[0].vpn_port <= FLAGS.vpn_end_port) + + def test_too_many_vpns(self): + vpns = [] + for i in xrange(networkdata.NetworkData.num_ports_for_ip(FLAGS.vpn_ip)): + vpns.append(networkdata.NetworkData.create("vpnuser%s" % i)) + self.assertRaises(networkdata.NoMorePorts, networkdata.NetworkData.create, "boom") + for vpn in vpns: + vpn.destroy() + def test_release_before_deallocate(self): pass -- cgit From cc64a872c685b931bf76e2323986b427cad777c3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 18:12:19 -0700 Subject: method is called set_network_host --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 349007efa..00dabba28 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -519,7 +519,7 @@ class CloudController(object): host = network_service.get_host_for_project(context.project.id) if not host: result = yield rpc.call(FLAGS.network_topic, - {"method": "get_network_topic", + {"method": "set_network_host", "args": {"user_id": context.user.id, "project_id": context.project.id}}) host = result['result'] -- cgit From d1709793045de2f77f4a1fb06f63d27cbcf640d1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 4 Aug 2010 18:37:00 -0700 Subject: clean up nova-manage. If vpn data isn't set for user it skips it --- bin/nova-manage | 23 +++++++++++------------ nova/auth/manager.py | 39 ++++++++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b0f0029ed..7835c7a77 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -29,16 +29,12 @@ from nova import flags from nova import utils from nova.auth import manager from nova.compute import model -from nova.compute import network from nova.cloudpipe import pipelib from nova.endpoint import cloud FLAGS = flags.FLAGS -class NetworkCommands(object): - def restart(self): - network.restart_nets() class VpnCommands(object): def __init__(self): @@ -170,6 +166,13 @@ class ProjectCommands(object): arguments: name""" self.manager.delete_project(name) + def environment(self, project_id, user_id, filename='novarc'): + """exports environment variables to an sourcable file + arguments: project_id user_id [filename='novarc]""" + rc = self.manager.get_environment_rc(project_id, user_id) + with open(filename, 'w') as f: + f.write(rc) + def list(self): """lists all projects arguments: """ @@ -182,14 +185,11 @@ class ProjectCommands(object): self.manager.remove_from_project(user, project) def zip(self, project_id, user_id, filename='nova.zip'): - """exports credentials for user to a zip file + """exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - project = self.manager.get_project(project_id) - if project: - with open(filename, 'w') as f: - f.write(project.get_credentials(user_id)) - else: - print "Project %s doesn't exist" % project + zip = self.manager.get_credentials(project_id, user_id) + with open(filename, 'w') as f: + f.write(zip) def usage(script_name): @@ -197,7 +197,6 @@ def usage(script_name): categories = [ - ('network', NetworkCommands), ('user', UserCommands), ('project', ProjectCommands), ('role', RoleCommands), diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 463cfdf4a..312b569aa 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -58,6 +58,8 @@ flags.DEFINE_string('credentials_template', flags.DEFINE_string('vpn_client_template', utils.abspath('cloudpipe/client.ovpn.template'), 'Template for creating users vpn file') +flags.DEFINE_string('credential_vpn_file', 'nova-vpn.conf', + 'Filename of certificate in credentials zip') flags.DEFINE_string('credential_key_file', 'pk.pem', 'Filename of private key in credentials zip') flags.DEFINE_string('credential_cert_file', 'cert.pem', @@ -663,25 +665,27 @@ class AuthManager(object): rc = self.__generate_rc(user.access, user.secret, pid) private_key, signed_cert = self._generate_x509_cert(user.id, pid) - vpn = Vpn.lookup(pid) - if not vpn: - raise exception.Error("No vpn data allocated for project %s" % - project.name) - configfile = open(FLAGS.vpn_client_template,"r") - s = string.Template(configfile.read()) - configfile.close() - config = s.substitute(keyfile=FLAGS.credential_key_file, - certfile=FLAGS.credential_cert_file, - ip=vpn.ip, - port=vpn.port) - tmpdir = tempfile.mkdtemp() zf = os.path.join(tmpdir, "temp.zip") zippy = zipfile.ZipFile(zf, 'w') zippy.writestr(FLAGS.credential_rc_file, rc) zippy.writestr(FLAGS.credential_key_file, private_key) zippy.writestr(FLAGS.credential_cert_file, signed_cert) - zippy.writestr("nebula-client.conf", config) + + network_data = networkdata.NetworkData.lookup(pid) + if network_data: + configfile = open(FLAGS.vpn_client_template,"r") + s = string.Template(configfile.read()) + configfile.close() + config = s.substitute(keyfile=FLAGS.credential_key_file, + certfile=FLAGS.credential_cert_file, + ip=network_data.ip, + port=network_data.port) + zippy.writestr(FLAGS.credential_vpn_file, config) + else: + logging.warn("No vpn data for project %s" % + pid) + zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(user.id)) zippy.close() with open(zf, 'rb') as f: @@ -690,6 +694,15 @@ class AuthManager(object): shutil.rmtree(tmpdir) return buffer + def get_environment_rc(self, user, project=None): + """Get credential zip for user in project""" + if not isinstance(user, User): + user = self.get_user(user) + if project is None: + project = user.id + pid = Project.safe_id(project) + return self.__generate_rc(user.access, user.secret, pid) + def __generate_rc(self, access, secret, pid): """Generate rc file for user""" rc = open(FLAGS.credentials_template).read() -- cgit From d79fd0df0bf9c59483b30c0d8c3a811580a1ee39 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 5 Aug 2010 04:31:21 -0700 Subject: Changed volumes to use a pool instead of globbing filesystem for concurrency reasons. Fixed broken tests. --- nova/tests/volume_unittest.py | 77 +++++++++++++++++++++++-------- nova/volume/service.py | 102 +++++++++++++++++++++++------------------- 2 files changed, 114 insertions(+), 65 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 0f4f0e34d..2a07afe69 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -17,6 +17,10 @@ # under the License. import logging +import shutil +import tempfile + +from twisted.internet import defer from nova import compute from nova import exception @@ -34,10 +38,16 @@ class VolumeTestCase(test.TrialTestCase): super(VolumeTestCase, self).setUp() self.compute = compute.service.ComputeService() self.volume = None + self.tempdir = tempfile.mkdtemp() self.flags(connection_type='fake', - fake_storage=True) + fake_storage=True, + aoe_export_dir=self.tempdir) self.volume = volume_service.VolumeService() + def tearDown(self): + shutil.rmtree(self.tempdir) + + @defer.inlineCallbacks def test_run_create_volume(self): vol_size = '0' user_id = 'fake' @@ -48,34 +58,40 @@ class VolumeTestCase(test.TrialTestCase): volume_service.get_volume(volume_id)['volume_id']) rv = self.volume.delete_volume(volume_id) - self.assertFailure(volume_service.get_volume(volume_id), - exception.Error) + self.assertRaises(exception.Error, volume_service.get_volume, volume_id) + @defer.inlineCallbacks def test_too_big_volume(self): vol_size = '1001' user_id = 'fake' project_id = 'fake' - self.assertRaises(TypeError, - self.volume.create_volume, - vol_size, user_id, project_id) + try: + yield self.volume.create_volume(vol_size, user_id, project_id) + self.fail("Should have thrown TypeError") + except TypeError: + pass + @defer.inlineCallbacks def test_too_many_volumes(self): vol_size = '1' user_id = 'fake' project_id = 'fake' num_shelves = FLAGS.last_shelf_id - FLAGS.first_shelf_id + 1 - total_slots = FLAGS.slots_per_shelf * num_shelves + total_slots = FLAGS.blades_per_shelf * num_shelves vols = [] + from nova import datastore + redis = datastore.Redis.instance() for i in xrange(total_slots): vid = yield self.volume.create_volume(vol_size, user_id, project_id) vols.append(vid) self.assertFailure(self.volume.create_volume(vol_size, user_id, project_id), - volume_service.NoMoreVolumes) + volume_service.NoMoreBlades) for id in vols: yield self.volume.delete_volume(id) + @defer.inlineCallbacks def test_run_attach_detach_volume(self): # Create one volume and one compute to test with instance_id = "storage-test" @@ -84,22 +100,26 @@ class VolumeTestCase(test.TrialTestCase): project_id = 'fake' mountpoint = "/dev/sdf" volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) - volume_obj = volume_service.get_volume(volume_id) volume_obj.start_attach(instance_id, mountpoint) - rv = yield self.compute.attach_volume(volume_id, - instance_id, - mountpoint) + if FLAGS.fake_tests: + volume_obj.finish_attach() + else: + rv = yield self.compute.attach_volume(instance_id, + volume_id, + mountpoint) self.assertEqual(volume_obj['status'], "in-use") - self.assertEqual(volume_obj['attachStatus'], "attached") + self.assertEqual(volume_obj['attach_status'], "attached") self.assertEqual(volume_obj['instance_id'], instance_id) self.assertEqual(volume_obj['mountpoint'], mountpoint) - self.assertRaises(exception.Error, - self.volume.delete_volume, - volume_id) - - rv = yield self.volume.detach_volume(volume_id) + self.assertFailure(self.volume.delete_volume(volume_id), exception.Error) + volume_obj.start_detach() + if FLAGS.fake_tests: + volume_obj.finish_detach() + else: + rv = yield self.volume.detach_volume(instance_id, + volume_id) volume_obj = volume_service.get_volume(volume_id) self.assertEqual(volume_obj['status'], "available") @@ -108,6 +128,27 @@ class VolumeTestCase(test.TrialTestCase): volume_service.get_volume, volume_id) + @defer.inlineCallbacks + def test_multiple_volume_race_condition(self): + vol_size = "5" + user_id = "fake" + project_id = 'fake' + shelf_blades = [] + def _check(volume_id): + vol = volume_service.get_volume(volume_id) + shelf_blade = '%s.%s' % (vol['shelf_id'], vol['blade_id']) + self.assert_(shelf_blade not in shelf_blades) + shelf_blades.append(shelf_blade) + logging.debug("got %s" % shelf_blade) + vol.destroy() + deferreds = [] + for i in range(5): + d = self.volume.create_volume(vol_size, user_id, project_id) + d.addCallback(_check) + d.addErrback(self.fail) + deferreds.append(d) + yield defer.DeferredList(deferreds) + def test_multi_node(self): # TODO(termie): Figure out how to test with two nodes, # each of them having a different FLAG for storage_node diff --git a/nova/volume/service.py b/nova/volume/service.py index 9dd63e88f..9c52ee469 100644 --- a/nova/volume/service.py +++ b/nova/volume/service.py @@ -22,12 +22,8 @@ destroying persistent storage volumes, ala EBS. Currently uses Ata-over-Ethernet. """ -import glob import logging import os -import shutil -import socket -import tempfile from twisted.internet import defer @@ -47,9 +43,6 @@ flags.DEFINE_string('volume_group', 'nova-volumes', 'Name for the VG that will contain exported volumes') flags.DEFINE_string('aoe_eth_dev', 'eth0', 'Which device to export the volumes on') -flags.DEFINE_string('storage_name', - socket.gethostname(), - 'name of this service') flags.DEFINE_integer('first_shelf_id', utils.last_octet(utils.get_my_ip()) * 10, 'AoE starting shelf_id for this service') @@ -59,9 +52,9 @@ flags.DEFINE_integer('last_shelf_id', flags.DEFINE_string('aoe_export_dir', '/var/lib/vblade-persist/vblades', 'AoE directory where exports are created') -flags.DEFINE_integer('slots_per_shelf', +flags.DEFINE_integer('blades_per_shelf', 16, - 'Number of AoE slots per shelf') + 'Number of AoE blades per shelf') flags.DEFINE_string('storage_availability_zone', 'nova', 'availability zone of this service') @@ -69,7 +62,7 @@ flags.DEFINE_boolean('fake_storage', False, 'Should we make real storage volumes to attach?') -class NoMoreVolumes(exception.Error): +class NoMoreBlades(exception.Error): pass def get_volume(volume_id): @@ -77,8 +70,9 @@ def get_volume(volume_id): volume_class = Volume if FLAGS.fake_storage: volume_class = FakeVolume - if datastore.Redis.instance().sismember('volumes', volume_id): - return volume_class(volume_id=volume_id) + vol = volume_class.lookup(volume_id) + if vol: + return vol raise exception.Error("Volume does not exist") class VolumeService(service.Service): @@ -91,18 +85,9 @@ class VolumeService(service.Service): super(VolumeService, self).__init__() self.volume_class = Volume if FLAGS.fake_storage: - FLAGS.aoe_export_dir = tempfile.mkdtemp() self.volume_class = FakeVolume self._init_volume_group() - def __del__(self): - # TODO(josh): Get rid of this destructor, volumes destroy themselves - if FLAGS.fake_storage: - try: - shutil.rmtree(FLAGS.aoe_export_dir) - except Exception, err: - pass - @defer.inlineCallbacks @validate.rangetest(size=(0, 1000)) def create_volume(self, size, user_id, project_id): @@ -113,8 +98,6 @@ class VolumeService(service.Service): """ logging.debug("Creating volume of size: %s" % (size)) vol = yield self.volume_class.create(size, user_id, project_id) - datastore.Redis.instance().sadd('volumes', vol['volume_id']) - datastore.Redis.instance().sadd('volumes:%s' % (FLAGS.storage_name), vol['volume_id']) logging.debug("restarting exports") yield self._restart_exports() defer.returnValue(vol['volume_id']) @@ -134,13 +117,11 @@ class VolumeService(service.Service): def delete_volume(self, volume_id): logging.debug("Deleting volume with id of: %s" % (volume_id)) vol = get_volume(volume_id) - if vol['status'] == "attached": + if vol['attach_status'] == "attached": raise exception.Error("Volume is still attached") - if vol['node_name'] != FLAGS.storage_name: + if vol['node_name'] != FLAGS.node_name: raise exception.Error("Volume is not local to this node") yield vol.destroy() - datastore.Redis.instance().srem('volumes', vol['volume_id']) - datastore.Redis.instance().srem('volumes:%s' % (FLAGS.storage_name), vol['volume_id']) defer.returnValue(True) @defer.inlineCallbacks @@ -172,14 +153,15 @@ class Volume(datastore.BasicModel): return self.volume_id def default_state(self): - return {"volume_id": self.volume_id} + return {"volume_id": self.volume_id, + "node_name": "unassigned"} @classmethod @defer.inlineCallbacks def create(cls, size, user_id, project_id): volume_id = utils.generate_uid('vol') vol = cls(volume_id) - vol['node_name'] = FLAGS.storage_name + vol['node_name'] = FLAGS.node_name vol['size'] = size vol['user_id'] = user_id vol['project_id'] = project_id @@ -225,10 +207,31 @@ class Volume(datastore.BasicModel): self['attach_status'] = "detached" self.save() + def save(self): + is_new = self.is_new_record() + super(Volume, self).save() + if is_new: + redis = datastore.Redis.instance() + key = self.__devices_key + # TODO(vish): these should be added by admin commands + more = redis.scard(self._redis_association_name("node", + self['node_name'])) + if (not redis.exists(key) and not more): + for shelf_id in range(FLAGS.first_shelf_id, + FLAGS.last_shelf_id + 1): + for blade_id in range(FLAGS.blades_per_shelf): + redis.sadd(key, "%s.%s" % (shelf_id, blade_id)) + self.associate_with("node", self['node_name']) + @defer.inlineCallbacks def destroy(self): yield self._remove_export() yield self._delete_lv() + self.unassociate_with("node", self['node_name']) + if self.get('shelf_id', None) and self.get('blade_id', None): + redis = datastore.Redis.instance() + key = self.__devices_key + redis.sadd(key, "%s.%s" % (self['shelf_id'], self['blade_id'])) super(Volume, self).destroy() @defer.inlineCallbacks @@ -248,17 +251,26 @@ class Volume(datastore.BasicModel): "sudo lvremove -f %s/%s" % (FLAGS.volume_group, self['volume_id']), error_ok=1) + @property + def __devices_key(self): + return 'volume_devices:%s' % FLAGS.node_name + @defer.inlineCallbacks def _setup_export(self): - (shelf_id, blade_id) = get_next_aoe_numbers() + redis = datastore.Redis.instance() + key = self.__devices_key + device = redis.spop(key) + if not device: + raise NoMoreBlades() + (shelf_id, blade_id) = device.split('.') self['aoe_device'] = "e%s.%s" % (shelf_id, blade_id) self['shelf_id'] = shelf_id self['blade_id'] = blade_id self.save() - yield self._exec_export() + yield self._exec_setup_export() @defer.inlineCallbacks - def _exec_export(self): + def _exec_setup_export(self): yield process.simple_execute( "sudo vblade-persist setup %s %s %s /dev/%s/%s" % (self['shelf_id'], @@ -269,6 +281,13 @@ class Volume(datastore.BasicModel): @defer.inlineCallbacks def _remove_export(self): + if not self.get('shelf_id', None) or not self.get('blade_id', None): + defer.returnValue(False) + yield self._exec_remove_export() + defer.returnValue(True) + + @defer.inlineCallbacks + def _exec_remove_export(self): yield process.simple_execute( "sudo vblade-persist stop %s %s" % (self['shelf_id'], self['blade_id']), error_ok=1) @@ -277,29 +296,18 @@ class Volume(datastore.BasicModel): self['blade_id']), error_ok=1) + class FakeVolume(Volume): def _create_lv(self): pass - def _exec_export(self): + def _exec_setup_export(self): fname = os.path.join(FLAGS.aoe_export_dir, self['aoe_device']) f = file(fname, "w") f.close() - def _remove_export(self): - pass + def _exec_remove_export(self): + os.unlink(os.path.join(FLAGS.aoe_export_dir, self['aoe_device'])) def _delete_lv(self): pass - -def get_next_aoe_numbers(): - for shelf_id in xrange(FLAGS.first_shelf_id, FLAGS.last_shelf_id + 1): - aoes = glob.glob("%s/e%s.*" % (FLAGS.aoe_export_dir, shelf_id)) - if not aoes: - blade_id = 0 - else: - blade_id = int(max([int(a.rpartition('.')[2]) for a in aoes])) + 1 - if blade_id < FLAGS.slots_per_shelf: - logging.debug("Next shelf.blade is %s.%s", shelf_id, blade_id) - return (shelf_id, blade_id) - raise NoMoreVolumes() -- cgit From 024ad9951dcf33f5a3468e9a790f1636770b2837 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 5 Aug 2010 12:29:50 -0700 Subject: rename networkdata to vpn --- nova/auth/manager.py | 6 +-- nova/network/networkdata.py | 116 ----------------------------------------- nova/network/service.py | 4 +- nova/network/vpn.py | 116 +++++++++++++++++++++++++++++++++++++++++ nova/tests/network_unittest.py | 10 ++-- 5 files changed, 126 insertions(+), 126 deletions(-) delete mode 100644 nova/network/networkdata.py create mode 100644 nova/network/vpn.py diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 312b569aa..cf920d607 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -37,7 +37,7 @@ from nova import objectstore # for flags from nova import utils from nova.auth import ldapdriver # for flags from nova.auth import signer -from nova.network import networkdata +from nova.network import vpn FLAGS = flags.FLAGS @@ -531,7 +531,7 @@ class AuthManager(object): @return: A tuple containing (ip, port) or None, None if vpn has not been allocated for user. """ - network_data = networkdata.NetworkData.lookup(Project.safe_id(project)) + network_data = vpn.NetworkData.lookup(Project.safe_id(project)) if not network_data: raise exception.NotFound('project network data has not been set') return (network_data.ip, network_data.port) @@ -672,7 +672,7 @@ class AuthManager(object): zippy.writestr(FLAGS.credential_key_file, private_key) zippy.writestr(FLAGS.credential_cert_file, signed_cert) - network_data = networkdata.NetworkData.lookup(pid) + network_data = vpn.NetworkData.lookup(pid) if network_data: configfile = open(FLAGS.vpn_client_template,"r") s = string.Template(configfile.read()) diff --git a/nova/network/networkdata.py b/nova/network/networkdata.py deleted file mode 100644 index cec84287c..000000000 --- a/nova/network/networkdata.py +++ /dev/null @@ -1,116 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Network Data for projects""" - -from nova import datastore -from nova import exception -from nova import flags -from nova import utils - -FLAGS = flags.FLAGS - - -flags.DEFINE_string('vpn_ip', utils.get_my_ip(), - 'Public IP for the cloudpipe VPN servers') -flags.DEFINE_integer('vpn_start_port', 1000, - 'Start port for the cloudpipe VPN servers') -flags.DEFINE_integer('vpn_end_port', 2000, - 'End port for the cloudpipe VPN servers') - -class NoMorePorts(exception.Error): - pass - - -class NetworkData(datastore.BasicModel): - """Manages network host, and vpn ip and port for projects""" - def __init__(self, project_id): - self.project_id = project_id - super(NetworkData, self).__init__() - - @property - def identifier(self): - """Identifier used for key in redis""" - return self.project_id - - @classmethod - def create(cls, project_id): - """Creates a vpn for project - - This method finds a free ip and port and stores the associated - values in the datastore. - """ - # TODO(vish): will we ever need multiiple ips per host? - port = cls.find_free_port_for_ip(FLAGS.vpn_ip) - network_data = cls(project_id) - # save ip for project - network_data['host'] = FLAGS.node_name - network_data['project'] = project_id - network_data['ip'] = FLAGS.vpn_ip - network_data['port'] = port - network_data.save() - return network_data - - @classmethod - def find_free_port_for_ip(cls, ip): - """Finds a free port for a given ip from the redis set""" - # TODO(vish): these redis commands should be generalized and - # placed into a base class. Conceptually, it is - # similar to an association, but we are just - # storing a set of values instead of keys that - # should be turned into objects. - redis = datastore.Redis.instance() - key = 'ip:%s:ports' % ip - # TODO(vish): these ports should be allocated through an admin - # command instead of a flag - if (not redis.exists(key) and - not redis.exists(cls._redis_association_name('ip', ip))): - for i in range(FLAGS.vpn_start_port, FLAGS.vpn_end_port + 1): - redis.sadd(key, i) - - port = redis.spop(key) - if not port: - raise NoMorePorts() - return port - - @classmethod - def num_ports_for_ip(cls, ip): - """Calculates the number of free ports for a given ip""" - return datastore.Redis.instance().scard('ip:%s:ports' % ip) - - @property - def ip(self): - """The ip assigned to the project""" - return self['ip'] - - @property - def port(self): - """The port assigned to the project""" - return int(self['port']) - - def save(self): - """Saves the association to the given ip""" - self.associate_with('ip', self.ip) - super(NetworkData, self).save() - - def destroy(self): - """Cleans up datastore and adds port back to pool""" - self.unassociate_with('ip', self.ip) - datastore.Redis.instance().sadd('ip:%s:ports' % self.ip, self.port) - super(NetworkData, self).destroy() - diff --git a/nova/network/service.py b/nova/network/service.py index afc20c0d5..1a61f49d4 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -28,7 +28,7 @@ from nova.auth import manager from nova.exception import NotFound from nova.network import exception from nova.network import model -from nova.network import networkdata +from nova.network import vpn FLAGS = flags.FLAGS @@ -215,7 +215,7 @@ class VlanNetworkService(BaseNetworkService): def _on_set_network_host(self, user_id, project_id, *args, **kwargs): """Called when this host becomes the host for a project""" - networkdata.NetworkData.create(project_id) + vpn.NetworkData.create(project_id) @classmethod def setup_compute_network(self, user_id, project_id, security_group, diff --git a/nova/network/vpn.py b/nova/network/vpn.py new file mode 100644 index 000000000..cec84287c --- /dev/null +++ b/nova/network/vpn.py @@ -0,0 +1,116 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Network Data for projects""" + +from nova import datastore +from nova import exception +from nova import flags +from nova import utils + +FLAGS = flags.FLAGS + + +flags.DEFINE_string('vpn_ip', utils.get_my_ip(), + 'Public IP for the cloudpipe VPN servers') +flags.DEFINE_integer('vpn_start_port', 1000, + 'Start port for the cloudpipe VPN servers') +flags.DEFINE_integer('vpn_end_port', 2000, + 'End port for the cloudpipe VPN servers') + +class NoMorePorts(exception.Error): + pass + + +class NetworkData(datastore.BasicModel): + """Manages network host, and vpn ip and port for projects""" + def __init__(self, project_id): + self.project_id = project_id + super(NetworkData, self).__init__() + + @property + def identifier(self): + """Identifier used for key in redis""" + return self.project_id + + @classmethod + def create(cls, project_id): + """Creates a vpn for project + + This method finds a free ip and port and stores the associated + values in the datastore. + """ + # TODO(vish): will we ever need multiiple ips per host? + port = cls.find_free_port_for_ip(FLAGS.vpn_ip) + network_data = cls(project_id) + # save ip for project + network_data['host'] = FLAGS.node_name + network_data['project'] = project_id + network_data['ip'] = FLAGS.vpn_ip + network_data['port'] = port + network_data.save() + return network_data + + @classmethod + def find_free_port_for_ip(cls, ip): + """Finds a free port for a given ip from the redis set""" + # TODO(vish): these redis commands should be generalized and + # placed into a base class. Conceptually, it is + # similar to an association, but we are just + # storing a set of values instead of keys that + # should be turned into objects. + redis = datastore.Redis.instance() + key = 'ip:%s:ports' % ip + # TODO(vish): these ports should be allocated through an admin + # command instead of a flag + if (not redis.exists(key) and + not redis.exists(cls._redis_association_name('ip', ip))): + for i in range(FLAGS.vpn_start_port, FLAGS.vpn_end_port + 1): + redis.sadd(key, i) + + port = redis.spop(key) + if not port: + raise NoMorePorts() + return port + + @classmethod + def num_ports_for_ip(cls, ip): + """Calculates the number of free ports for a given ip""" + return datastore.Redis.instance().scard('ip:%s:ports' % ip) + + @property + def ip(self): + """The ip assigned to the project""" + return self['ip'] + + @property + def port(self): + """The port assigned to the project""" + return int(self['port']) + + def save(self): + """Saves the association to the given ip""" + self.associate_with('ip', self.ip) + super(NetworkData, self).save() + + def destroy(self): + """Cleans up datastore and adds port back to pool""" + self.unassociate_with('ip', self.ip) + datastore.Redis.instance().sadd('ip:%s:ports' % self.ip, self.port) + super(NetworkData, self).destroy() + diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 49147d4ec..c7d3e86f0 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -25,8 +25,8 @@ from nova import test from nova import utils from nova.auth import manager from nova.network import model -from nova.network import networkdata from nova.network import service +from nova.network import vpn from nova.network.exception import NoMoreAddresses FLAGS = flags.FLAGS @@ -156,16 +156,16 @@ class NetworkTestCase(test.TrialTestCase): self.dnsmasq.release_ip(mac, firstaddress, hostname, net.bridge_name) def test_212_vpn_ip_and_port_looks_valid(self): - networkdata.NetworkData.create(self.projects[0].id) + vpn.NetworkData.create(self.projects[0].id) self.assert_(self.projects[0].vpn_ip) self.assert_(self.projects[0].vpn_port >= FLAGS.vpn_start_port) self.assert_(self.projects[0].vpn_port <= FLAGS.vpn_end_port) def test_too_many_vpns(self): vpns = [] - for i in xrange(networkdata.NetworkData.num_ports_for_ip(FLAGS.vpn_ip)): - vpns.append(networkdata.NetworkData.create("vpnuser%s" % i)) - self.assertRaises(networkdata.NoMorePorts, networkdata.NetworkData.create, "boom") + for i in xrange(vpn.NetworkData.num_ports_for_ip(FLAGS.vpn_ip)): + vpns.append(vpn.NetworkData.create("vpnuser%s" % i)) + self.assertRaises(vpn.NoMorePorts, vpn.NetworkData.create, "boom") for vpn in vpns: vpn.destroy() -- cgit From b77d261b0288f89d2b25a52e7ad7c8073e357cb1 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 5 Aug 2010 13:51:44 -0700 Subject: First pass at making a file pass pep8 and pylint tests as an example. --- nova/tests/objectstore_unittest.py | 237 +++++++++++++++++++++---------------- pylintrc | 6 + 2 files changed, 142 insertions(+), 101 deletions(-) create mode 100644 pylintrc diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index dd00377e7..dece4b5d5 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -16,6 +16,10 @@ # License for the specific language governing permissions and limitations # under the License. +""" +Unittets for S3 objectstore clone. +""" + import boto import glob import hashlib @@ -24,76 +28,69 @@ import os import shutil import tempfile +from boto.s3.connection import S3Connection, OrdinaryCallingFormat +from twisted.internet import reactor, threads, defer +from twisted.web import http, server + from nova import flags from nova import objectstore -from nova.objectstore import bucket # for buckets_path flag -from nova.objectstore import image # for images_path flag from nova import test from nova.auth import manager +from nova.exception import NotEmpty, NotFound +from nova.objectstore import image from nova.objectstore.handler import S3 -from nova.exception import NotEmpty, NotFound, NotAuthorized -from boto.s3.connection import S3Connection, OrdinaryCallingFormat -from twisted.internet import reactor, threads, defer -from twisted.web import http, server FLAGS = flags.FLAGS -oss_tempdir = tempfile.mkdtemp(prefix='test_oss-') - +# Create a unique temporary directory. We don't delete after test to +# allow checking the contents after running tests. Users and/or tools +# running the tests need to remove the tests directories. +OSS_TEMPDIR = tempfile.mkdtemp(prefix='test_oss-') -# delete tempdirs from previous runs (we don't delete after test to allow -# checking the contents after running tests) -# TODO: This fails on the test box with a permission denied error -# Also, doing these things in a global tempdir means that different runs of -# the test suite on the same box could clobber each other. -#for path in glob.glob(os.path.abspath(os.path.join(oss_tempdir, '../test_oss-*'))): -# if path != oss_tempdir: -# shutil.rmtree(path) +# Create bucket/images path +os.makedirs(os.path.join(OSS_TEMPDIR, 'images')) +os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) -# create bucket/images path -os.makedirs(os.path.join(oss_tempdir, 'images')) -os.makedirs(os.path.join(oss_tempdir, 'buckets')) - class ObjectStoreTestCase(test.BaseTestCase): - def setUp(self): + """Test objectstore API directly.""" + + def setUp(self): # pylint: disable-msg=C0103 + """Setup users and projects.""" super(ObjectStoreTestCase, self).setUp() - self.flags(buckets_path=os.path.join(oss_tempdir, 'buckets'), - images_path=os.path.join(oss_tempdir, 'images'), + self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), + images_path=os.path.join(OSS_TEMPDIR, 'images'), ca_path=os.path.join(os.path.dirname(__file__), 'CA')) logging.getLogger().setLevel(logging.DEBUG) - self.um = manager.AuthManager() - try: - self.um.create_user('user1') - except: pass - try: - self.um.create_user('user2') - except: pass - try: - self.um.create_user('admin_user', admin=True) - except: pass - try: - self.um.create_project('proj1', 'user1', 'a proj', ['user1']) - except: pass - try: - self.um.create_project('proj2', 'user2', 'a proj', ['user2']) - except: pass - class Context(object): pass + self.auth_manager = manager.AuthManager() + self.auth_manager.create_user('user1') + self.auth_manager.create_user('user2') + self.auth_manager.create_user('admin_user', admin=True) + self.auth_manager.create_project('proj1', 'user1', 'a proj', ['user1']) + self.auth_manager.create_project('proj2', 'user2', 'a proj', ['user2']) + + class Context(object): + """Dummy context for running tests.""" + user = None + project = None + self.context = Context() - def tearDown(self): - self.um.delete_project('proj1') - self.um.delete_project('proj2') - self.um.delete_user('user1') - self.um.delete_user('user2') - self.um.delete_user('admin_user') + def tearDown(self): # pylint: disable-msg=C0103 + """Tear down users and projects.""" + self.auth_manager.delete_project('proj1') + self.auth_manager.delete_project('proj2') + self.auth_manager.delete_user('user1') + self.auth_manager.delete_user('user2') + self.auth_manager.delete_user('admin_user') super(ObjectStoreTestCase, self).tearDown() def test_buckets(self): - self.context.user = self.um.get_user('user1') - self.context.project = self.um.get_project('proj1') + """Test the bucket API.""" + self.context.user = self.auth_manager.get_user('user1') + self.context.project = self.auth_manager.get_project('proj1') objectstore.bucket.Bucket.create('new_bucket', self.context) bucket = objectstore.bucket.Bucket('new_bucket') @@ -101,12 +98,12 @@ class ObjectStoreTestCase(test.BaseTestCase): self.assert_(bucket.is_authorized(self.context)) # another user is not authorized - self.context.user = self.um.get_user('user2') - self.context.project = self.um.get_project('proj2') + self.context.user = self.auth_manager.get_user('user2') + self.context.project = self.auth_manager.get_project('proj2') self.assertFalse(bucket.is_authorized(self.context)) # admin is authorized to use bucket - self.context.user = self.um.get_user('admin_user') + self.context.user = self.auth_manager.get_user('admin_user') self.context.project = None self.assertTrue(bucket.is_authorized(self.context)) @@ -136,8 +133,9 @@ class ObjectStoreTestCase(test.BaseTestCase): self.assertRaises(NotFound, objectstore.bucket.Bucket, 'new_bucket') def test_images(self): - self.context.user = self.um.get_user('user1') - self.context.project = self.um.get_project('proj1') + "Test the image API." + self.context.user = self.auth_manager.get_user('user1') + self.context.project = self.auth_manager.get_project('proj1') # create a bucket for our bundle objectstore.bucket.Bucket.create('image_bucket', self.context) @@ -149,10 +147,12 @@ class ObjectStoreTestCase(test.BaseTestCase): bucket[os.path.basename(path)] = open(path, 'rb').read() # register an image - objectstore.image.Image.register_aws_image('i-testing', 'image_bucket/1mb.manifest.xml', self.context) + image.Image.register_aws_image('i-testing', + 'image_bucket/1mb.manifest.xml', + self.context) # verify image - my_img = objectstore.image.Image('i-testing') + my_img = image.Image('i-testing') result_image_file = os.path.join(my_img.path, 'image') self.assertEqual(os.stat(result_image_file).st_size, 1048576) @@ -160,38 +160,48 @@ class ObjectStoreTestCase(test.BaseTestCase): self.assertEqual(sha, '3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3') # verify image permissions - self.context.user = self.um.get_user('user2') - self.context.project = self.um.get_project('proj2') + self.context.user = self.auth_manager.get_user('user2') + self.context.project = self.auth_manager.get_project('proj2') self.assertFalse(my_img.is_authorized(self.context)) class TestHTTPChannel(http.HTTPChannel): - # Otherwise we end up with an unclean reactor - def checkPersistence(self, _, __): + """Dummy site required for twisted.web""" + + def checkPersistence(self, _, __): # pylint: disable-msg=C0103 + """Otherwise we end up with an unclean reactor.""" return False class TestSite(server.Site): + """Dummy site required for twisted.web""" protocol = TestHTTPChannel class S3APITestCase(test.TrialTestCase): - def setUp(self): + """Test objectstore through S3 API.""" + + def setUp(self): # pylint: disable-msg=C0103 + """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() - FLAGS.auth_driver='nova.auth.ldapdriver.FakeLdapDriver', - FLAGS.buckets_path = os.path.join(oss_tempdir, 'buckets') + FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver', + FLAGS.buckets_path = os.path.join(OSS_TEMPDIR, 'buckets') - self.um = manager.AuthManager() - self.admin_user = self.um.create_user('admin', admin=True) - self.admin_project = self.um.create_project('admin', self.admin_user) + self.auth_manager = manager.AuthManager() + self.admin_user = self.auth_manager.create_user('admin', admin=True) + self.admin_project = self.auth_manager.create_project('admin', + self.admin_user) shutil.rmtree(FLAGS.buckets_path) os.mkdir(FLAGS.buckets_path) root = S3() self.site = TestSite(root) - self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1') + # pylint: disable-msg=E1101 + self.listening_port = reactor.listenTCP(0, self.site, + interface='127.0.0.1') + # pylint: enable-msg=E1101 self.tcp_port = self.listening_port.getHost().port @@ -205,65 +215,90 @@ class S3APITestCase(test.TrialTestCase): is_secure=False, calling_format=OrdinaryCallingFormat()) - # Don't attempt to reuse connections def get_http_connection(host, is_secure): + """Get a new S3 connection, don't attempt to reuse connections.""" return self.conn.new_http_connection(host, is_secure) + self.conn.get_http_connection = get_http_connection - def _ensure_empty_list(self, l): - self.assertEquals(len(l), 0, "List was not empty") + def _ensure_no_buckets(self, buckets): # pylint: disable-msg=C0111 + self.assertEquals(len(buckets), 0, "Bucket list was not empty") return True - def _ensure_only_bucket(self, l, name): - self.assertEquals(len(l), 1, "List didn't have exactly one element in it") - self.assertEquals(l[0].name, name, "Wrong name") + def _ensure_one_bucket(self, buckets, name): # pylint: disable-msg=C0111 + self.assertEquals(len(buckets), 1, + "Bucket list didn't have exactly one element in it") + self.assertEquals(buckets[0].name, name, "Wrong name") + return True def test_000_list_buckets(self): - d = threads.deferToThread(self.conn.get_all_buckets) - d.addCallback(self._ensure_empty_list) - return d + """Make sure we are starting with no buckets.""" + deferred = threads.deferToThread(self.conn.get_all_buckets) + deferred.addCallback(self._ensure_no_buckets) + return deferred def test_001_create_and_delete_bucket(self): + """Test bucket creation and deletion.""" bucket_name = 'testbucket' - d = threads.deferToThread(self.conn.create_bucket, bucket_name) - d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets)) + deferred = threads.deferToThread(self.conn.create_bucket, bucket_name) + deferred.addCallback(lambda _: + threads.deferToThread(self.conn.get_all_buckets)) - def ensure_only_bucket(l, name): - self.assertEquals(len(l), 1, "List didn't have exactly one element in it") - self.assertEquals(l[0].name, name, "Wrong name") - d.addCallback(ensure_only_bucket, bucket_name) + deferred.addCallback(self._ensure_one_bucket, bucket_name) - d.addCallback(lambda _:threads.deferToThread(self.conn.delete_bucket, bucket_name)) - d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets)) - d.addCallback(self._ensure_empty_list) - return d + deferred.addCallback(lambda _: + threads.deferToThread(self.conn.delete_bucket, + bucket_name)) + deferred.addCallback(lambda _: + threads.deferToThread(self.conn.get_all_buckets)) + deferred.addCallback(self._ensure_no_buckets) + return deferred def test_002_create_bucket_and_key_and_delete_key_again(self): + """Test key operations on buckets.""" bucket_name = 'testbucket' key_name = 'somekey' key_contents = 'somekey' - d = threads.deferToThread(self.conn.create_bucket, bucket_name) - d.addCallback(lambda b:threads.deferToThread(b.new_key, key_name)) - d.addCallback(lambda k:threads.deferToThread(k.set_contents_from_string, key_contents)) + deferred = threads.deferToThread(self.conn.create_bucket, bucket_name) + deferred.addCallback(lambda b: + threads.deferToThread(b.new_key, key_name)) + deferred.addCallback(lambda k: + threads.deferToThread(k.set_contents_from_string, + key_contents)) + def ensure_key_contents(bucket_name, key_name, contents): + """Verify contents for a key in the given bucket.""" bucket = self.conn.get_bucket(bucket_name) key = bucket.get_key(key_name) - self.assertEquals(key.get_contents_as_string(), contents, "Bad contents") - d.addCallback(lambda _:threads.deferToThread(ensure_key_contents, bucket_name, key_name, key_contents)) + self.assertEquals(key.get_contents_as_string(), contents, + "Bad contents") + + deferred.addCallback(lambda _: + threads.deferToThread(ensure_key_contents, + bucket_name, key_name, + key_contents)) + def delete_key(bucket_name, key_name): + """Delete a key for the given bucket.""" bucket = self.conn.get_bucket(bucket_name) key = bucket.get_key(key_name) key.delete() - d.addCallback(lambda _:threads.deferToThread(delete_key, bucket_name, key_name)) - d.addCallback(lambda _:threads.deferToThread(self.conn.get_bucket, bucket_name)) - d.addCallback(lambda b:threads.deferToThread(b.get_all_keys)) - d.addCallback(self._ensure_empty_list) - return d - - def tearDown(self): - self.um.delete_user('admin') - self.um.delete_project('admin') - return defer.DeferredList([defer.maybeDeferred(self.listening_port.stopListening)]) - super(S3APITestCase, self).tearDown() + + deferred.addCallback(lambda _: + threads.deferToThread(delete_key, bucket_name, + key_name)) + deferred.addCallback(lambda _: + threads.deferToThread(self.conn.get_bucket, + bucket_name)) + deferred.addCallback(lambda b: threads.deferToThread(b.get_all_keys)) + deferred.addCallback(self._ensure_no_buckets) + return deferred + + def tearDown(self): # pylint: disable-msg=C0103 + """Tear down auth and test server.""" + self.auth_manager.delete_user('admin') + self.auth_manager.delete_project('admin') + stop_listening = defer.maybeDeferred(self.listening_port.stopListening) + return defer.DeferredList([stop_listening]) diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..a853e5bed --- /dev/null +++ b/pylintrc @@ -0,0 +1,6 @@ +[Basic] +method-rgx=[a-z_][a-z0-9_]{2,50}$ + +[Design] +max-public-methods=100 +min-public-methods=0 -- cgit From 778e8152751ebdbb2adad544cc705691395d335d Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 5 Aug 2010 16:20:26 -0700 Subject: Fixed image modification authorization, API cleanup --- nova/endpoint/cloud.py | 2 ++ nova/objectstore/handler.py | 3 ++- nova/objectstore/image.py | 8 ++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 0ee278f84..cc6216fec 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -677,6 +677,8 @@ class CloudController(object): # TODO(devcamcar): Support users and groups other than 'all'. if attribute != 'launchPermission': raise exception.ApiError('attribute not supported: %s' % attribute) + if not 'user_group' in kwargs: + raise exception.ApiError('user or group not specified') if len(kwargs['user_group']) != 1 and kwargs['user_group'][0] != 'all': raise exception.ApiError('only group "all" is supported') if not operation_type in ['add', 'remove']: diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index b4d7e6179..f625a2aa1 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -266,7 +266,8 @@ class ImagesResource(Resource): """ returns a json listing of all images that a user has permissions to see """ - images = [i for i in image.Image.all() if i.is_authorized(request.context)] + images = [i for i in image.Image.all() \ + if i.is_authorized(request.context, readonly=True)] request.write(json.dumps([i.metadata for i in images])) request.finish() diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index bea2e9637..860298ba6 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -65,9 +65,13 @@ class Image(object): except: pass - def is_authorized(self, context): + def is_authorized(self, context, readonly=False): + # NOTE(devcamcar): Public images can be read by anyone, + # but only modified by admin or owner. try: - return self.metadata['isPublic'] or context.user.is_admin() or self.metadata['imageOwnerId'] == context.project.id + return (self.metadata['isPublic'] and readonly) or \ + context.user.is_admin() or \ + self.metadata['imageOwnerId'] == context.project.id except: return False -- cgit From 63708752366300b4267a9dfbf926f89f9df3f4df Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 18:26:39 -0500 Subject: Fixes bug#614090 -- nova.virt.images._fetch_local_image being called with 4 args but only has 3 --- nova/virt/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/images.py b/nova/virt/images.py index 872eb6d6a..48a87b514 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -65,7 +65,7 @@ def _fetch_s3_image(image, path, user, project): cmd += ['-o', path] return process.SharedPool().execute(executable=cmd[0], args=cmd[1:]) -def _fetch_local_image(image, path, _): +def _fetch_local_image(image, path, user, project): source = _image_path('%s/image' % image) return process.simple_execute('cp %s %s' % (source, path)) -- cgit From 5cda99300a437feefac39131bb714e9f85d765ce Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 5 Aug 2010 16:56:23 -0700 Subject: Made group membership check only search group instead of subtree. Roles in a group are removed when a user is removed from that group. Added test --- nova/auth/fakeldap.py | 11 ++++++++--- nova/auth/ldapdriver.py | 25 +++++++++++++++++-------- nova/tests/auth_unittest.py | 10 +++++++++- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 4c32ed9d9..b420924af 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -28,6 +28,8 @@ import json from nova import datastore +SCOPE_BASE = 0 +SCOPE_ONELEVEL = 1 # not implemented SCOPE_SUBTREE = 2 MOD_ADD = 0 MOD_DELETE = 1 @@ -188,15 +190,18 @@ class FakeLDAP(object): Args: dn -- dn to search under - scope -- only SCOPE_SUBTREE is supported + scope -- only SCOPE_BASE and SCOPE_SUBTREE are supported query -- query to filter objects by fields -- fields to return. Returns all fields if not specified """ - if scope != SCOPE_SUBTREE: + if scope != SCOPE_BASE and scope != SCOPE_SUBTREE: raise NotImplementedError(str(scope)) redis = datastore.Redis.instance() - keys = redis.keys("%s*%s" % (self.__redis_prefix, dn)) + if scope == SCOPE_BASE: + keys = ["%s%s" % (self.__redis_prefix, dn)] + else: + keys = redis.keys("%s*%s" % (self.__redis_prefix, dn)) objects = [] for key in keys: # get the attributes from redis diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 055e8332b..ec739e134 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -272,26 +272,30 @@ class LdapDriver(object): """Check if project exists""" return self.get_project(name) != None - def __find_object(self, dn, query = None): + def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" - objects = self.__find_objects(dn, query) + objects = self.__find_objects(dn, query, scope) if len(objects) == 0: return None return objects[0] - def __find_dns(self, dn, query=None): + def __find_dns(self, dn, query=None, scope=None): """Find dns by query""" + if scope is None: # one of the flags is 0!! + scope = self.ldap.SCOPE_SUBTREE try: - res = self.conn.search_s(dn, self.ldap.SCOPE_SUBTREE, query) + res = self.conn.search_s(dn, scope, query) except self.ldap.NO_SUCH_OBJECT: return [] # just return the DNs return [dn for dn, attributes in res] - def __find_objects(self, dn, query = None): + def __find_objects(self, dn, query=None, scope=None): """Find objects by query""" + if scope is None: # one of the flags is 0!! + scope = self.ldap.SCOPE_SUBTREE try: - res = self.conn.search_s(dn, self.ldap.SCOPE_SUBTREE, query) + res = self.conn.search_s(dn, scope, query) except self.ldap.NO_SUCH_OBJECT: return [] # just return the attributes @@ -361,7 +365,8 @@ class LdapDriver(object): if not self.__group_exists(group_dn): return False res = self.__find_object(group_dn, - '(member=%s)' % self.__uid_to_dn(uid)) + '(member=%s)' % self.__uid_to_dn(uid), + self.ldap.SCOPE_BASE) return res != None def __add_to_group(self, uid, group_dn): @@ -391,7 +396,11 @@ class LdapDriver(object): if not self.__is_in_group(uid, group_dn): raise exception.NotFound("User %s is not a member of the group" % (uid,)) - self.__safe_remove_from_group(uid, group_dn) + # NOTE(vish): remove user from group and any sub_groups + sub_dns = self.__find_group_dns_with_member( + group_dn, uid) + for sub_dn in sub_dns: + self.__safe_remove_from_group(uid, sub_dn) def __safe_remove_from_group(self, uid, group_dn): """Remove user from group, deleting group if user is last member""" diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 2167c2385..e00297cb1 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -135,10 +135,18 @@ class AuthTestCase(test.BaseTestCase): self.manager.add_to_project('test2', 'testproj') self.assertTrue(self.manager.get_project('testproj').has_member('test2')) - def test_208_can_remove_user_from_project(self): + def test_207_can_remove_user_from_project(self): self.manager.remove_from_project('test2', 'testproj') self.assertFalse(self.manager.get_project('testproj').has_member('test2')) + def test_208_can_remove_add_user_with_role(self): + self.manager.add_to_project('test2', 'testproj') + self.manager.add_role('test2', 'developer', 'testproj') + self.manager.remove_from_project('test2', 'testproj') + self.assertFalse(self.manager.has_role('test2', 'developer', 'testproj')) + self.manager.add_to_project('test2', 'testproj') + self.manager.remove_from_project('test2', 'testproj') + def test_209_can_generate_x509(self): # MUST HAVE RUN CLOUD SETUP BY NOW self.cloud = cloud.CloudController() -- cgit From 64e34d8004662879708f69e476881adc9d6ba45b Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 5 Aug 2010 22:54:08 -0400 Subject: Use webob to simplify wsgi middleware --- nova/endpoint/new_wsgi.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 nova/endpoint/new_wsgi.py diff --git a/nova/endpoint/new_wsgi.py b/nova/endpoint/new_wsgi.py new file mode 100644 index 000000000..c7eee20fc --- /dev/null +++ b/nova/endpoint/new_wsgi.py @@ -0,0 +1,122 @@ +import eventlet +import eventlet.wsgi +eventlet.patcher.monkey_patch(all=False, socket=True) +import carrot.connection +import carrot.messaging +import itertools +import routes + +# See http://pythonpaste.org/webob/ for usage +from webob.dec import wsgify +from webob import exc, Request, Response + +class WSGILayer(object): + def __init__(self, application=None): + self.application = application + + def __call__(self, environ, start_response): + # Subclasses will probably want to implement __call__ like this: + # + # @wsgify + # def __call__(self, req): + # # Any of the following objects work as responses: + # + # # Option 1: simple string + # resp = 'message\n' + # + # # Option 2: a nicely formatted HTTP exception page + # resp = exc.HTTPForbidden(detail='Nice try') + # + # # Option 3: a webob Response object (in case you need to play with + # # headers, or you want to be treated like an iterable, or or or) + # resp = Response(); resp.app_iter = open('somefile') + # + # # Option 4: any wsgi app to be run next + # resp = self.application + # + # # Option 5: you can get a Response object for a wsgi app, too, to + # # play with headers etc + # resp = req.get_response(self.application) + # + # + # # You can then just return your response... + # return resp # option 1 + # # ... or set req.response and return None. + # req.response = resp # option 2 + # + # See the end of http://pythonpaste.org/webob/modules/dec.html + # for more info. + raise NotImplementedError("You must implement __call__") + + +class Debug(WSGILayer): + @wsgify + def __call__(self, req): + for k, v in req.environ.items(): + print k, "=", v + return self.application + +class Auth(WSGILayer): + @wsgify + def __call__(self, req): + if not 'openstack.auth.token' in req.environ: + # Check auth params here + if True: + req.environ['openstack.auth.token'] = '12345' + else: + return exc.HTTPForbidden(detail="Go away") + + response = req.get_response(self.application) + response.headers['X-Openstack-Auth'] = 'Success' + return response + +class Router(WSGILayer): + def __init__(self, application=None): + super(Router, self).__init__(application) + self.map = routes.Mapper() + self._connect() + + @wsgify + def __call__(self, req): + match = self.map.match(req.path_info) + if match is None: + return self.application + req.environ['openstack.match'] = match + return match['controller'] + + def _connect(self): + raise NotImplementedError("You must implement _connect") + +class FileRouter(Router): + def _connect(self): + self.map.connect(None, '/files/{file}', controller=File()) + self.map.connect(None, '/rfiles/{file}', controller=Reverse(File())) + +class Message(WSGILayer): + @wsgify + def __call__(self, req): + return 'message\n' + +class Reverse(WSGILayer): + @wsgify + def __call__(self, req): + inner_resp = req.get_response(self.application) + print "+" * 80 + Debug()(req) + print "*" * 80 + resp = Response() + resp.app_iter = itertools.imap(lambda x: x[::-1], inner_resp.app_iter) + return resp + +class File(WSGILayer): + @wsgify + def __call__(self, req): + try: + myfile = open(req.environ['openstack.match']['file']) + except IOError, e: + raise exc.HTTPNotFound() + req.response = Response() + req.response.app_iter = myfile + +sock = eventlet.listen(('localhost', 12345)) +eventlet.wsgi.server(sock, Debug(Auth(FileRouter(Message())))) -- cgit From f27d775bee0089e0c86f9a0421a57ab41d0a3a57 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 5 Aug 2010 23:35:16 -0400 Subject: WsgiStack class, eventletserver.serve. Trying to work toward a simple API that anyone can use to start an eventlet-based server composed of several WSGI apps. --- nova/endpoint/eventletserver.py | 7 +++++++ nova/endpoint/new_wsgi.py | 30 ++++++++++++++++++++++-------- nova/endpoint/rackspace.py | 2 -- 3 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 nova/endpoint/eventletserver.py diff --git a/nova/endpoint/eventletserver.py b/nova/endpoint/eventletserver.py new file mode 100644 index 000000000..b8c15ff5d --- /dev/null +++ b/nova/endpoint/eventletserver.py @@ -0,0 +1,7 @@ +import eventlet +import eventlet.wsgi +eventlet.patcher.monkey_patch(all=False, socket=True) + +def serve(app, port): + sock = eventlet.listen(('0.0.0.0', port)) + eventlet.wsgi.server(sock, app) diff --git a/nova/endpoint/new_wsgi.py b/nova/endpoint/new_wsgi.py index c7eee20fc..0f096ddb7 100644 --- a/nova/endpoint/new_wsgi.py +++ b/nova/endpoint/new_wsgi.py @@ -1,11 +1,10 @@ -import eventlet -import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True) +import eventletserver import carrot.connection import carrot.messaging import itertools import routes + # See http://pythonpaste.org/webob/ for usage from webob.dec import wsgify from webob import exc, Request, Response @@ -49,6 +48,19 @@ class WSGILayer(object): raise NotImplementedError("You must implement __call__") +class WsgiStack(WSGILayer): + def __init__(self, wsgi_layers): + bottom_up = list(reversed(wsgi_layers)) + app, remaining = bottom_up[0], bottom_up[1:] + for layer in remaining: + layer.application = app + app = layer + super(WsgiStack, self).__init__(app) + + @wsgify + def __call__(self, req): + return self.application + class Debug(WSGILayer): @wsgify def __call__(self, req): @@ -101,9 +113,6 @@ class Reverse(WSGILayer): @wsgify def __call__(self, req): inner_resp = req.get_response(self.application) - print "+" * 80 - Debug()(req) - print "*" * 80 resp = Response() resp.app_iter = itertools.imap(lambda x: x[::-1], inner_resp.app_iter) return resp @@ -118,5 +127,10 @@ class File(WSGILayer): req.response = Response() req.response.app_iter = myfile -sock = eventlet.listen(('localhost', 12345)) -eventlet.wsgi.server(sock, Debug(Auth(FileRouter(Message())))) +wsgi_layers = [ + Auth(), + Debug(), + FileRouter(), + Message(), + ] +eventletserver.serve(app=WsgiStack(wsgi_layers), port=12345) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 870aa0629..323032eb1 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -26,8 +26,6 @@ import logging import multiprocessing import os import time -import tornado.web -from twisted.internet import defer from nova import datastore from nova import exception -- cgit From 66c8abfb9f00ea06517e102f02ef8bdc9469aae8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 6 Aug 2010 14:31:03 -0700 Subject: fix search/replace error --- nova/tests/network_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index c7d3e86f0..879ee02a4 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -166,8 +166,8 @@ class NetworkTestCase(test.TrialTestCase): for i in xrange(vpn.NetworkData.num_ports_for_ip(FLAGS.vpn_ip)): vpns.append(vpn.NetworkData.create("vpnuser%s" % i)) self.assertRaises(vpn.NoMorePorts, vpn.NetworkData.create, "boom") - for vpn in vpns: - vpn.destroy() + for network_datum in vpns: + network_datum.destroy() def test_release_before_deallocate(self): pass -- cgit From a33dce2da8dc8e25d0943732adfa6b14b1e48c7b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 6 Aug 2010 15:48:46 -0700 Subject: a few more commands were putting output on stderr. In general, exceptions on stderr output seems like a bad idea --- nova/volume/service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/volume/service.py b/nova/volume/service.py index 9c52ee469..66163a812 100644 --- a/nova/volume/service.py +++ b/nova/volume/service.py @@ -128,8 +128,8 @@ class VolumeService(service.Service): def _restart_exports(self): if FLAGS.fake_storage: return - yield process.simple_execute("sudo vblade-persist auto all") - # NOTE(vish): this command sometimes sends output to stderr for warnings + # NOTE(vish): these commands sometimes sends output to stderr for warnings + yield process.simple_execute("sudo vblade-persist auto all", error_ok=1) yield process.simple_execute("sudo vblade-persist start all", error_ok=1) @defer.inlineCallbacks @@ -243,7 +243,8 @@ class Volume(datastore.BasicModel): yield process.simple_execute( "sudo lvcreate -L %s -n %s %s" % (sizestr, self['volume_id'], - FLAGS.volume_group)) + FLAGS.volume_group), + error_ok=1) @defer.inlineCallbacks def _delete_lv(self): @@ -277,7 +278,7 @@ class Volume(datastore.BasicModel): self['blade_id'], FLAGS.aoe_eth_dev, FLAGS.volume_group, - self['volume_id'])) + self['volume_id']), error_ok=1) @defer.inlineCallbacks def _remove_export(self): -- cgit From 91e085b2c272ebd30955a83d3871c402f6749316 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Fri, 6 Aug 2010 18:06:57 -0700 Subject: Changed the network imports to use new network layout. --- nova/endpoint/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 323032eb1..7a3fbe141 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -34,7 +34,7 @@ from nova import rpc from nova import utils from nova.auth import manager from nova.compute import model -from nova.compute import network +from nova.network import model as network from nova.endpoint import images from nova.endpoint import wsgi -- cgit From fd625a55c3725b5cff4449a687b0d54d0d49bd2e Mon Sep 17 00:00:00 2001 From: Eric Day Date: Sat, 7 Aug 2010 12:12:10 -0700 Subject: Reworked WSGI helper module and converted rackspace API endpoint to use it. --- bin/nova-rsapi | 21 +-- nova/endpoint/eventletserver.py | 7 - nova/endpoint/new_wsgi.py | 136 ------------------ nova/endpoint/rackspace.py | 302 +++++++++++++--------------------------- nova/endpoint/wsgi.py | 40 ------ nova/wsgi.py | 173 +++++++++++++++++++++++ 6 files changed, 271 insertions(+), 408 deletions(-) delete mode 100644 nova/endpoint/eventletserver.py delete mode 100644 nova/endpoint/new_wsgi.py delete mode 100644 nova/endpoint/wsgi.py create mode 100644 nova/wsgi.py diff --git a/bin/nova-rsapi b/bin/nova-rsapi index a17efccc0..026880d5a 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -1,4 +1,5 @@ #!/usr/bin/env python +# pylint: disable-msg=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the @@ -17,31 +18,17 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - WSGI daemon for the main API endpoint. + Daemon for the Rackspace API endpoint. """ -import logging -from tornado import ioloop -from wsgiref import simple_server - from nova import flags -from nova import rpc -from nova import server from nova import utils -from nova.auth import manager +from nova import wsgi from nova.endpoint import rackspace FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -def main(_argv): - api_instance = rackspace.Api() - http_server = simple_server.WSGIServer(('0.0.0.0', FLAGS.cc_port), simple_server.WSGIRequestHandler) - http_server.set_app(api_instance.handler) - logging.debug('Started HTTP server on port %i' % FLAGS.cc_port) - while True: - http_server.handle_request() - if __name__ == '__main__': utils.default_flagfile() - server.serve('nova-rsapi', main) + wsgi.run_server(rackspace.API(), FLAGS.cc_port) diff --git a/nova/endpoint/eventletserver.py b/nova/endpoint/eventletserver.py deleted file mode 100644 index b8c15ff5d..000000000 --- a/nova/endpoint/eventletserver.py +++ /dev/null @@ -1,7 +0,0 @@ -import eventlet -import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True) - -def serve(app, port): - sock = eventlet.listen(('0.0.0.0', port)) - eventlet.wsgi.server(sock, app) diff --git a/nova/endpoint/new_wsgi.py b/nova/endpoint/new_wsgi.py deleted file mode 100644 index 0f096ddb7..000000000 --- a/nova/endpoint/new_wsgi.py +++ /dev/null @@ -1,136 +0,0 @@ -import eventletserver -import carrot.connection -import carrot.messaging -import itertools -import routes - - -# See http://pythonpaste.org/webob/ for usage -from webob.dec import wsgify -from webob import exc, Request, Response - -class WSGILayer(object): - def __init__(self, application=None): - self.application = application - - def __call__(self, environ, start_response): - # Subclasses will probably want to implement __call__ like this: - # - # @wsgify - # def __call__(self, req): - # # Any of the following objects work as responses: - # - # # Option 1: simple string - # resp = 'message\n' - # - # # Option 2: a nicely formatted HTTP exception page - # resp = exc.HTTPForbidden(detail='Nice try') - # - # # Option 3: a webob Response object (in case you need to play with - # # headers, or you want to be treated like an iterable, or or or) - # resp = Response(); resp.app_iter = open('somefile') - # - # # Option 4: any wsgi app to be run next - # resp = self.application - # - # # Option 5: you can get a Response object for a wsgi app, too, to - # # play with headers etc - # resp = req.get_response(self.application) - # - # - # # You can then just return your response... - # return resp # option 1 - # # ... or set req.response and return None. - # req.response = resp # option 2 - # - # See the end of http://pythonpaste.org/webob/modules/dec.html - # for more info. - raise NotImplementedError("You must implement __call__") - - -class WsgiStack(WSGILayer): - def __init__(self, wsgi_layers): - bottom_up = list(reversed(wsgi_layers)) - app, remaining = bottom_up[0], bottom_up[1:] - for layer in remaining: - layer.application = app - app = layer - super(WsgiStack, self).__init__(app) - - @wsgify - def __call__(self, req): - return self.application - -class Debug(WSGILayer): - @wsgify - def __call__(self, req): - for k, v in req.environ.items(): - print k, "=", v - return self.application - -class Auth(WSGILayer): - @wsgify - def __call__(self, req): - if not 'openstack.auth.token' in req.environ: - # Check auth params here - if True: - req.environ['openstack.auth.token'] = '12345' - else: - return exc.HTTPForbidden(detail="Go away") - - response = req.get_response(self.application) - response.headers['X-Openstack-Auth'] = 'Success' - return response - -class Router(WSGILayer): - def __init__(self, application=None): - super(Router, self).__init__(application) - self.map = routes.Mapper() - self._connect() - - @wsgify - def __call__(self, req): - match = self.map.match(req.path_info) - if match is None: - return self.application - req.environ['openstack.match'] = match - return match['controller'] - - def _connect(self): - raise NotImplementedError("You must implement _connect") - -class FileRouter(Router): - def _connect(self): - self.map.connect(None, '/files/{file}', controller=File()) - self.map.connect(None, '/rfiles/{file}', controller=Reverse(File())) - -class Message(WSGILayer): - @wsgify - def __call__(self, req): - return 'message\n' - -class Reverse(WSGILayer): - @wsgify - def __call__(self, req): - inner_resp = req.get_response(self.application) - resp = Response() - resp.app_iter = itertools.imap(lambda x: x[::-1], inner_resp.app_iter) - return resp - -class File(WSGILayer): - @wsgify - def __call__(self, req): - try: - myfile = open(req.environ['openstack.match']['file']) - except IOError, e: - raise exc.HTTPNotFound() - req.response = Response() - req.response.app_iter = myfile - -wsgi_layers = [ - Auth(), - Debug(), - FileRouter(), - Message(), - ] -eventletserver.serve(app=WsgiStack(wsgi_layers), port=12345) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 7a3fbe141..f6735a260 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -17,206 +17,95 @@ # under the License. """ -Rackspace API +Rackspace API Endpoint """ -import base64 import json -import logging -import multiprocessing -import os import time -from nova import datastore -from nova import exception +import webob.dec +import webob.exc + from nova import flags from nova import rpc from nova import utils +from nova import wsgi from nova.auth import manager -from nova.compute import model +from nova.compute import model as compute from nova.network import model as network -from nova.endpoint import images -from nova.endpoint import wsgi FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') -class Unauthorized(Exception): - pass - -class NotFound(Exception): - pass - - -class Api(object): +class API(wsgi.Middleware): + """Entry point for all requests.""" def __init__(self): - """build endpoints here""" - self.controllers = { - "v1.0": RackspaceAuthenticationApi(), - "servers": RackspaceCloudServerApi() - } - - def handler(self, environ, responder): - """ - This is the entrypoint from wsgi. Read PEP 333 and wsgi.org for - more intormation. The key points are responder is a callback that - needs to run before you return, and takes two arguments, response - code string ("200 OK") and headers (["X-How-Cool-Am-I: Ultra-Suede"]) - and the return value is the body of the response. - """ - environ['nova.context'] = self.build_context(environ) - controller, path = wsgi.Util.route( - environ['PATH_INFO'], - self.controllers - ) - logging.debug("Route %s to %s", str(path), str(controller)) - if not controller: - responder("404 Not Found", []) - return "" - try: - rv = controller.process(path, environ) - if type(rv) is tuple: - responder(rv[0], rv[1]) - rv = rv[2] - else: - responder("200 OK", []) - return rv - except Unauthorized: - responder("401 Unauthorized", []) - return "" - except NotFound: - responder("404 Not Found", []) - return "" - - - def build_context(self, env): - rv = {} - if env.has_key("HTTP_X_AUTH_TOKEN"): - rv['user'] = manager.AuthManager().get_user_from_access_key( - env['HTTP_X_AUTH_TOKEN'] - ) - if rv['user']: - rv['project'] = manager.AuthManager().get_project( - rv['user'].name - ) - return rv - - -class RackspaceApiEndpoint(object): - def process(self, path, env): - """ - Main entrypoint for all controllers (what gets run by the wsgi handler). - Check authentication based on key, raise Unauthorized if invalid. - - Select the most appropriate action based on request type GET, POST, etc, - then pass it through to the implementing controller. Defalut to GET if - the implementing child doesn't respond to a particular type. - """ - if not self.check_authentication(env): - raise Unauthorized("Unable to authenticate") - - method = env['REQUEST_METHOD'].lower() - callback = getattr(self, method, None) - if not callback: - callback = getattr(self, "get") - logging.debug("%s processing %s with %s", self, method, callback) - return callback(path, env) - - def get(self, path, env): - """ - The default GET will look at the path and call an appropriate - action within this controller based on the the structure of the path. - - Given the following path lengths (with the first part stripped of by - router, as it is the controller name): - = 0 -> index - = 1 -> first component (/servers/details -> details) - >= 2 -> second path component (/servers/ID/ips/* -> ips) - - This should return - A String if 200 OK and no additional headers - (CODE, HEADERS, BODY) for custom response code and headers - """ - if len(path) == 0 and hasattr(self, "index"): - logging.debug("%s running index", self) - return self.index(env) - if len(path) >= 2: - action = path[1] - else: - action = path.pop(0) - - logging.debug("%s running action %s", self, action) - if hasattr(self, action): - method = getattr(self, action) - return method(path, env) - else: - raise NotFound("Missing method %s" % path[0]) - - def check_authentication(self, env): - if not env['nova.context']['user']: - return False - return True - - -class RackspaceAuthenticationApi(object): - - def process(self, path, env): - return self.index(path, env) - - # TODO(todd): make a actual session with a unique token - # just pass the auth key back through for now - def index(self, _path, env): - response = '204 No Content' - headers = [ - ('X-Server-Management-Url', 'http://%s' % env['HTTP_HOST']), - ('X-Storage-Url', 'http://%s' % env['HTTP_HOST']), - ('X-CDN-Managment-Url', 'http://%s' % env['HTTP_HOST']), - ('X-Auth-Token', env['HTTP_X_AUTH_KEY']) - ] - body = "" - return (response, headers, body) - - -class RackspaceCloudServerApi(RackspaceApiEndpoint): + super(API, self).__init__(Router(webob.exc.HTTPNotFound())) + + def __call__(self, environ, start_response): + context = {} + if "HTTP_X_AUTH_TOKEN" in environ: + context['user'] = manager.AuthManager().get_user_from_access_key( + environ['HTTP_X_AUTH_TOKEN']) + if context['user']: + context['project'] = manager.AuthManager().get_project( + context['user'].name) + if "user" not in context: + return webob.exc.HTTPForbidden()(environ, start_response) + environ['nova.context'] = context + return self.application(environ, start_response) + + +class Router(wsgi.Router): + """Route requests to the next WSGI application.""" + + def _build_map(self): + """Build routing map for authentication and cloud.""" + self._connect("/v1.0", controller=AuthenticationAPI()) + cloud = CloudServerAPI() + self._connect("/servers", controller=cloud.launch_server, + conditions={"method": ["POST"]}) + self._connect("/servers/{server_id}", controller=cloud.delete_server, + conditions={'method': ["DELETE"]}) + self._connect("/servers", controller=cloud) + + +class AuthenticationAPI(wsgi.Application): + """Handle all authorization requests through WSGI applications.""" + + @webob.dec.wsgify + def __call__(self, req): # pylint: disable-msg=W0221 + # TODO(todd): make a actual session with a unique token + # just pass the auth key back through for now + res = webob.Response() + res.status = '204 No Content' + res.headers.add('X-Server-Management-Url', req.host_url) + res.headers.add('X-Storage-Url', req.host_url) + res.headers.add('X-CDN-Managment-Url', req.host_url) + res.headers.add('X-Auth-Token', req.headers['X-Auth-Key']) + return res + + +class CloudServerAPI(wsgi.Application): + """Handle all server requests through WSGI applications.""" def __init__(self): - self.instdir = model.InstanceDirectory() + super(CloudServerAPI, self).__init__() + self.instdir = compute.InstanceDirectory() self.network = network.PublicNetworkController() - def post(self, path, env): - if len(path) == 0: - return self.launch_server(env) - - def delete(self, path_parts, env): - if self.delete_server(path_parts[0]): - return ("202 Accepted", [], "") - else: - return ("404 Not Found", [], - "Did not find image, or it was not in a running state") - - - def index(self, env): - return self.detail(env) - - def detail(self, args, env): + @webob.dec.wsgify + def __call__(self, req): # pylint: disable-msg=W0221 value = {"servers": []} for inst in self.instdir.all: value["servers"].append(self.instance_details(inst)) return json.dumps(value) - ## - ## - - def launch_server(self, env): - data = json.loads(env['wsgi.input'].read(int(env['CONTENT_LENGTH']))) - inst = self.build_server_instance(data, env['nova.context']) - self.schedule_launch_of_instance(inst) - return json.dumps({"server": self.instance_details(inst)}) - - def instance_details(self, inst): + def instance_details(self, inst): # pylint: disable-msg=R0201 + "Build the data structure to represent details for an instance." return { "id": inst.get("instance_id", None), "imageId": inst.get("image_id", None), @@ -224,11 +113,9 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): "hostId": inst.get("node_name", None), "status": inst.get("state", "pending"), "addresses": { - "public": [self.network.get_public_ip_for_instance( - inst.get("instance_id", None) - )], - "private": [inst.get("private_dns_name", None)] - }, + "public": [network.get_public_ip_for_instance( + inst.get("instance_id", None))], + "private": [inst.get("private_dns_name", None)]}, # implemented only by Rackspace, not AWS "name": inst.get("name", "Not-Specified"), @@ -237,11 +124,22 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): "progress": "Not-Supported", "metadata": { "Server Label": "Not-Supported", - "Image Version": "Not-Supported" - } - } + "Image Version": "Not-Supported"}} + + @webob.dec.wsgify + def launch_server(self, req): + """Launch a new instance.""" + data = json.loads(req.body) + inst = self.build_server_instance(data, req.environ['nova.context']) + rpc.cast( + FLAGS.compute_topic, { + "method": "run_instance", + "args": {"instance_id": inst.instance_id}}) + + return json.dumps({"server": self.instance_details(inst)}) def build_server_instance(self, env, context): + """Build instance data structure and save it to the data store.""" reservation = utils.generate_uid('r') ltime = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) inst = self.instdir.new() @@ -253,45 +151,33 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): inst['reservation_id'] = reservation inst['launch_time'] = ltime inst['mac_address'] = utils.generate_mac() - address = network.allocate_ip( + address = self.network.allocate_ip( inst['user_id'], inst['project_id'], - mac=inst['mac_address'] - ) + mac=inst['mac_address']) inst['private_dns_name'] = str(address) inst['bridge_name'] = network.BridgedNetwork.get_network_for_project( inst['user_id'], inst['project_id'], - 'default' # security group - )['bridge_name'] + 'default')['bridge_name'] # key_data, key_name, ami_launch_index # TODO(todd): key data or root password inst.save() return inst - def schedule_launch_of_instance(self, inst): - rpc.cast( - FLAGS.compute_topic, - { - "method": "run_instance", - "args": {"instance_id": inst.instance_id} - } - ) - - def delete_server(self, instance_id): - owner_hostname = self.host_for_instance(instance_id) - # it isn't launched? + @webob.dec.wsgify + @wsgi.route_args + def delete_server(self, req, route_args): # pylint: disable-msg=R0201 + """Delete an instance.""" + owner_hostname = None + instance = compute.Instance.lookup(route_args['server_id']) + if instance: + owner_hostname = instance["node_name"] if not owner_hostname: - return None + return webob.exc.HTTPNotFound("Did not find image, or it was " + "not in a running state.") rpc_transport = "%s:%s" % (FLAGS.compute_topic, owner_hostname) rpc.cast(rpc_transport, {"method": "reboot_instance", - "args": {"instance_id": instance_id}}) - return True - - def host_for_instance(self, instance_id): - instance = model.Instance.lookup(instance_id) - if not instance: - return None - return instance["node_name"] - + "args": {"instance_id": route_args['server_id']}}) + req.status = "202 Accepted" diff --git a/nova/endpoint/wsgi.py b/nova/endpoint/wsgi.py deleted file mode 100644 index b7bb588c3..000000000 --- a/nova/endpoint/wsgi.py +++ /dev/null @@ -1,40 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -Utility methods for working with WSGI servers -''' - -class Util(object): - - @staticmethod - def route(reqstr, controllers): - if len(reqstr) == 0: - return Util.select_root_controller(controllers), [] - parts = [x for x in reqstr.split("/") if len(x) > 0] - if len(parts) == 0: - return Util.select_root_controller(controllers), [] - return controllers[parts[0]], parts[1:] - - @staticmethod - def select_root_controller(controllers): - if '' in controllers: - return controllers[''] - else: - return None - diff --git a/nova/wsgi.py b/nova/wsgi.py new file mode 100644 index 000000000..4fd6e59e3 --- /dev/null +++ b/nova/wsgi.py @@ -0,0 +1,173 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utility methods for working with WSGI servers +""" + +import logging +import sys + +import eventlet +import eventlet.wsgi +eventlet.patcher.monkey_patch(all=False, socket=True) +import routes +import routes.middleware + + +logging.getLogger("routes.middleware").addHandler(logging.StreamHandler()) + + +def run_server(application, port): + """Run a WSGI server with the given application.""" + sock = eventlet.listen(('0.0.0.0', port)) + eventlet.wsgi.server(sock, application) + + +class Application(object): + """Base WSGI application wrapper. Subclasses need to implement __call__.""" + + def __call__(self, environ, start_response): + r"""Subclasses will probably want to implement __call__ like this: + + @webob.dec.wsgify + def __call__(self, req): + # Any of the following objects work as responses: + + # Option 1: simple string + res = 'message\n' + + # Option 2: a nicely formatted HTTP exception page + res = exc.HTTPForbidden(detail='Nice try') + + # Option 3: a webob Response object (in case you need to play with + # headers, or you want to be treated like an iterable, or or or) + res = Response(); + res.app_iter = open('somefile') + + # Option 4: any wsgi app to be run next + res = self.application + + # Option 5: you can get a Response object for a wsgi app, too, to + # play with headers etc + res = req.get_response(self.application) + + # You can then just return your response... + return res + # ... or set req.response and return None. + req.response = res + + See the end of http://pythonpaste.org/webob/modules/dec.html + for more info. + """ + raise NotImplementedError("You must implement __call__") + + +class Middleware(Application): # pylint: disable-msg=W0223 + """Base WSGI middleware wrapper. These classes require an + application to be initialized that will be called next.""" + + def __init__(self, application): # pylint: disable-msg=W0231 + self.application = application + + +class Debug(Middleware): + """Helper class that can be insertd into any WSGI application chain + to get information about the request and response.""" + + def __call__(self, environ, start_response): + for key, value in environ.items(): + print key, "=", value + print + wrapper = debug_start_response(start_response) + return debug_print_body(self.application(environ, wrapper)) + + +def debug_start_response(start_response): + """Wrap the start_response to capture when called.""" + + def wrapper(status, headers, exc_info=None): + """Print out all headers when start_response is called.""" + print status + for (key, value) in headers: + print key, "=", value + print + start_response(status, headers, exc_info) + + return wrapper + + +def debug_print_body(body): + """Print the body of the response as it is sent back.""" + + class Wrapper(object): + """Iterate through all the body parts and print before returning.""" + + def __iter__(self): + for part in body: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print + + return Wrapper() + + +class ParsedRoutes(Middleware): + """Processed parsed routes from routes.middleware.RoutesMiddleware + and call either the controller if found or the default application + otherwise.""" + + def __call__(self, environ, start_response): + if environ['routes.route'] is None: + return self.application(environ, start_response) + app = environ['wsgiorg.routing_args'][1]['controller'] + return app(environ, start_response) + + +class Router(Middleware): # pylint: disable-msg=R0921 + """Wrapper to help setup routes.middleware.RoutesMiddleware.""" + + def __init__(self, application): + self.map = routes.Mapper() + self._build_map() + application = ParsedRoutes(application) + application = routes.middleware.RoutesMiddleware(application, self.map) + super(Router, self).__init__(application) + + def __call__(self, environ, start_response): + return self.application(environ, start_response) + + def _build_map(self): + """Method to create new connections for the routing map.""" + raise NotImplementedError("You must implement _build_map") + + def _connect(self, *args, **kwargs): + """Wrapper for the map.connect method.""" + self.map.connect(*args, **kwargs) + + +def route_args(application): + """Decorator to make grabbing routing args more convenient.""" + + def wrapper(self, req): + """Call application with req and parsed routing args from.""" + return application(self, req, req.environ['wsgiorg.routing_args'][1]) + + return wrapper -- cgit From 46c0f66d123e2b4af101bb12408ad6de5eb6855b Mon Sep 17 00:00:00 2001 From: Eric Day Date: Sat, 7 Aug 2010 19:51:17 -0700 Subject: Cleaned up pep8/pylint for bin/* files. I did not fix rsapi since this is already cleaned up in another branch. --- bin/nova-api | 6 +- bin/nova-dhcpbridge | 42 ++++++----- bin/nova-import-canonical-imagestore | 33 ++++----- bin/nova-instancemonitor | 18 ++--- bin/nova-manage | 138 +++++++++++++++++++---------------- bin/nova-objectstore | 8 +- pylintrc | 3 + run_tests.py | 3 +- 8 files changed, 131 insertions(+), 120 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 1f2009c30..13baf22a7 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -29,8 +29,6 @@ from nova import flags from nova import rpc from nova import server from nova import utils -from nova.auth import manager -from nova.compute import model from nova.endpoint import admin from nova.endpoint import api from nova.endpoint import cloud @@ -39,10 +37,10 @@ FLAGS = flags.FLAGS def main(_argv): + """Load the controllers and start the tornado I/O loop.""" controllers = { 'Cloud': cloud.CloudController(), - 'Admin': admin.AdminController() - } + 'Admin': admin.AdminController()} _app = api.APIServerApplication(controllers) conn = rpc.Connection.instance() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index b3e7d456a..ed1af206a 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -18,8 +18,6 @@ # under the License. """ -nova-dhcpbridge - Handle lease database updates from DHCP servers. """ @@ -42,34 +40,43 @@ from nova.network import service FLAGS = flags.FLAGS -def add_lease(mac, ip, hostname, interface): +def add_lease(_mac, ip, _hostname, _interface): + """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: service.VlanNetworkService().lease_ip(ip) else: - rpc.cast("%s.%s" (FLAGS.network_topic, FLAGS.node_name), + rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "lease_ip", - "args" : {"fixed_ip": ip}}) + "args": {"fixed_ip": ip}}) + -def old_lease(mac, ip, hostname, interface): +def old_lease(_mac, _ip, _hostname, _interface): + """Do nothing, just an old lease update.""" logging.debug("Adopted old lease or got a change of mac/hostname") -def del_lease(mac, ip, hostname, interface): + +def del_lease(_mac, ip, _hostname, _interface): + """Remove the leased IP from the databases.""" if FLAGS.fake_rabbit: service.VlanNetworkService().release_ip(ip) else: - rpc.cast("%s.%s" (FLAGS.network_topic, FLAGS.node_name), + rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "release_ip", - "args" : {"fixed_ip": ip}}) + "args": {"fixed_ip": ip}}) + def init_leases(interface): + """Get the list of hosts for an interface.""" net = model.get_network_by_interface(interface) res = "" for host_name in net.hosts: - res += "%s\n" % linux_net.hostDHCP(net, host_name, net.hosts[host_name]) + res += "%s\n" % linux_net.hostDHCP(net, host_name, + net.hosts[host_name]) return res def main(): + """Parse environment and arguments and call the approproate action.""" flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) @@ -79,18 +86,19 @@ def main(): FLAGS.redis_db = 8 FLAGS.network_size = 32 FLAGS.connection_type = 'fake' - FLAGS.fake_network=True - FLAGS.auth_driver='nova.auth.ldapdriver.FakeLdapDriver' + FLAGS.fake_network = True + FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' action = argv[1] - if action in ['add','del','old']: + if action in ['add', 'del', 'old']: mac = argv[2] ip = argv[3] hostname = argv[4] - logging.debug("Called %s for mac %s with ip %s and hostname %s on interface %s" % (action, mac, ip, hostname, interface)) - globals()[action+'_lease'](mac, ip, hostname, interface) + logging.debug("Called %s for mac %s with ip %s and " + "hostname %s on interface %s", + action, mac, ip, hostname, interface) + globals()[action + '_lease'](mac, ip, hostname, interface) else: print init_leases(interface) - exit(0) if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 2e79f09b7..5165109b2 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -37,20 +37,17 @@ FLAGS = flags.FLAGS api_url = 'https://imagestore.canonical.com/api/dashboard' -image_cache = None -def images(): - global image_cache - if not image_cache: - try: - images = json.load(urllib2.urlopen(api_url))['images'] - image_cache = [i for i in images if i['title'].find('amd64') > -1] - except Exception: - print 'unable to download canonical image list' - sys.exit(1) - return image_cache - -# FIXME(ja): add checksum/signature checks + +def get_images(): + """Get a list of the images from the imagestore URL.""" + images = json.load(urllib2.urlopen(api_url))['images'] + images = [img for img in images if img['title'].find('amd64') > -1] + return images + + def download(img): + """Download an image to the local filesystem.""" + # FIXME(ja): add checksum/signature checks tempdir = tempfile.mkdtemp(prefix='cis-') kernel_id = None @@ -79,20 +76,22 @@ def download(img): shutil.rmtree(tempdir) + def main(): + """Main entry point.""" utils.default_flagfile() argv = FLAGS(sys.argv) + images = get_images() if len(argv) == 2: - for img in images(): + for img in images: if argv[1] == 'all' or argv[1] == img['title']: download(img) else: print 'usage: %s (title|all)' print 'available images:' - for image in images(): - print image['title'] + for img in images: + print img['title'] if __name__ == '__main__': main() - diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index b195089b7..911fb6f42 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -22,7 +22,6 @@ """ import logging -from twisted.internet import task from twisted.application import service from nova import twistd @@ -30,7 +29,11 @@ from nova.compute import monitor logging.getLogger('boto').setLevel(logging.WARN) -def main(): + +if __name__ == '__main__': + twistd.serve(__file__) + +if __name__ == '__builtin__': logging.warn('Starting instance monitor') m = monitor.InstanceMonitor() @@ -38,14 +41,3 @@ def main(): # parses this file, return it so that we can get it into globals below application = service.Application('nova-instancemonitor') m.setServiceParent(application) - return application - -if __name__ == '__main__': - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = main() - - - - diff --git a/bin/nova-manage b/bin/nova-manage index 7835c7a77..2dd569df0 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -37,12 +37,15 @@ FLAGS = flags.FLAGS class VpnCommands(object): + """Class for managing VPNs.""" + def __init__(self): self.manager = manager.AuthManager() self.instdir = model.InstanceDirectory() self.pipe = pipelib.CloudPipe(cloud.CloudController()) def list(self): + """Print a listing of the VPNs for all projects.""" print "%-12s\t" % 'project', print "%-12s\t" % 'ip:port', print "%s" % 'state' @@ -50,9 +53,10 @@ class VpnCommands(object): print "%-12s\t" % project.name, print "%s:%s\t" % (project.vpn_ip, project.vpn_port), - vpn = self.__vpn_for(project.id) + vpn = self._vpn_for(project.id) if vpn: - out, err = utils.execute("ping -c1 -w1 %s > /dev/null; echo $?" % vpn['private_dns_name']) + command = "ping -c1 -w1 %s > /dev/null; echo $?" + out, _err = utils.execute(command % vpn['private_dns_name']) if out.strip() == '0': net = 'up' else: @@ -66,25 +70,32 @@ class VpnCommands(object): else: print None - def __vpn_for(self, project_id): + def _vpn_for(self, project_id): + """Get the VPN instance for a project ID.""" for instance in self.instdir.all: - if (instance.state.has_key('image_id') + if ('image_id' in instance.state and instance['image_id'] == FLAGS.vpn_image_id - and not instance['state_description'] in ['shutting_down', 'shutdown'] + and not instance['state_description'] in + ['shutting_down', 'shutdown'] and instance['project_id'] == project_id): return instance def spawn(self): + """Run all VPNs.""" for p in reversed(self.manager.get_projects()): - if not self.__vpn_for(p.id): - print 'spawning %s' % p.id - self.pipe.launch_vpn_instance(p.id) - time.sleep(10) + if not self._vpn_for(p.id): + print 'spawning %s' % p.id + self.pipe.launch_vpn_instance(p.id) + time.sleep(10) def run(self, project_id): + """Start the VPN for a given project.""" self.pipe.launch_vpn_instance(project_id) + class RoleCommands(object): + """Class for managing roles.""" + def __init__(self): self.manager = manager.AuthManager() @@ -107,25 +118,24 @@ class RoleCommands(object): arguments: user, role [project]""" self.manager.remove_role(user, role, project) + class UserCommands(object): + """Class for managing users.""" + def __init__(self): self.manager = manager.AuthManager() - def __print_export(self, user): - print 'export EC2_ACCESS_KEY=%s' % user.access - print 'export EC2_SECRET_KEY=%s' % user.secret - def admin(self, name, access=None, secret=None): """creates a new admin and prints exports arguments: name [access] [secret]""" user = self.manager.create_user(name, access, secret, True) - self.__print_export(user) + print_export(user) def create(self, name, access=None, secret=None): """creates a new user and prints exports arguments: name [access] [secret]""" user = self.manager.create_user(name, access, secret, False) - self.__print_export(user) + print_export(user) def delete(self, name): """deletes an existing user @@ -137,7 +147,7 @@ class UserCommands(object): arguments: name""" user = self.manager.get_user(name) if user: - self.__print_export(user) + print_export(user) else: print "User %s doesn't exist" % name @@ -147,53 +157,58 @@ class UserCommands(object): for user in self.manager.get_users(): print user.name + +def print_export(user): + """Print export variables to use with API.""" + print 'export EC2_ACCESS_KEY=%s' % user.access + print 'export EC2_SECRET_KEY=%s' % user.secret + + class ProjectCommands(object): + """Class for managing projects.""" + def __init__(self): self.manager = manager.AuthManager() def add(self, project, user): - """adds user to project + """Adds user to project arguments: project user""" self.manager.add_to_project(user, project) def create(self, name, project_manager, description=None): - """creates a new project + """Creates a new project arguments: name project_manager [description]""" - user = self.manager.create_project(name, project_manager, description) + self.manager.create_project(name, project_manager, description) def delete(self, name): - """deletes an existing project + """Deletes an existing project arguments: name""" self.manager.delete_project(name) def environment(self, project_id, user_id, filename='novarc'): - """exports environment variables to an sourcable file + """Exports environment variables to an sourcable file arguments: project_id user_id [filename='novarc]""" rc = self.manager.get_environment_rc(project_id, user_id) with open(filename, 'w') as f: f.write(rc) def list(self): - """lists all projects + """Lists all projects arguments: """ for project in self.manager.get_projects(): print project.name def remove(self, project, user): - """removes user from project + """Removes user from project arguments: project user""" self.manager.remove_from_project(user, project) - def zip(self, project_id, user_id, filename='nova.zip'): - """exports credentials for project to a zip file + def create_zip(self, project_id, user_id, filename='nova.zip'): + """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - zip = self.manager.get_credentials(project_id, user_id) + zip_file = self.manager.get_credentials(project_id, user_id) with open(filename, 'w') as f: - f.write(zip) - - -def usage(script_name): - print script_name + " category action []" + f.write(zip_file) categories = [ @@ -205,62 +220,61 @@ categories = [ def lazy_match(name, key_value_tuples): - """finds all objects that have a key that case insensitively contains [name] - key_value_tuples is a list of tuples of the form (key, value) + """Finds all objects that have a key that case insensitively contains + [name] key_value_tuples is a list of tuples of the form (key, value) returns a list of tuples of the form (key, value)""" - return [(k, v) for (k, v) in key_value_tuples if k.lower().find(name.lower()) == 0] + result = [] + for (k, v) in key_value_tuples: + if k.lower().find(name.lower()) == 0: + result.append((k, v)) + if len(result) == 0: + print "%s does not match any options:" % name + for k, _v in key_value_tuples: + print "\t%s" % k + sys.exit(2) + if len(result) > 1: + print "%s matched multiple options:" % name + for k, _v in result: + print "\t%s" % k + sys.exit(2) + return result def methods_of(obj): - """get all callable methods of an object that don't start with underscore + """Get all callable methods of an object that don't start with underscore returns a list of tuples of the form (method_name, method)""" - return [(i, getattr(obj, i)) for i in dir(obj) if callable(getattr(obj, i)) and not i.startswith('_')] + result = [] + for i in dir(obj): + if callable(getattr(obj, i)) and not i.startswith('_'): + result.append((i, getattr(obj, i))) + return result -if __name__ == '__main__': +def main(): + """Parse options and call the appropriate class/method.""" utils.default_flagfile('/etc/nova/nova-manage.conf') argv = FLAGS(sys.argv) script_name = argv.pop(0) if len(argv) < 1: - usage(script_name) + print script_name + " category action []" print "Available categories:" - for k, v in categories: + for k, _ in categories: print "\t%s" % k sys.exit(2) category = argv.pop(0) matches = lazy_match(category, categories) - if len(matches) == 0: - print "%s does not match any categories:" % category - for k, v in categories: - print "\t%s" % k - sys.exit(2) - if len(matches) > 1: - print "%s matched multiple categories:" % category - for k, v in matches: - print "\t%s" % k - sys.exit(2) # instantiate the command group object category, fn = matches[0] command_object = fn() actions = methods_of(command_object) if len(argv) < 1: - usage(script_name) + print script_name + " category action []" print "Available actions for %s category:" % category - for k, v in actions: + for k, _v in actions: print "\t%s" % k sys.exit(2) action = argv.pop(0) matches = lazy_match(action, actions) - if len(matches) == 0: - print "%s does not match any actions" % action - for k, v in actions: - print "\t%s" % k - sys.exit(2) - if len(matches) > 1: - print "%s matched multiple actions:" % action - for k, v in matches: - print "\t%s" % k - sys.exit(2) action, fn = matches[0] # call the action with the remaining arguments try: @@ -271,3 +285,5 @@ if __name__ == '__main__': print "%s %s: %s" % (category, action, fn.__doc__) sys.exit(2) +if __name__ == '__main__': + main() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index c0fa815c0..02f2bcb48 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -30,15 +30,9 @@ from nova.objectstore import handler FLAGS = flags.FLAGS -def main(): - app = handler.get_application() - print app - return app - -# NOTE(soren): Stolen from nova-compute if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': utils.default_flagfile() - application = main() + application = handler.get_application() diff --git a/pylintrc b/pylintrc index a853e5bed..258d3556d 100644 --- a/pylintrc +++ b/pylintrc @@ -1,3 +1,6 @@ +[Messages Control] +disable-msg=C0103 + [Basic] method-rgx=[a-z_][a-z0-9_]{2,50}$ diff --git a/run_tests.py b/run_tests.py index 5a8966f02..f62088253 100644 --- a/run_tests.py +++ b/run_tests.py @@ -68,7 +68,8 @@ flags.DEFINE_bool('flush_db', True, 'Flush the database before running fake tests') flags.DEFINE_string('tests_stderr', 'run_tests.err.log', - 'Path to where to pipe STDERR during test runs. Default = "run_tests.err.log"') + 'Path to where to pipe STDERR during test runs. ' + 'Default = "run_tests.err.log"') if __name__ == '__main__': OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) -- cgit From 86a7e62f0b72763088b0a34516ffc30f22ca937e Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 09:49:47 -0700 Subject: adding pep8 and pylint for regular cleanup tasks --- tools/pip-requires | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/pip-requires b/tools/pip-requires index 4eb47ca2b..24aefb25e 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,3 +1,5 @@ +pep8==0.5.0 +pylint==0.21.1 IPy==0.70 M2Crypto==0.20.2 amqplib==0.6.1 -- cgit From abd9bed8f7f88617c0a402faef47da13963ccea7 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 09:50:22 -0700 Subject: attempting some cleanup work --- nova/endpoint/cloud.py | 77 ++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 878d54a15..ee22863a9 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -47,6 +47,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') + def _gen_key(user_id, key_name): """ Tuck this into AuthManager """ try: @@ -102,15 +103,16 @@ class CloudController(object): result = {} for instance in self.instdir.all: if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) + line = '%s slots=%d' % (instance['private_dns_name'], + INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: result[instance['key_name']].append(line) else: result[instance['key_name']] = [line] return result - def get_metadata(self, ip): - i = self.get_instance_by_ip(ip) + def get_metadata(self, ipaddress): + i = self.get_instance_by_ip(ipaddress) if i is None: return None mpi = self._get_mpi_data(i['project_id']) @@ -147,7 +149,7 @@ class CloudController(object): }, 'public-hostname': i.get('dns_name', ''), 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP - 'public-keys' : keys, + 'public-keys': keys, 'ramdisk-id': i.get('ramdisk_id', ''), 'reservation-id': i['reservation_id'], 'security-groups': i.get('groups', ''), @@ -203,7 +205,7 @@ class CloudController(object): 'keyFingerprint': key_pair.fingerprint, }) - return { 'keypairsSet': result } + return {'keypairsSet': result} @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): @@ -232,7 +234,7 @@ class CloudController(object): @rbac.allow('all') def describe_security_groups(self, context, group_names, **kwargs): - groups = { 'securityGroupSet': [] } + groups = {'securityGroupSet': []} # Stubbed for now to unblock other things. return groups @@ -251,7 +253,7 @@ class CloudController(object): instance = self._get_instance(context, instance_id[0]) return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "get_console_output", - "args" : {"instance_id": instance_id[0]}}) + "args": {"instance_id": instance_id[0]}}) def _get_user_id(self, context): if context and context.user: @@ -285,10 +287,10 @@ class CloudController(object): if volume['attach_status'] == 'attached': v['attachmentSet'] = [{'attachTime': volume['attach_time'], 'deleteOnTermination': volume['delete_on_termination'], - 'device' : volume['mountpoint'], - 'instanceId' : volume['instance_id'], - 'status' : 'attached', - 'volume_id' : volume['volume_id']}] + 'device': volume['mountpoint'], + 'instanceId': volume['instance_id'], + 'status': 'attached', + 'volume_id': volume['volume_id']}] else: v['attachmentSet'] = [{}] return v @@ -298,7 +300,7 @@ class CloudController(object): def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", - "args" : {"size": size, + "args": {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary @@ -348,15 +350,15 @@ class CloudController(object): compute_node = instance['node_name'] rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), {"method": "attach_volume", - "args" : {"volume_id": volume_id, - "instance_id" : instance_id, - "mountpoint" : device}}) - return defer.succeed({'attachTime' : volume['attach_time'], - 'device' : volume['mountpoint'], - 'instanceId' : instance_id, - 'requestId' : context.request_id, - 'status' : volume['attach_status'], - 'volumeId' : volume_id}) + "args": {"volume_id": volume_id, + "instance_id": instance_id, + "mountpoint": device}}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) @rbac.allow('projectmanager', 'sysadmin') @@ -372,18 +374,18 @@ class CloudController(object): instance = self._get_instance(context, instance_id) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "detach_volume", - "args" : {"instance_id": instance_id, + "args": {"instance_id": instance_id, "volume_id": volume_id}}) except exception.NotFound: # If the instance doesn't exist anymore, # then we need to call detach blind volume.finish_detach() - return defer.succeed({'attachTime' : volume['attach_time'], - 'device' : volume['mountpoint'], - 'instanceId' : instance_id, - 'requestId' : context.request_id, - 'status' : volume['attach_status'], - 'volumeId' : volume_id}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -425,7 +427,8 @@ class CloudController(object): i['key_name'] = instance.get('key_name', None) if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), instance.get('node_name','')) + instance.get('project_id', None), + instance.get('node_name', '')) i['product_codes_set'] = self._convert_to_set( instance.get('product_codes', None), 'product_code') i['instance_type'] = instance.get('instance_type', None) @@ -442,7 +445,7 @@ class CloudController(object): reservations[res_id] = r reservations[res_id]['instances_set'].append(i) - instance_response = {'reservationSet' : list(reservations.values()) } + instance_response = {'reservationSet': list(reservations.values())} return instance_response @rbac.allow('all') @@ -457,7 +460,7 @@ class CloudController(object): address['project_id'] == context.project.id): address_rv = { 'public_ip': address['address'], - 'instance_id' : address.get('instance_id', 'free') + 'instance_id': address.get('instance_id', 'free') } if context.user.is_admin(): address_rv['instance_id'] = "%s (%s, %s)" % ( @@ -477,7 +480,7 @@ class CloudController(object): "args": {"user_id": context.user.id, "project_id": context.project.id}}) public_ip = alloc_result['result'] - defer.returnValue({'addressSet': [{'publicIp' : public_ip}]}) + defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') @defer.inlineCallbacks @@ -591,7 +594,7 @@ class CloudController(object): inst.save() rpc.cast(FLAGS.compute_topic, {"method": "run_instance", - "args": {"instance_id" : inst.instance_id}}) + "args": {"instance_id": inst.instance_id}}) logging.debug("Casting to node for %s's instance with IP of %s" % (context.user.name, inst['private_dns_name'])) # TODO: Make Network figure out the network name from ip. @@ -646,7 +649,7 @@ class CloudController(object): instance = self._get_instance(context, i) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "reboot_instance", - "args" : {"instance_id": i}}) + "args": {"instance_id": i}}) return defer.succeed(True) @rbac.allow('projectmanager', 'sysadmin') @@ -656,7 +659,7 @@ class CloudController(object): volume_node = volume['node_name'] rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), {"method": "delete_volume", - "args" : {"volume_id": volume_id}}) + "args": {"volume_id": volume_id}}) return defer.succeed(True) @rbac.allow('all') @@ -689,9 +692,9 @@ class CloudController(object): image = images.list(context, image_id)[0] except IndexError: raise exception.ApiError('invalid id: %s' % image_id) - result = { 'image_id': image_id, 'launchPermission': [] } + result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: - result['launchPermission'].append({ 'group': 'all' }) + result['launchPermission'].append({'group': 'all'}) return defer.succeed(result) @rbac.allow('projectmanager', 'sysadmin') -- cgit From e59b769cf1ad12f63788d2e90fd3a4412f9db6f4 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 11:39:14 -0700 Subject: variable name cleanup --- nova/endpoint/cloud.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ee22863a9..8b937306e 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -210,18 +210,18 @@ class CloudController(object): @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): try: - d = defer.Deferred() - p = context.handler.application.settings.get('pool') + dcall = defer.Deferred() + pool = context.handler.application.settings.get('pool') def _complete(kwargs): if 'exception' in kwargs: - d.errback(kwargs['exception']) + dcall.errback(kwargs['exception']) return - d.callback({'keyName': key_name, + dcall.callback({'keyName': key_name, 'keyFingerprint': kwargs['fingerprint'], 'keyMaterial': kwargs['private_key']}) - p.apply_async(_gen_key, [context.user.id, key_name], + pool.apply_async(_gen_key, [context.user.id, key_name], callback=_complete) - return d + return dcall except manager.UserError as e: raise -- cgit From 3fe167e1e398b3d602699b8219dcbfc8fec86859 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 11:40:03 -0700 Subject: removing what appears to be an unused try/except statement - nova.auth.manager.UserError doesn't exist in this codebase. Leftover? Something intended to be there but never added? --- nova/endpoint/cloud.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 8b937306e..ad9188ff3 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -209,22 +209,18 @@ class CloudController(object): @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): - try: - dcall = defer.Deferred() - pool = context.handler.application.settings.get('pool') - def _complete(kwargs): - if 'exception' in kwargs: - dcall.errback(kwargs['exception']) - return - dcall.callback({'keyName': key_name, - 'keyFingerprint': kwargs['fingerprint'], - 'keyMaterial': kwargs['private_key']}) - pool.apply_async(_gen_key, [context.user.id, key_name], - callback=_complete) - return dcall - - except manager.UserError as e: - raise + dcall = defer.Deferred() + pool = context.handler.application.settings.get('pool') + def _complete(kwargs): + if 'exception' in kwargs: + dcall.errback(kwargs['exception']) + return + dcall.callback({'keyName': key_name, + 'keyFingerprint': kwargs['fingerprint'], + 'keyMaterial': kwargs['private_key']}) + pool.apply_async(_gen_key, [context.user.id, key_name], + callback=_complete) + return dcall @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): -- cgit From 86150042191005a9bf04ef243396667cb9dad1b0 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 13:20:50 -0700 Subject: convention and variable naming cleanup for pylint/pep8 --- nova/network/model.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/nova/network/model.py b/nova/network/model.py index daac035e4..eada776c7 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -97,11 +97,11 @@ class Vlan(datastore.BasicModel): def dict_by_vlan(cls): """a hash of vlan:project""" set_name = cls._redis_set_name(cls.__name__) - rv = {} - h = datastore.Redis.instance().hgetall(set_name) - for v in h.keys(): - rv[h[v]] = v - return rv + retvals = {} + hashset = datastore.Redis.instance().hgetall(set_name) + for val in hashset.keys(): + retvals[hashset[val]] = val + return retvals @classmethod @datastore.absorb_connection_error @@ -136,7 +136,8 @@ class Vlan(datastore.BasicModel): # CLEANUP: # TODO(ja): Save the IPs at the top of each subnet for cloudpipe vpn clients -# TODO(ja): does vlanpool "keeper" need to know the min/max - shouldn't FLAGS always win? +# TODO(ja): does vlanpool "keeper" need to know the min/max - +# shouldn't FLAGS always win? # TODO(joshua): Save the IPs at the top of each subnet for cloudpipe vpn clients class BaseNetwork(datastore.BasicModel): @@ -217,7 +218,9 @@ class BaseNetwork(datastore.BasicModel): def available(self): # the .2 address is always CloudPipe # and the top are for vpn clients - for idx in range(self.num_static_ips, len(self.network)-(1 + FLAGS.cnt_vpn_clients)): + num_ips = self.num_static_ips + num_clients = FLAGS.cnt_vpn_clients + for idx in range(num_ips, len(self.network)-(1 + num_clients)): address = str(self.network[idx]) if not address in self.hosts.keys(): yield address @@ -338,8 +341,9 @@ class DHCPNetwork(BridgedNetwork): private_ip = str(self.network[2]) linux_net.confirm_rule("FORWARD -d %s -p udp --dport 1194 -j ACCEPT" % (private_ip, )) - linux_net.confirm_rule("PREROUTING -t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (self.project.vpn_ip, self.project.vpn_port, private_ip)) + linux_net.confirm_rule( + "PREROUTING -t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" + % (self.project.vpn_ip, self.project.vpn_port, private_ip)) def deexpress(self, address=None): # if this is the last address, stop dns @@ -374,13 +378,14 @@ class PublicAddress(datastore.BasicModel): return addr -DEFAULT_PORTS = [("tcp",80), ("tcp",22), ("udp",1194), ("tcp",443)] +DEFAULT_PORTS = [("tcp", 80), ("tcp", 22), ("udp", 1194), ("tcp", 443)] class PublicNetworkController(BaseNetwork): override_type = 'network' def __init__(self, *args, **kwargs): network_id = "public:default" - super(PublicNetworkController, self).__init__(network_id, FLAGS.public_range) + super(PublicNetworkController, self).__init__(network_id, + FLAGS.public_range) self['user_id'] = "public" self['project_id'] = "public" self["create_time"] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) @@ -415,7 +420,7 @@ class PublicNetworkController(BaseNetwork): def deallocate_ip(self, ip_str): # NOTE(vish): cleanup is now done on release by the parent class - self.release_ip(ip_str) + self.release_ip(ip_str) def associate_address(self, public_ip, private_ip, instance_id): if not public_ip in self.assigned: @@ -461,8 +466,9 @@ class PublicNetworkController(BaseNetwork): linux_net.confirm_rule("FORWARD -d %s -p icmp -j ACCEPT" % (private_ip)) for (protocol, port) in DEFAULT_PORTS: - linux_net.confirm_rule("FORWARD -d %s -p %s --dport %s -j ACCEPT" - % (private_ip, protocol, port)) + linux_net.confirm_rule( + "FORWARD -d %s -p %s --dport %s -j ACCEPT" + % (private_ip, protocol, port)) def deexpress(self, address=None): addr = self.get_host(address) -- cgit From 21c1d379199c528024c5e85571609e77e53c6ee7 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 13:31:40 -0700 Subject: light cleanup - convention stuff mostly --- nova/auth/manager.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index d44ed52b2..e5efbca24 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -29,15 +29,17 @@ import uuid import zipfile from nova import crypto -from nova import datastore from nova import exception from nova import flags -from nova import objectstore # for flags from nova import utils -from nova.auth import ldapdriver # for flags from nova.auth import signer from nova.network import vpn +#unused imports +#from nova import datastore +#from nova.auth import ldapdriver # for flags +#from nova import objectstore # for flags + FLAGS = flags.FLAGS # NOTE(vish): a user with one of these roles will be a superuser and @@ -99,6 +101,7 @@ class AuthBase(object): class User(AuthBase): """Object representing a user""" def __init__(self, id, name, access, secret, admin): + AuthBase.__init__(self) self.id = id self.name = name self.access = access @@ -159,6 +162,7 @@ class KeyPair(AuthBase): fingerprint is stored. The user's private key is not saved. """ def __init__(self, id, name, owner_id, public_key, fingerprint): + AuthBase.__init__(self) self.id = id self.name = name self.owner_id = owner_id @@ -176,6 +180,7 @@ class KeyPair(AuthBase): class Project(AuthBase): """Represents a Project returned from the datastore""" def __init__(self, id, name, project_manager_id, description, member_ids): + AuthBase.__init__(self) self.id = id self.name = name self.project_manager_id = project_manager_id @@ -234,7 +239,7 @@ class AuthManager(object): AuthManager also manages associated data related to Auth objects that need to be more accessible, such as vpn ips and ports. """ - _instance=None + _instance = None def __new__(cls, *args, **kwargs): """Returns the AuthManager singleton""" if not cls._instance: @@ -248,7 +253,7 @@ class AuthManager(object): reset the driver if it is not set or a new driver is specified. """ if driver or not getattr(self, 'driver', None): - self.driver = utils.import_class(driver or FLAGS.auth_driver) + self.driver = utils.import_class(driver or FLAGS.auth_driver) def authenticate(self, access, signature, params, verb='GET', server_string='127.0.0.1:8773', path='/', -- cgit From a31fe618b94f87cf03a090db04dace732c58951c Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Mon, 9 Aug 2010 09:47:08 -0400 Subject: pylint fixes for /nova/test.py --- nova/test.py | 74 +++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/nova/test.py b/nova/test.py index 6fbcab5e4..820cdda56 100644 --- a/nova/test.py +++ b/nova/test.py @@ -1,4 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 +# pylint: disable-msg=C0103 +# pylint: disable-msg=W0511 # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -22,15 +24,14 @@ Allows overriding of flags for use of fakes, and some black magic for inline callbacks. """ -import logging import mox import stubout +import sys import time -import unittest + from tornado import ioloop from twisted.internet import defer -from twisted.python import failure -from twisted.trial import unittest as trial_unittest +from twisted.trial import unittest from nova import fakerabbit from nova import flags @@ -41,20 +42,21 @@ flags.DEFINE_bool('fake_tests', True, 'should we use everything for testing') -def skip_if_fake(f): +def skip_if_fake(func): + """Decorator that skips a test if running in fake mode""" def _skipper(*args, **kw): + """Wrapped skipper function""" if FLAGS.fake_tests: - raise trial_unittest.SkipTest('Test cannot be run in fake mode') + raise unittest.SkipTest('Test cannot be run in fake mode') else: - return f(*args, **kw) - - _skipper.func_name = f.func_name + return func(*args, **kw) return _skipper -class TrialTestCase(trial_unittest.TestCase): - +class TrialTestCase(unittest.TestCase): + """Test case base class for all unit tests""" def setUp(self): + """Run before each test method to initialize test environment""" super(TrialTestCase, self).setUp() # emulate some of the mox stuff, we can't use the metaclass @@ -64,6 +66,7 @@ class TrialTestCase(trial_unittest.TestCase): self.flag_overrides = {} def tearDown(self): + """Runs after each test method to finalize/tear down test environment""" super(TrialTestCase, self).tearDown() self.reset_flags() self.mox.UnsetStubs() @@ -75,6 +78,7 @@ class TrialTestCase(trial_unittest.TestCase): fakerabbit.reset_all() def flags(self, **kw): + """Override flag variables for a test""" for k, v in kw.iteritems(): if k in self.flag_overrides: self.reset_flags() @@ -84,13 +88,17 @@ class TrialTestCase(trial_unittest.TestCase): setattr(FLAGS, k, v) def reset_flags(self): + """Resets all flag variables for the test. Runs after each test""" for k, v in self.flag_overrides.iteritems(): setattr(FLAGS, k, v) class BaseTestCase(TrialTestCase): - def setUp(self): + # TODO(jaypipes): Can this be moved into the TrialTestCase class? + """Base test case class for all unit tests.""" + def setUp(self): # pylint: disable-msg=W0511 + """Run before each test method to initialize test environment""" super(BaseTestCase, self).setUp() # TODO(termie): we could possibly keep a more global registry of # the injected listeners... this is fine for now though @@ -98,33 +106,27 @@ class BaseTestCase(TrialTestCase): self.ioloop = ioloop.IOLoop.instance() self._waiting = None - self._doneWaiting = False - self._timedOut = False - self.set_up() - - def set_up(self): - pass - - def tear_down(self): - pass + self._done_waiting = False + self._timed_out = False def tearDown(self): + """Runs after each test method to finalize/tear down test environment""" super(BaseTestCase, self).tearDown() for x in self.injected: x.stop() if FLAGS.fake_rabbit: fakerabbit.reset_all() - self.tear_down() def _waitForTest(self, timeout=60): """ Push the ioloop along to wait for our test to complete. """ self._waiting = self.ioloop.add_timeout(time.time() + timeout, self._timeout) def _wait(): - if self._timedOut: + """Wrapped wait function. Called on timeout.""" + if self._timed_out: self.fail('test timed out') self._done() - if self._doneWaiting: + if self._done_waiting: self.ioloop.stop() return # we can use add_callback here but this uses less cpu when testing @@ -134,13 +136,16 @@ class BaseTestCase(TrialTestCase): self.ioloop.start() def _done(self): + """Callback used for cleaning up deferred test methods.""" if self._waiting: try: self.ioloop.remove_timeout(self._waiting) - except Exception: + except Exception: # pylint: disable-msg=W0703 + # TODO(jaypipes): This produces a pylint warning. Should + # we really be catching Exception and then passing here? pass self._waiting = None - self._doneWaiting = True + self._done_waiting = True def _maybeInlineCallbacks(self, f): """ If we're doing async calls in our tests, wait on them. @@ -189,6 +194,7 @@ class BaseTestCase(TrialTestCase): return d def _catchExceptions(self, result, failure): + """Catches all exceptions and handles keyboard interrupts.""" exc = (failure.type, failure.value, failure.getTracebackObject()) if isinstance(failure.value, self.failureException): result.addFailure(self, exc) @@ -200,11 +206,12 @@ class BaseTestCase(TrialTestCase): self._done() def _timeout(self): + """Helper method which trips the timeouts""" self._waiting = False - self._timedOut = True + self._timed_out = True def run(self, result=None): - if result is None: result = self.defaultTestResult() + """Runs the test case""" result.startTest(self) testMethod = getattr(self, self._testMethodName) @@ -214,7 +221,7 @@ class BaseTestCase(TrialTestCase): except KeyboardInterrupt: raise except: - result.addError(self, self._exc_info()) + result.addError(self, sys.exc_info()) return ok = False @@ -225,19 +232,20 @@ class BaseTestCase(TrialTestCase): self._waitForTest() ok = True except self.failureException: - result.addFailure(self, self._exc_info()) + result.addFailure(self, sys.exc_info()) except KeyboardInterrupt: raise except: - result.addError(self, self._exc_info()) + result.addError(self, sys.exc_info()) try: self.tearDown() except KeyboardInterrupt: raise except: - result.addError(self, self._exc_info()) + result.addError(self, sys.exc_info()) ok = False - if ok: result.addSuccess(self) + if ok: + result.addSuccess(self) finally: result.stopTest(self) -- cgit From 850acbdd9521cda8600235259fa68b8f2ab488ef Mon Sep 17 00:00:00 2001 From: Eric Day Date: Mon, 9 Aug 2010 07:31:41 -0700 Subject: Fixed docstring format per Jay's review. --- nova/endpoint/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index f6735a260..75b828e91 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -105,7 +105,7 @@ class CloudServerAPI(wsgi.Application): return json.dumps(value) def instance_details(self, inst): # pylint: disable-msg=R0201 - "Build the data structure to represent details for an instance." + """Build the data structure to represent details for an instance.""" return { "id": inst.get("instance_id", None), "imageId": inst.get("image_id", None), -- cgit From 2a069abf622029c3f3e7273ad1cc3fb17b529e63 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Mon, 9 Aug 2010 10:46:33 -0400 Subject: pylint fixes for nova/server.py --- nova/server.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/server.py b/nova/server.py index 7a1901a2f..96550f078 100644 --- a/nova/server.py +++ b/nova/server.py @@ -52,13 +52,8 @@ def stop(pidfile): """ # Get the pid from the pidfile try: - pf = file(pidfile,'r') - pid = int(pf.read().strip()) - pf.close() + pid = int(open(pidfile,'r').read().strip()) except IOError: - pid = None - - if not pid: message = "pidfile %s does not exist. Daemon not running?\n" sys.stderr.write(message % pidfile) return # not an error in a restart @@ -79,14 +74,15 @@ def stop(pidfile): def serve(name, main): + """Controller for server""" argv = FLAGS(sys.argv) if not FLAGS.pidfile: FLAGS.pidfile = '%s.pid' % name - logging.debug("Full set of FLAGS: \n\n\n" ) + logging.debug("Full set of FLAGS: \n\n\n") for flag in FLAGS: - logging.debug("%s : %s" % (flag, FLAGS.get(flag, None) )) + logging.debug("%s : %s", flag, FLAGS.get(flag, None)) action = 'start' if len(argv) > 1: @@ -102,7 +98,11 @@ def serve(name, main): else: print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) + daemonize(argv, name, main) + +def daemonize(args, name, main): + """Does the work of daemonizing the process""" logging.getLogger('amqplib').setLevel(logging.WARN) if FLAGS.daemonize: logger = logging.getLogger() @@ -115,7 +115,7 @@ def serve(name, main): else: if not FLAGS.logfile: FLAGS.logfile = '%s.log' % name - logfile = logging.handlers.FileHandler(FLAGS.logfile) + logfile = logging.FileHandler(FLAGS.logfile) logfile.setFormatter(formatter) logger.addHandler(logfile) stdin, stdout, stderr = None, None, None @@ -137,4 +137,4 @@ def serve(name, main): stdout=stdout, stderr=stderr ): - main(argv) + main(args) -- cgit From 948162e3bdd96cdbe5db9a0c25722ac63c04e264 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Mon, 9 Aug 2010 12:20:47 -0400 Subject: Disables warning about TODO in code comments in pylintrc --- nova/test.py | 1 - pylintrc | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/test.py b/nova/test.py index 820cdda56..966cbf5fc 100644 --- a/nova/test.py +++ b/nova/test.py @@ -1,6 +1,5 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # pylint: disable-msg=C0103 -# pylint: disable-msg=W0511 # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. diff --git a/pylintrc b/pylintrc index a853e5bed..36ec7b346 100644 --- a/pylintrc +++ b/pylintrc @@ -1,6 +1,10 @@ [Basic] method-rgx=[a-z_][a-z0-9_]{2,50}$ +[MESSAGES CONTROL] +# TODOs in code comments are fine... +disable-msg=W0511 + [Design] max-public-methods=100 min-public-methods=0 -- cgit From c6c222800ccc1203fc0edd7716dd808ec8f6bdc4 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Mon, 9 Aug 2010 12:37:15 -0400 Subject: Fix up variable names instead of disabling pylint naming rule. Makes variables able to be a single letter in pylintrc --- nova/test.py | 29 ++++++++++++++--------------- pylintrc | 6 ++++++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/nova/test.py b/nova/test.py index 966cbf5fc..c7e08734f 100644 --- a/nova/test.py +++ b/nova/test.py @@ -1,5 +1,4 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# pylint: disable-msg=C0103 # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -54,7 +53,7 @@ def skip_if_fake(func): class TrialTestCase(unittest.TestCase): """Test case base class for all unit tests""" - def setUp(self): + def setUp(self): # pylint: disable-msg=C0103 """Run before each test method to initialize test environment""" super(TrialTestCase, self).setUp() @@ -64,7 +63,7 @@ class TrialTestCase(unittest.TestCase): self.stubs = stubout.StubOutForTesting() self.flag_overrides = {} - def tearDown(self): + def tearDown(self): # pylint: disable-msg=C0103 """Runs after each test method to finalize/tear down test environment""" super(TrialTestCase, self).tearDown() self.reset_flags() @@ -96,7 +95,7 @@ class TrialTestCase(unittest.TestCase): class BaseTestCase(TrialTestCase): # TODO(jaypipes): Can this be moved into the TrialTestCase class? """Base test case class for all unit tests.""" - def setUp(self): # pylint: disable-msg=W0511 + def setUp(self): # pylint: disable-msg=C0103 """Run before each test method to initialize test environment""" super(BaseTestCase, self).setUp() # TODO(termie): we could possibly keep a more global registry of @@ -108,7 +107,7 @@ class BaseTestCase(TrialTestCase): self._done_waiting = False self._timed_out = False - def tearDown(self): + def tearDown(self):# pylint: disable-msg=C0103 """Runs after each test method to finalize/tear down test environment""" super(BaseTestCase, self).tearDown() for x in self.injected: @@ -116,7 +115,7 @@ class BaseTestCase(TrialTestCase): if FLAGS.fake_rabbit: fakerabbit.reset_all() - def _waitForTest(self, timeout=60): + def _wait_for_test(self, timeout=60): """ Push the ioloop along to wait for our test to complete. """ self._waiting = self.ioloop.add_timeout(time.time() + timeout, self._timeout) @@ -146,7 +145,7 @@ class BaseTestCase(TrialTestCase): self._waiting = None self._done_waiting = True - def _maybeInlineCallbacks(self, f): + def _maybe_inline_callbacks(self, func): """ If we're doing async calls in our tests, wait on them. This is probably the most complicated hunk of code we have so far. @@ -169,7 +168,7 @@ class BaseTestCase(TrialTestCase): d.addCallback(_describe) d.addCallback(_checkDescribe) d.addCallback(lambda x: self._done()) - self._waitForTest() + self._wait_for_test() Example (inline callbacks! yay!): @@ -183,16 +182,16 @@ class BaseTestCase(TrialTestCase): # TODO(termie): this can be a wrapper function instead and # and we can make a metaclass so that we don't # have to copy all that "run" code below. - g = f() + g = func() if not hasattr(g, 'send'): self._done() return defer.succeed(g) - inlined = defer.inlineCallbacks(f) + inlined = defer.inlineCallbacks(func) d = inlined() return d - def _catchExceptions(self, result, failure): + def _catch_exceptions(self, result, failure): """Catches all exceptions and handles keyboard interrupts.""" exc = (failure.type, failure.value, failure.getTracebackObject()) if isinstance(failure.value, self.failureException): @@ -213,7 +212,7 @@ class BaseTestCase(TrialTestCase): """Runs the test case""" result.startTest(self) - testMethod = getattr(self, self._testMethodName) + test_method = getattr(self, self._testMethodName) try: try: self.setUp() @@ -225,10 +224,10 @@ class BaseTestCase(TrialTestCase): ok = False try: - d = self._maybeInlineCallbacks(testMethod) - d.addErrback(lambda x: self._catchExceptions(result, x)) + d = self._maybe_inline_callbacks(test_method) + d.addErrback(lambda x: self._catch_exceptions(result, x)) d.addBoth(lambda x: self._done() and x) - self._waitForTest() + self._wait_for_test() ok = True except self.failureException: result.addFailure(self, sys.exc_info()) diff --git a/pylintrc b/pylintrc index 36ec7b346..045d05939 100644 --- a/pylintrc +++ b/pylintrc @@ -1,4 +1,10 @@ [Basic] +# Variables can be 1 to 31 characters long, with +# lowercase and underscores +variable-rgx=[a-z_][a-z0-9_]{0,30}$ + +# Method names should be at least 3 characters long +# and be lowecased with underscores method-rgx=[a-z_][a-z0-9_]{2,50}$ [MESSAGES CONTROL] -- cgit From 09b5be11e7b61aa0ae344cec669e4f62dd18c0b2 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 9 Aug 2010 17:45:00 -0400 Subject: Run correctly even if called while in tools/ directory, as 'python install_venv.py' --- tools/install_venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/install_venv.py b/tools/install_venv.py index 0b35fc8e9..96bb12efb 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -7,7 +7,7 @@ import subprocess import sys -ROOT = os.path.dirname(os.path.dirname(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) VENV = os.path.join(ROOT, '.nova-venv') PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') TWISTED_NOVA='http://nova.openstack.org/Twisted-10.0.0Nova.tar.gz' -- cgit From 8990a62b0e654dcacac06246733a17fa0502bcc7 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 9 Aug 2010 17:53:10 -0700 Subject: fixing - removing unused imports per Eric & Jay review --- nova/auth/manager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index e5efbca24..6d71a7ad6 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -35,10 +35,6 @@ from nova import utils from nova.auth import signer from nova.network import vpn -#unused imports -#from nova import datastore -#from nova.auth import ldapdriver # for flags -#from nova import objectstore # for flags FLAGS = flags.FLAGS -- cgit From 8c7558ed5ae7dd0b78a91a385dbd9b044ec7c8db Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Tue, 10 Aug 2010 12:44:38 -0400 Subject: Changes the run_tests.sh and /tools/install_venv.py scripts to be more user-friendly and not depend on PIP while not in the virtual environment. Running run_tests.sh should not just work out of the box on all systems supporting easy_install... --- run_tests.sh | 7 +++--- tools/install_venv.py | 59 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 9b2de7aea..85d7c8834 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,8 +6,7 @@ with_venv=tools/with_venv.sh if [ -e ${venv} ]; then ${with_venv} python run_tests.py $@ else - echo "You need to install the Nova virtualenv before you can run this." - echo "" - echo "Please run tools/install_venv.py" - exit 1 + echo "No virtual environment found...creating one" + python tools/install_venv.py + ${with_venv} python run_tests.py $@ fi diff --git a/tools/install_venv.py b/tools/install_venv.py index 0b35fc8e9..adf24b365 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -1,3 +1,23 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + """ Installation script for Nova's development virtualenv """ @@ -12,15 +32,14 @@ VENV = os.path.join(ROOT, '.nova-venv') PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') TWISTED_NOVA='http://nova.openstack.org/Twisted-10.0.0Nova.tar.gz' - def die(message, *args): print >>sys.stderr, message % args sys.exit(1) - def run_command(cmd, redirect_output=True, error_ok=False): - # Useful for debugging: - #print >>sys.stderr, ' '.join(cmd) + """Runs a command in an out-of-process shell, returning the + output of that command + """ if redirect_output: stdout = subprocess.PIPE else: @@ -32,33 +51,43 @@ def run_command(cmd, redirect_output=True, error_ok=False): die('Command "%s" failed.\n%s', ' '.join(cmd), output) return output +HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install']).strip()) +HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv']).strip()) def check_dependencies(): - """Make sure pip and virtualenv are on the path.""" - print 'Checking for pip...', - if not run_command(['which', 'pip']).strip(): - die('ERROR: pip not found.\n\nNova development requires pip,' - ' please install it using your favorite package management tool') - print 'done.' + """Make sure virtualenv is in the path.""" print 'Checking for virtualenv...', - if not run_command(['which', 'virtualenv']).strip(): - die('ERROR: virtualenv not found.\n\nNova development requires virtualenv,' - ' please install it using your favorite package management tool') + if not HAS_VIRTUALENV: + print 'not found.' + # Try installing it via easy_install... + if HAS_EASY_INSTALL: + if not run_command(['which', 'easy_install']): + print 'Installing virtualenv via easy_install...', + die('ERROR: virtualenv not found.\n\nNova development requires virtualenv,' + ' please install it using your favorite package management tool') + print 'done.' print 'done.' def create_virtualenv(venv=VENV): + """Creates the virtual environment and installs PIP only into the + virtual environment + """ print 'Creating venv...', run_command(['virtualenv', '-q', '--no-site-packages', VENV]) print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', 'pip']).strip(): + die("Failed to install pip.") + print 'done.' def install_dependencies(venv=VENV): print 'Installing dependencies with pip (this can take a while)...' - run_command(['pip', 'install', '-E', venv, '-r', PIP_REQUIRES], + run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, '-r', PIP_REQUIRES], redirect_output=False) - run_command(['pip', 'install', '-E', venv, TWISTED_NOVA], + run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, TWISTED_NOVA], redirect_output=False) -- cgit From f5695429db27110d8a95df3b66e4045c59d88c6a Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Tue, 10 Aug 2010 12:51:03 -0400 Subject: Quick fix on location of printouts when trying to install virtualenv. --- tools/install_venv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/install_venv.py b/tools/install_venv.py index adf24b365..494535b5e 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -36,6 +36,7 @@ def die(message, *args): print >>sys.stderr, message % args sys.exit(1) + def run_command(cmd, redirect_output=True, error_ok=False): """Runs a command in an out-of-process shell, returning the output of that command @@ -51,9 +52,11 @@ def run_command(cmd, redirect_output=True, error_ok=False): die('Command "%s" failed.\n%s', ' '.join(cmd), output) return output + HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install']).strip()) HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv']).strip()) + def check_dependencies(): """Make sure virtualenv is in the path.""" @@ -62,8 +65,8 @@ def check_dependencies(): print 'not found.' # Try installing it via easy_install... if HAS_EASY_INSTALL: + print 'Installing virtualenv via easy_install...', if not run_command(['which', 'easy_install']): - print 'Installing virtualenv via easy_install...', die('ERROR: virtualenv not found.\n\nNova development requires virtualenv,' ' please install it using your favorite package management tool') print 'done.' -- cgit From 7a1709561f1fed6e46a1c31aaa8e3ac54b9eebd3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 10:25:52 -0700 Subject: rename create_zip to zipfile so lazy match works --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 2dd569df0..6af092922 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -203,7 +203,7 @@ class ProjectCommands(object): arguments: project user""" self.manager.remove_from_project(user, project) - def create_zip(self, project_id, user_id, filename='nova.zip'): + def zipfile(self, project_id, user_id, filename='nova.zip'): """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" zip_file = self.manager.get_credentials(project_id, user_id) -- cgit From 538fe868a8c89f892bffbfc0001b64e3bf1c9cf5 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 10 Aug 2010 15:28:35 -0400 Subject: Oops, we need eventlet as well. --- tools/pip-requires | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/pip-requires b/tools/pip-requires index 4eb47ca2b..e3591e92d 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -4,6 +4,7 @@ amqplib==0.6.1 anyjson==0.2.4 boto==2.0b1 carrot==0.10.5 +eventlet==0.9.10 lockfile==0.8 python-daemon==1.5.5 python-gflags==1.3 -- cgit From 14c7bca9cb8451e2ec8224fb5699c6f2ad480dac Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 17:34:20 -0700 Subject: Adds get_roles commands to manager and driver classes --- nova/auth/ldapdriver.py | 34 +++++++++++++++++++++++++++------- nova/auth/manager.py | 18 ++++++++++++++++++ nova/tests/auth_unittest.py | 18 +++++++++++++++++- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ec739e134..aaaf8553c 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -181,7 +181,7 @@ class LdapDriver(object): if member_uids != None: for member_uid in member_uids: if not self.__user_exists(member_uid): - raise exception.NotFound("Project can't be created " + raise exception.NotFound("Project can't be created " "because user %s doesn't exist" % member_uid) members.append(self.__uid_to_dn(member_uid)) # always add the manager as a member because members is required @@ -236,6 +236,26 @@ class LdapDriver(object): role_dn = self.__role_to_dn(role, project_id) return self.__remove_from_group(uid, role_dn) + def get_user_roles(self, uid, project_id=None): + """Retrieve list of roles for user (or user and project)""" + if project_id is None: + # NOTE(vish): This is unneccesarily slow, but since we can't + # guarantee that the global roles are located + # together in the ldap tree, we're doing this version. + roles = [] + for role in FLAGS.allowed_roles: + role_dn = self.__role_to_dn(role) + if self.__is_in_group(uid, role_dn): + roles.append(role) + return roles + else: + project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) + roles = self.__find_objects(project_dn, + '(&(&(objectclass=groupOfNames)' + '(!(objectclass=novaProject)))' + '(member=%s))' % self.__uid_to_dn(uid)) + return [role['cn'][0] for role in roles] + def delete_user(self, uid): """Delete a user""" if not self.__user_exists(uid): @@ -253,24 +273,24 @@ class LdapDriver(object): self.conn.delete_s('cn=%s,uid=%s,%s' % (key_name, uid, FLAGS.ldap_user_subtree)) - def delete_project(self, name): + def delete_project(self, project_id): """Delete a project""" - project_dn = 'cn=%s,%s' % (name, FLAGS.ldap_project_subtree) + project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) self.__delete_roles(project_dn) self.__delete_group(project_dn) - def __user_exists(self, name): + def __user_exists(self, uid): """Check if user exists""" - return self.get_user(name) != None + return self.get_user(uid) != None def __key_pair_exists(self, uid, key_name): """Check if key pair exists""" return self.get_user(uid) != None return self.get_key_pair(uid, key_name) != None - def __project_exists(self, name): + def __project_exists(self, project_id): """Check if project exists""" - return self.get_project(name) != None + return self.get_project(project_id) != None def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 6d71a7ad6..8195182fc 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -38,6 +38,10 @@ from nova.network import vpn FLAGS = flags.FLAGS +flags.DEFINE_list('allowed_roles', + ['cloudadmin', 'itsec', 'sysadmin', 'netadmin', 'developer'], + 'Allowed roles for project') + # NOTE(vish): a user with one of these roles will be a superuser and # have access to all api commands flags.DEFINE_list('superuser_roles', ['cloudadmin'], @@ -455,6 +459,20 @@ class AuthManager(object): with self.driver() as drv: drv.remove_role(User.safe_id(user), role, Project.safe_id(project)) + def get_roles(self): + """Get list of allowed roles""" + return FLAGS.allowed_roles + + def get_user_roles(self, user, project=None): + """Get user global or per-project roles""" + roles = [] + with self.driver() as drv: + roles = drv.get_user_roles(User.safe_id(user), + Project.safe_id(project)) + if project is not None and self.is_project_manager(user, project): + roles.append('projectmanager') + return roles + def get_project(self, pid): """Get project object by id""" with self.driver() as drv: diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index f7e0625a3..2d99c8e36 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -179,7 +179,23 @@ class AuthTestCase(test.BaseTestCase): project.add_role('test1', 'sysadmin') self.assertTrue(project.has_role('test1', 'sysadmin')) - def test_211_can_remove_project_role(self): + def test_211_can_list_project_roles(self): + project = self.manager.get_project('testproj') + user = self.manager.get_user('test1') + self.manager.add_role(user, 'netadmin', project) + roles = self.manager.get_user_roles(user) + self.assertTrue('sysadmin' in roles) + self.assertFalse('netadmin' in roles) + self.assertFalse('projectmanager' in roles) + project_roles = self.manager.get_user_roles(user, project) + self.assertTrue('sysadmin' in project_roles) + self.assertTrue('netadmin' in project_roles) + self.assertTrue('projectmanager' in project_roles) + # has role should be false because global role is missing + self.assertFalse(self.manager.has_role(user, 'netadmin', project)) + + + def test_212_can_remove_project_role(self): project = self.manager.get_project('testproj') self.assertTrue(project.has_role('test1', 'sysadmin')) project.remove_role('test1', 'sysadmin') -- cgit From 19b9164c4eaae0c2c9144f9e839fbafcac7c3ed3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 17:42:58 -0700 Subject: Throw exceptions for illegal roles on role add --- nova/auth/manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 8195182fc..e338dfc83 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -436,6 +436,10 @@ class AuthManager(object): @type project: Project or project_id @param project: Project in which to add local role. """ + if role not in FLAGS.allowed_roles: + raise exception.NotFound("The %s role can not be found" % role) + if project is not None and role in FLAGS.global_roles: + raise exception.NotFound("The %s role is global only" % role) with self.driver() as drv: drv.add_role(User.safe_id(user), role, Project.safe_id(project)) -- cgit From cff3cccc342c7d09cd2ec6c95431e1b373eba620 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 18:04:23 -0700 Subject: change get_roles to have a flag for project_roles or not. Don't show 'projectmanager' in list of roles --- nova/auth/manager.py | 15 +++++++-------- nova/tests/auth_unittest.py | 2 -- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index e338dfc83..064fd78bc 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -463,19 +463,18 @@ class AuthManager(object): with self.driver() as drv: drv.remove_role(User.safe_id(user), role, Project.safe_id(project)) - def get_roles(self): + def get_roles(self, project_roles=True): """Get list of allowed roles""" - return FLAGS.allowed_roles + if project_roles: + return list(set(FLAGS.allowed_roles) - set(FLAGS.global_roles)) + else: + return FLAGS.allowed_roles def get_user_roles(self, user, project=None): """Get user global or per-project roles""" - roles = [] with self.driver() as drv: - roles = drv.get_user_roles(User.safe_id(user), - Project.safe_id(project)) - if project is not None and self.is_project_manager(user, project): - roles.append('projectmanager') - return roles + return drv.get_user_roles(User.safe_id(user), + Project.safe_id(project)) def get_project(self, pid): """Get project object by id""" diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 2d99c8e36..0b404bfdc 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -186,11 +186,9 @@ class AuthTestCase(test.BaseTestCase): roles = self.manager.get_user_roles(user) self.assertTrue('sysadmin' in roles) self.assertFalse('netadmin' in roles) - self.assertFalse('projectmanager' in roles) project_roles = self.manager.get_user_roles(user, project) self.assertTrue('sysadmin' in project_roles) self.assertTrue('netadmin' in project_roles) - self.assertTrue('projectmanager' in project_roles) # has role should be false because global role is missing self.assertFalse(self.manager.has_role(user, 'netadmin', project)) -- cgit From 2955018b58a731f48dcdee64d889b4be104250f1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 19:00:35 -0700 Subject: fix spacing issue in ldapdriver --- nova/auth/ldapdriver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index aaaf8553c..453fa196c 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -181,8 +181,9 @@ class LdapDriver(object): if member_uids != None: for member_uid in member_uids: if not self.__user_exists(member_uid): - raise exception.NotFound("Project can't be created " - "because user %s doesn't exist" % member_uid) + raise exception.NotFound("Project can't be created " + "because user %s doesn't exist" + % member_uid) members.append(self.__uid_to_dn(member_uid)) # always add the manager as a member because members is required if not manager_dn in members: -- cgit