From 843644aed6477b4411ec3f07d1a5271df41c9798 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sat, 18 Jun 2011 23:10:41 -0400 Subject: General cleanup and refactor of a lot of the API/WSGI service code. --- bin/nova-api | 18 +++++- nova/api/openstack/wsgi.py | 4 +- nova/log.py | 13 ++++ nova/service.py | 96 +++++++++++------------------ nova/test.py | 23 ------- nova/tests/integrated/integrated_helpers.py | 13 ++-- nova/wsgi.py | 63 +++++++++---------- 7 files changed, 104 insertions(+), 126 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a1088c23d..6db68be9c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -24,6 +24,8 @@ import gettext import os import sys +import eventlet.pool + # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), @@ -46,6 +48,13 @@ LOG = logging.getLogger('nova.api') FLAGS = flags.FLAGS + +def launch(service_name): + _service = service.WSGIService(service_name) + _service.start() + _service.wait() + + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) @@ -57,5 +66,10 @@ if __name__ == '__main__': flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - service = service.serve_wsgi(service.ApiService) - service.wait() + + pool = eventlet.pool.Pool() + pool.execute(launch, "ec2") + pool.execute(launch, "osapi") + pool.wait_all() + + print >>sys.stderr, "Exiting..." diff --git a/nova/api/openstack/wsgi.py b/nova/api/openstack/wsgi.py index a57b7f72b..6033c1f5b 100644 --- a/nova/api/openstack/wsgi.py +++ b/nova/api/openstack/wsgi.py @@ -356,7 +356,7 @@ class Resource(wsgi.Application): def __call__(self, request): """WSGI method that controls (de)serialization and method dispatch.""" - LOG.debug("%(method)s %(url)s" % {"method": request.method, + LOG.info("%(method)s %(url)s" % {"method": request.method, "url": request.url}) try: @@ -384,7 +384,7 @@ class Resource(wsgi.Application): msg_dict = dict(url=request.url, e=e) msg = _("%(url)s returned a fault: %(e)s" % msg_dict) - LOG.debug(msg) + LOG.info(msg) return response diff --git a/nova/log.py b/nova/log.py index 6909916a1..d3bffdb21 100644 --- a/nova/log.py +++ b/nova/log.py @@ -314,3 +314,16 @@ logging.setLoggerClass(NovaLogger) def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" logging.root.log(AUDIT, msg, *args, **kwargs) + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.DEBUG): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + diff --git a/nova/service.py b/nova/service.py index 74f9f04d8..768390414 100644 --- a/nova/service.py +++ b/nova/service.py @@ -232,45 +232,45 @@ class Service(object): logging.exception(_('model server went away')) -class WsgiService(object): - """Base class for WSGI based services. - - For each api you define, you must also define these flags: - :_listen: The address on which to listen - :_listen_port: The port on which to listen - - """ - - def __init__(self, conf, apis): - self.conf = conf - self.apis = apis - self.wsgi_app = None +class WSGIService(object): + """Provides ability to launch API from a 'paste' configuration.""" + + def __init__(self, name, config_name=None): + """Initialize, but do not start, an API service.""" + self.name = name + self._config_name = config_name or FLAGS.api_paste_config + self._config_location = self._find_config() + self._config = self._load_config() + self.application = self._load_application() + host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") + port = getattr(FLAGS, '%s_listen_port' % name, 0) + self.server = wsgi.Server(name, self.application, host, port) + + def _find_config(self): + """Attempt to find 'paste' configuration file.""" + location = wsgi.paste_config_file(self._config_name) + logging.debug(_("Using paste.deploy config at: %s"), location) + return location + + def _load_config(self): + """Read and return the 'paste' configuration file.""" + return wsgi.load_paste_configuration(self._config_location, self.name) + + def _load_application(self): + """Using the loaded configuration, return the WSGI application.""" + return wsgi.load_paste_app(self._config_location, self.name) def start(self): - self.wsgi_app = _run_wsgi(self.conf, self.apis) - - def wait(self): - self.wsgi_app.wait() - - def get_socket_info(self, api_name): - """Returns the (host, port) that an API was started on.""" - return self.wsgi_app.socket_info[api_name] + """Start serving this API using loaded configuration.""" + self.server.start() + def stop(self): + """Stop serving this API.""" + self.server.stop() -class ApiService(WsgiService): - """Class for our nova-api service.""" - - @classmethod - def create(cls, conf=None): - if not conf: - conf = wsgi.paste_config_file(FLAGS.api_paste_config) - if not conf: - message = (_('No paste configuration found for: %s'), - FLAGS.api_paste_config) - raise exception.Error(message) - api_endpoints = ['ec2', 'osapi'] - service = cls(conf, api_endpoints) - return service + def wait(self): + """Wait for the service to stop serving this API.""" + self.server.wait() def serve(*services): @@ -321,29 +321,3 @@ def serve_wsgi(cls, conf=None): service.start() return service - - -def _run_wsgi(paste_config_file, apis): - logging.debug(_('Using paste.deploy config at: %s'), paste_config_file) - apps = [] - for api in apis: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - logging.debug(_('No paste configuration for app: %s'), api) - continue - logging.debug(_('App Config: %(api)s\n%(config)r') % locals()) - logging.info(_('Running %s API'), api) - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, - getattr(FLAGS, '%s_listen_port' % api), - getattr(FLAGS, '%s_listen' % api), - api)) - if len(apps) == 0: - logging.error(_('No known API applications configured in %s.'), - paste_config_file) - return - - server = wsgi.Server() - for app in apps: - server.start(*app) - return server diff --git a/nova/test.py b/nova/test.py index 4a0a18fe7..ab1eaf5fd 100644 --- a/nova/test.py +++ b/nova/test.py @@ -38,7 +38,6 @@ from nova import flags from nova import rpc from nova import utils from nova import service -from nova import wsgi from nova.virt import fake @@ -81,7 +80,6 @@ class TestCase(unittest.TestCase): self.injected = [] self._services = [] self._monkey_patch_attach() - self._monkey_patch_wsgi() self._original_flags = FLAGS.FlagValuesDict() rpc.ConnectionPool = rpc.Pool(max_size=FLAGS.rpc_conn_pool_size) @@ -107,7 +105,6 @@ class TestCase(unittest.TestCase): # Reset our monkey-patches rpc.Consumer.attach_to_eventlet = self.original_attach - wsgi.Server.start = self.original_start # Stop any timers for x in self.injected: @@ -163,26 +160,6 @@ class TestCase(unittest.TestCase): _wrapped.func_name = self.original_attach.func_name rpc.Consumer.attach_to_eventlet = _wrapped - def _monkey_patch_wsgi(self): - """Allow us to kill servers spawned by wsgi.Server.""" - self.original_start = wsgi.Server.start - - @functools.wraps(self.original_start) - def _wrapped_start(inner_self, *args, **kwargs): - original_spawn_n = inner_self.pool.spawn_n - - @functools.wraps(original_spawn_n) - def _wrapped_spawn_n(*args, **kwargs): - rv = greenthread.spawn(*args, **kwargs) - self._services.append(rv) - - inner_self.pool.spawn_n = _wrapped_spawn_n - self.original_start(inner_self, *args, **kwargs) - inner_self.pool.spawn_n = original_spawn_n - - _wrapped_start.func_name = self.original_start.func_name - wsgi.Server.start = _wrapped_start - # Useful assertions def assertDictMatch(self, d1, d2, approx_equal=False, tolerance=0.001): """Assert two dicts are equivalent. diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py index 522c7cb0e..ba6bef62f 100644 --- a/nova/tests/integrated/integrated_helpers.py +++ b/nova/tests/integrated/integrated_helpers.py @@ -171,15 +171,14 @@ class _IntegratedTestBase(test.TestCase): self.api = self.user.openstack_api def _start_api_service(self): - api_service = service.ApiService.create() - api_service.start() + #ec2 = service.WSGIService("ec2") + #ec2.start() - if not api_service: - raise Exception("API Service was None") + osapi = service.WSGIService("osapi") + osapi.start() - self.api_service = api_service - - host, port = api_service.get_socket_info('osapi') + host = osapi.server.host + port = osapi.server.port self.auth_url = 'http://%s:%s/v1.1' % (host, port) def tearDown(self): diff --git a/nova/wsgi.py b/nova/wsgi.py index 33ba852bc..439ec6a12 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -43,46 +43,45 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.wsgi') -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.DEBUG): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg) - - class Server(object): """Server class to manage multiple WSGI sockets and applications.""" - def __init__(self, threads=1000): - self.pool = eventlet.GreenPool(threads) - self.socket_info = {} - - def start(self, application, port, host='0.0.0.0', key=None, backlog=128): - """Run a WSGI server with the given application.""" - arg0 = sys.argv[0] - logging.audit(_('Starting %(arg0)s on %(host)s:%(port)s') % locals()) - socket = eventlet.listen((host, port), backlog=backlog) - self.pool.spawn_n(self._run, application, socket) - if key: - self.socket_info[key] = socket.getsockname() + default_pool_size = 1000 + logger_name = "eventlet.wsgi.server" + + def __init__(self, name, app, host, port, pool_size=None): + self.name = name + self.app = app + self.host = host + self.port = port + self._pool = eventlet.GreenPool(pool_size or self.default_pool_size) + self._log = logging.WritableLogger(logging.getLogger(self.logger_name)) + + def _start(self, socket): + """Blocking eventlet WSGI server launched from the real 'start'.""" + eventlet.wsgi.server(socket, + self.app, + custom_pool=self._pool, + log=self._log) + + def start(self, backlog=128): + """Serve given WSGI application using the given parameters.""" + socket = eventlet.listen((self.host, self.port), backlog=backlog) + self._server = eventlet.spawn(self._start, socket) + (self.host, self.port) = socket.getsockname() + LOG.info(_('Starting %(app)s on %(host)s:%(port)s') % self.__dict__) + + def stop(self): + """Stop this server by killing the greenthread running it.""" + self._server.kill() def wait(self): - """Wait until all servers have completed running.""" + """Wait until server has been stopped.""" try: - self.pool.waitall() + self._server.wait() except KeyboardInterrupt: pass - def _run(self, application, socket): - """Start a WSGI server in a new green thread.""" - logger = logging.getLogger('eventlet.wsgi.server') - eventlet.wsgi.server(socket, application, custom_pool=self.pool, - log=WritableLogger(logger)) - class Request(webob.Request): pass @@ -340,6 +339,8 @@ def paste_config_file(basename): if os.path.exists(configfile): return configfile + raise Exception(_("Unable to find paste.deploy config '%s'") % basename) + def load_paste_configuration(filename, appname): """Returns a paste configuration dict, or None.""" -- cgit From ea64f883b74fa3c702a3c47d4508a1e7a7f6b40d Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 14:09:09 -0400 Subject: Removed debugging, made objectstore tests pass again. --- bin/nova-api | 3 --- nova/log.py | 2 -- nova/tests/test_objectstore.py | 6 ++++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 6db68be9c..90c8b69ad 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -66,10 +66,7 @@ if __name__ == '__main__': flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - pool = eventlet.pool.Pool() pool.execute(launch, "ec2") pool.execute(launch, "osapi") pool.wait_all() - - print >>sys.stderr, "Exiting..." diff --git a/nova/log.py b/nova/log.py index d3bffdb21..3080daee2 100644 --- a/nova/log.py +++ b/nova/log.py @@ -325,5 +325,3 @@ class WritableLogger(object): def write(self, msg): self.logger.log(self.level, msg) - - diff --git a/nova/tests/test_objectstore.py b/nova/tests/test_objectstore.py index c78772f27..c35955c9c 100644 --- a/nova/tests/test_objectstore.py +++ b/nova/tests/test_objectstore.py @@ -70,11 +70,12 @@ class S3APITestCase(test.TestCase): os.mkdir(FLAGS.buckets_path) router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server() - server.start(router, FLAGS.s3_port, host=FLAGS.s3_host) + self.server = wsgi.Server("s3api", router, FLAGS.s3_host, FLAGS.s3_port) + self.server.start() if not boto.config.has_section('Boto'): boto.config.add_section('Boto') + boto.config.set('Boto', 'num_retries', '0') conn = s3.S3Connection(aws_access_key_id=self.admin_user.access, aws_secret_access_key=self.admin_user.secret, @@ -145,4 +146,5 @@ class S3APITestCase(test.TestCase): """Tear down auth and test server.""" self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') + self.server.stop() super(S3APITestCase, self).tearDown() -- cgit From 95213244fe341b7ec2723b92a5b793e89ee8403f Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 14:41:42 -0400 Subject: Cleaned up nova-api binary and logging a bit. --- bin/nova-api | 32 +++++++++----------------------- nova/__init__.py | 12 ++++++++++++ nova/service.py | 19 ------------------- nova/utils.py | 2 ++ nova/wsgi.py | 2 +- setup.cfg | 3 +++ 6 files changed, 27 insertions(+), 43 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 90c8b69ad..7d80b0b78 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -20,33 +20,20 @@ """Starter script for Nova API.""" -import gettext -import os import sys import eventlet.pool -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - from nova import flags from nova import log as logging from nova import service from nova import utils from nova import version -from nova import wsgi -LOG = logging.getLogger('nova.api') - +LOG = logging.getLogger("nova.api") FLAGS = flags.FLAGS +VERSION = version.version_string_with_vcs() def launch(service_name): @@ -55,18 +42,17 @@ def launch(service_name): _service.wait() -if __name__ == '__main__': +def main(): utils.default_flagfile() FLAGS(sys.argv) - logging.setup() - LOG.audit(_("Starting nova-api node (version %s)"), - version.version_string_with_vcs()) - LOG.debug(_("Full set of FLAGS:")) - for flag in FLAGS: - flag_get = FLAGS.get(flag, None) - LOG.debug("%(flag)s : %(flag_get)s" % locals()) +# logging.setup() + LOG.audit(_("Starting nova-api node (version %s)") % VERSION) pool = eventlet.pool.Pool() pool.execute(launch, "ec2") pool.execute(launch, "osapi") pool.wait_all() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/nova/__init__.py b/nova/__init__.py index 256db55a9..7b05611b9 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -30,3 +30,15 @@ .. moduleauthor:: Manish Singh .. moduleauthor:: Andy Smith """ + +import gettext + +import log as logging + + +def initialize(): + gettext.install("nova", unicode=1) + logging.setup() + + +initialize() diff --git a/nova/service.py b/nova/service.py index 768390414..ec2de9004 100644 --- a/nova/service.py +++ b/nova/service.py @@ -302,22 +302,3 @@ def serve(*services): def wait(): while True: greenthread.sleep(5) - - -def serve_wsgi(cls, conf=None): - try: - service = cls.create(conf) - except Exception: - logging.exception('in WsgiService.create()') - raise - finally: - # After we've loaded up all our dynamic bits, check - # whether we should print help - flags.DEFINE_flag(flags.HelpFlag()) - flags.DEFINE_flag(flags.HelpshortFlag()) - flags.DEFINE_flag(flags.HelpXMLFlag()) - FLAGS.ParseNewFlags() - - service.start() - - return service diff --git a/nova/utils.py b/nova/utils.py index 691134ada..22d0ce940 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -238,6 +238,8 @@ def default_flagfile(filename='nova.conf'): filename = "./nova.conf" if not os.path.exists(filename): filename = '/etc/nova/nova.conf' + if not os.path.exists(filename): + return flagfile = ['--flagfile=%s' % filename] sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] diff --git a/nova/wsgi.py b/nova/wsgi.py index 439ec6a12..4da8eff82 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -69,7 +69,7 @@ class Server(object): socket = eventlet.listen((self.host, self.port), backlog=backlog) self._server = eventlet.spawn(self._start, socket) (self.host, self.port) = socket.getsockname() - LOG.info(_('Starting %(app)s on %(host)s:%(port)s') % self.__dict__) + LOG.info(_('Starting %(name)s on %(host)s:%(port)s') % self.__dict__) def stop(self): """Stop this server by killing the greenthread running it.""" diff --git a/setup.cfg b/setup.cfg index 9c0a331e3..7efc46f23 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ +[develop] +user = True + [build_sphinx] all_files = 1 build-dir = doc/build -- cgit From 1e047dae71131a0080310990dc6899852d233941 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 16:27:46 -0400 Subject: Further nova-api cleanup. --- bin/nova-api | 63 +++++++++++++++++++++++++++++++++----------------------- nova/__init__.py | 1 + nova/log.py | 2 +- nova/service.py | 2 +- nova/wsgi.py | 5 +---- 5 files changed, 41 insertions(+), 32 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 7d80b0b78..ff4aa83d1 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,7 +1,5 @@ #!/usr/bin/env python -# pylint: disable=C0103 -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - +# # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -18,40 +16,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Starter script for Nova API.""" +"""Starter script for Nova API. -import sys +Starts both the EC2 and OpenStack APIs in separate processes. Pylint warnings +about re-imports should be ignored. -import eventlet.pool +""" -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import version +# pylint: disable=W0404 +import sys +import multiprocessing -LOG = logging.getLogger("nova.api") -FLAGS = flags.FLAGS -VERSION = version.version_string_with_vcs() +import nova.flags +import nova.log +import nova.service +import nova.version +import nova.utils def launch(service_name): - _service = service.WSGIService(service_name) - _service.start() - _service.wait() + """Launch WSGI service with name matching 'paste' config file section.""" + service = nova.service.WSGIService(service_name) + service.start() + try: + service.wait() + except KeyboardInterrupt: + service.stop() def main(): - utils.default_flagfile() - FLAGS(sys.argv) -# logging.setup() - LOG.audit(_("Starting nova-api node (version %s)") % VERSION) - - pool = eventlet.pool.Pool() - pool.execute(launch, "ec2") - pool.execute(launch, "osapi") - pool.wait_all() + """Begin process of launching both EC2 and OSAPI services.""" + version = nova.version.version_string_with_vcs() + logger = nova.log.getLogger("nova.api") + logger.audit(_("Starting nova-api node (version %s)") % version) + + nova.flags.FLAGS(sys.argv) + nova.utils.default_flagfile() + + pool = multiprocessing.Pool(2) + pool.map_async(launch, ["ec2", "osapi"]) + pool.close() + + try: + pool.join() + except KeyboardInterrupt: + logger.audit(_("Exiting...")) + pool.terminate() if __name__ == '__main__': diff --git a/nova/__init__.py b/nova/__init__.py index 7b05611b9..886eaee20 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -39,6 +39,7 @@ import log as logging def initialize(): gettext.install("nova", unicode=1) logging.setup() + logging.debug(_("Initialized logging.")) initialize() diff --git a/nova/log.py b/nova/log.py index 3080daee2..f8c0ba68d 100644 --- a/nova/log.py +++ b/nova/log.py @@ -319,7 +319,7 @@ def audit(msg, *args, **kwargs): class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" - def __init__(self, logger, level=logging.DEBUG): + def __init__(self, logger, level=logging.INFO): self.logger = logger self.level = level diff --git a/nova/service.py b/nova/service.py index ec2de9004..4c7d1adc4 100644 --- a/nova/service.py +++ b/nova/service.py @@ -249,7 +249,7 @@ class WSGIService(object): def _find_config(self): """Attempt to find 'paste' configuration file.""" location = wsgi.paste_config_file(self._config_name) - logging.debug(_("Using paste.deploy config at: %s"), location) + logging.info(_("Using paste.deploy config: %s"), location) return location def _load_config(self): diff --git a/nova/wsgi.py b/nova/wsgi.py index 4da8eff82..82ba006c8 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -77,10 +77,7 @@ class Server(object): def wait(self): """Wait until server has been stopped.""" - try: - self._server.wait() - except KeyboardInterrupt: - pass + self._server.wait() class Request(webob.Request): -- cgit From 79402ffbaeae18bb4adaa899743a688ef0bcb24b Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 18:00:38 -0400 Subject: Cleanup of the cleanup. --- bin/nova-api | 6 ++---- nova/tests/integrated/integrated_helpers.py | 3 --- setup.cfg | 3 --- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index ff4aa83d1..1fcda24f4 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,4 +1,5 @@ #!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -18,13 +19,10 @@ """Starter script for Nova API. -Starts both the EC2 and OpenStack APIs in separate processes. Pylint warnings -about re-imports should be ignored. +Starts both the EC2 and OpenStack APIs in separate processes. """ -# pylint: disable=W0404 - import sys import multiprocessing diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py index ba6bef62f..8809bf5f8 100644 --- a/nova/tests/integrated/integrated_helpers.py +++ b/nova/tests/integrated/integrated_helpers.py @@ -171,9 +171,6 @@ class _IntegratedTestBase(test.TestCase): self.api = self.user.openstack_api def _start_api_service(self): - #ec2 = service.WSGIService("ec2") - #ec2.start() - osapi = service.WSGIService("osapi") osapi.start() diff --git a/setup.cfg b/setup.cfg index 7efc46f23..9c0a331e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[develop] -user = True - [build_sphinx] all_files = 1 build-dir = doc/build -- cgit From 927aecb0a3ff1fe561b3c96a4fb9b18c8893c3ae Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 20:18:29 -0400 Subject: Introduced Loader concept, for paste decouple. --- bin/nova-api | 2 + nova/exception.py | 8 +++ nova/service.py | 38 +++++-------- nova/wsgi.py | 161 +++++++++++++++++++++++++++++++----------------------- 4 files changed, 116 insertions(+), 93 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 1fcda24f4..885fb0ba4 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -51,6 +51,8 @@ def main(): nova.flags.FLAGS(sys.argv) nova.utils.default_flagfile() + +# launch("osapi") pool = multiprocessing.Pool(2) pool.map_async(launch, ["ec2", "osapi"]) diff --git a/nova/exception.py b/nova/exception.py index f3a452228..0fcd3de95 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -589,3 +589,11 @@ class MigrationError(NovaException): class MalformedRequestBody(NovaException): message = _("Malformed message body: %(reason)s") + + +class PasteConfigNotFound(NotFound): + message = _("Could not find paste config at %(path)s") + + +class PasteAppNotFound(NotFound): + message = _("Could not load paste app '%(name)s' from %(path)s") diff --git a/nova/service.py b/nova/service.py index 4c7d1adc4..e27d61371 100644 --- a/nova/service.py +++ b/nova/service.py @@ -235,34 +235,24 @@ class Service(object): class WSGIService(object): """Provides ability to launch API from a 'paste' configuration.""" - def __init__(self, name, config_name=None): - """Initialize, but do not start, an API service.""" + def __init__(self, name, loader=None): + """Initialize, but do not start the WSGI service. + + :param name: The name of the WSGI service given to the loader. + :param loader: Loads the WSGI application using the given name. + :returns: None + + """ self.name = name - self._config_name = config_name or FLAGS.api_paste_config - self._config_location = self._find_config() - self._config = self._load_config() - self.application = self._load_application() - host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") - port = getattr(FLAGS, '%s_listen_port' % name, 0) - self.server = wsgi.Server(name, self.application, host, port) - - def _find_config(self): - """Attempt to find 'paste' configuration file.""" - location = wsgi.paste_config_file(self._config_name) - logging.info(_("Using paste.deploy config: %s"), location) - return location - - def _load_config(self): - """Read and return the 'paste' configuration file.""" - return wsgi.load_paste_configuration(self._config_location, self.name) - - def _load_application(self): - """Using the loaded configuration, return the WSGI application.""" - return wsgi.load_paste_app(self._config_location, self.name) + self.loader = loader or wsgi.Loader() + self.application = self.loader.load_app(name) + self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") + self.port = getattr(FLAGS, '%s_listen_port' % name, 0) + self.server = wsgi.Server(name, self.application) def start(self): """Start serving this API using loaded configuration.""" - self.server.start() + self.server.start(self.host, self.port) def stop(self): """Stop serving this API.""" diff --git a/nova/wsgi.py b/nova/wsgi.py index 82ba006c8..328dc083f 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -35,48 +35,78 @@ from paste import deploy from nova import exception from nova import flags -from nova import log as logging +from nova import log from nova import utils FLAGS = flags.FLAGS -LOG = logging.getLogger('nova.wsgi') +LOG = log.getLogger('nova.wsgi') class Server(object): """Server class to manage multiple WSGI sockets and applications.""" default_pool_size = 1000 - logger_name = "eventlet.wsgi.server" - def __init__(self, name, app, host, port, pool_size=None): + def __init__(self, name, app, pool_size=None): + """Initialize, but do not start, a WSGI server. + + :param name: The name to use for logging and other human purposes. + :param app: The WSGI application to serve. + :param pool_size: Maximum number of eventlets to spawn concurrently. + :returns: None + + """ self.name = name self.app = app - self.host = host - self.port = port - self._pool = eventlet.GreenPool(pool_size or self.default_pool_size) - self._log = logging.WritableLogger(logging.getLogger(self.logger_name)) + self.pool_size = pool_size or self.default_pool_size def _start(self, socket): - """Blocking eventlet WSGI server launched from the real 'start'.""" - eventlet.wsgi.server(socket, - self.app, - custom_pool=self._pool, - log=self._log) - - def start(self, backlog=128): - """Serve given WSGI application using the given parameters.""" - socket = eventlet.listen((self.host, self.port), backlog=backlog) + """Run the blocking eventlet WSGI server. + + :param socket: The socket where the WSGI server will serve it's app. + :returns: None + + """ + pool = eventlet.GreenPool(self.pool_size) + logger = log.WritableLogger(log.getLogger("eventlet.wsgi.server")) + eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) + + def start(self, host, port, backlog=128): + """Start serving a WSGI application. + + :param host: IP address to serve the application. + :param port: Port number to server the application. + :param backlog: Maximum number of queued connections. + :returns: None + + """ + socket = eventlet.listen((host, port), backlog=backlog) self._server = eventlet.spawn(self._start, socket) (self.host, self.port) = socket.getsockname() - LOG.info(_('Starting %(name)s on %(host)s:%(port)s') % self.__dict__) + LOG.info(_("Started %(name)s on %(host)s:%(port)s") % self.__dict__) def stop(self): - """Stop this server by killing the greenthread running it.""" + """Stop this server. + + This is not a very nice action, as currently the method by which a + server is stopped is by killing it's eventlet. + + :returns: None + + """ + LOG.debug(_("Stopping WSGI server.")) self._server.kill() def wait(self): - """Wait until server has been stopped.""" + """Block, until the server has stopped. + + Waits on the server's eventlet to finish, then returns. + + :returns: None + + """ + LOG.debug(_("Waiting for WSGI server to stop.")) self._server.wait() @@ -305,57 +335,50 @@ class Router(object): return app -def paste_config_file(basename): - """Find the best location in the system for a paste config file. +class Loader(object): + """Used to load WSGI applications from paste configurations.""" - Search Order - ------------ + def __init__(self, config_path=None): + """Initialize the loader, and attempt to find the config. - The search for a paste config file honors `FLAGS.state_path`, which in a - version checked out from bzr will be the `nova` directory in the top level - of the checkout, and in an installation for a package for your distribution - will likely point to someplace like /etc/nova. + :param config_path: Full or relative path to the paste config. + :returns: None - This method tries to load places likely to be used in development or - experimentation before falling back to the system-wide configuration - in `/etc/nova/`. + """ + config_path = config_path or FLAGS.api_paste_config + self.config_path = self._find_config(config_path) - * Current working directory - * the `etc` directory under state_path, because when working on a checkout - from bzr this will point to the default - * top level of FLAGS.state_path, for distributions - * /etc/nova, which may not be diffrerent from state_path on your distro + def _find_config(self, config_path): + """Find the paste configuration file using the given hint. - """ - configfiles = [basename, - os.path.join(FLAGS.state_path, 'etc', 'nova', basename), - os.path.join(FLAGS.state_path, 'etc', basename), - os.path.join(FLAGS.state_path, basename), - '/etc/nova/%s' % basename] - for configfile in configfiles: - if os.path.exists(configfile): - return configfile - - raise Exception(_("Unable to find paste.deploy config '%s'") % basename) - - -def load_paste_configuration(filename, appname): - """Returns a paste configuration dict, or None.""" - filename = os.path.abspath(filename) - config = None - try: - config = deploy.appconfig('config:%s' % filename, name=appname) - except LookupError: - pass - return config - - -def load_paste_app(filename, appname): - """Builds a wsgi app from a paste config, None if app not configured.""" - filename = os.path.abspath(filename) - app = None - try: - app = deploy.loadapp('config:%s' % filename, name=appname) - except LookupError: - pass - return app + :param config_path: Full or relative path to the paste config. + :returns: Full path of the paste config, if it exists. + :raises: `nova.exception.PasteConfigNotFound` + + """ + possible_locations = [ + config_path, + os.path.join(FLAGS.state_path, "etc", "nova", config_path), + os.path.join(FLAGS.state_path, "etc", config_path), + os.path.join(FLAGS.state_path, config_path), + "/etc/nova/%s" % config_path, + ] + + for path in possible_locations: + if os.path.exists(path): + return os.path.abspath(path) + + raise exception.PasteConfigNotFound(path=os.path.abspath(config_path)) + + def load_app(self, name): + """Return the paste URLMap wrapped WSGI application. + + :param name: Name of the application to load. + :returns: Paste URLMap object wrapping the requested application. + :raises: `nova.exception.PasteAppNotFound` + + """ + app = deploy.loadapp("config:%s" % self.config_path, name=name) + if app is None: + raise exception.PasteAppNotFound(name=name, path=self.config_path) + return app -- cgit From c1b70cc20a17e99fedb0f0a93139424fb89dd9e9 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 20:26:47 -0400 Subject: Cleanup. --- nova/service.py | 22 +++++++++++++++++++--- nova/tests/integrated/integrated_helpers.py | 5 ++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/nova/service.py b/nova/service.py index e27d61371..bb38cddb6 100644 --- a/nova/service.py +++ b/nova/service.py @@ -251,15 +251,31 @@ class WSGIService(object): self.server = wsgi.Server(name, self.application) def start(self): - """Start serving this API using loaded configuration.""" + """Start serving this service using loaded configuration. + + Also, retrieve updated port number in case '0' was passed in, which + indicates a random port should be used. + + :returns: None + + """ self.server.start(self.host, self.port) + self.port = self.server.port def stop(self): - """Stop serving this API.""" + """Stop serving this API. + + :returns: None + + """ self.server.stop() def wait(self): - """Wait for the service to stop serving this API.""" + """Wait for the service to stop serving this API. + + :returns: None + + """ self.server.wait() diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py index 8809bf5f8..26de86e74 100644 --- a/nova/tests/integrated/integrated_helpers.py +++ b/nova/tests/integrated/integrated_helpers.py @@ -173,10 +173,9 @@ class _IntegratedTestBase(test.TestCase): def _start_api_service(self): osapi = service.WSGIService("osapi") osapi.start() + self.auth_url = 'http://%s:%s/v1.1' % (osapi.host, osapi.port) + LOG.warn(self.auth_url) - host = osapi.server.host - port = osapi.server.port - self.auth_url = 'http://%s:%s/v1.1' % (host, port) def tearDown(self): self.context.cleanup() -- cgit From 5a26a6523cfba2fdeaf0abebac8921f2a3322b13 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 21:28:51 -0400 Subject: Added tests for WSGI loader. --- nova/wsgi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 328dc083f..ce0caf65b 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -378,7 +378,8 @@ class Loader(object): :raises: `nova.exception.PasteAppNotFound` """ - app = deploy.loadapp("config:%s" % self.config_path, name=name) - if app is None: + try: + return deploy.loadapp("config:%s" % self.config_path, name=name) + except LookupError as err: + LOG.error(err) raise exception.PasteAppNotFound(name=name, path=self.config_path) - return app -- cgit From 93d6a1c727ffa5ac2972a26fc8a1e38edc84684a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 21:29:43 -0400 Subject: No, really. Added tests for WSGI loader. --- nova/tests/test_wsgi.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 nova/tests/test_wsgi.py diff --git a/nova/tests/test_wsgi.py b/nova/tests/test_wsgi.py new file mode 100644 index 000000000..245b51d81 --- /dev/null +++ b/nova/tests/test_wsgi.py @@ -0,0 +1,79 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Unit tests for `nova.wsgi`.""" + +import os.path +import tempfile + +import unittest2 as unittest + +import nova.exception +import nova.test +import nova.wsgi + + +class TestNothingExists(unittest.TestCase): + """Loader tests where os.path.exists always returns False.""" + + def setUp(self): + self._os_path_exists = os.path.exists + os.path.exists = lambda _: False + + def test_config_not_found(self): + self.assertRaises( + nova.exception.PasteConfigNotFound, + nova.wsgi.Loader, + ) + + def tearDown(self): + os.path.exists = self._os_path_exists + + +class TestNormalFilesystem(unittest.TestCase): + """Loader tests where os.path.exists always returns True.""" + + _paste_config = """ +[app:test_app] +use = egg:Paste#static +document_root = /tmp + """ + + def setUp(self): + self.config = tempfile.NamedTemporaryFile(mode="w+t") + self.config.write(self._paste_config.lstrip()) + self.config.seek(0) + self.config.flush() + self.loader = nova.wsgi.Loader(self.config.name) + + def test_config_found(self): + self.assertEquals(self.config.name, self.loader.config_path) + + def test_app_not_found(self): + self.assertRaises( + nova.exception.PasteAppNotFound, + self.loader.load_app, + "non-existant app", + ) + + def test_app_found(self): + url_parser = self.loader.load_app("test_app") + self.assertEquals("/tmp", url_parser.directory) + + def tearDown(self): + self.config.close() -- cgit From dd870291a32d18d0f62592a73a03b9038ae5c3da Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 10:12:43 -0400 Subject: Cleanup and addition of tests for WSGI server. --- bin/nova-api | 2 +- nova/tests/test_wsgi.py | 21 ++++++++++++++++++--- nova/utils.py | 2 -- nova/wsgi.py | 18 ++++++++++++------ tools/pip-requires | 2 +- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 885fb0ba4..89112a159 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# + # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. diff --git a/nova/tests/test_wsgi.py b/nova/tests/test_wsgi.py index 245b51d81..c19bd3ef6 100644 --- a/nova/tests/test_wsgi.py +++ b/nova/tests/test_wsgi.py @@ -28,7 +28,7 @@ import nova.test import nova.wsgi -class TestNothingExists(unittest.TestCase): +class TestLoaderNothingExists(unittest.TestCase): """Loader tests where os.path.exists always returns False.""" def setUp(self): @@ -45,8 +45,8 @@ class TestNothingExists(unittest.TestCase): os.path.exists = self._os_path_exists -class TestNormalFilesystem(unittest.TestCase): - """Loader tests where os.path.exists always returns True.""" +class TestLoaderNormalFilesystem(unittest.TestCase): + """Loader tests with normal filesystem (unmodified os.path module).""" _paste_config = """ [app:test_app] @@ -77,3 +77,18 @@ document_root = /tmp def tearDown(self): self.config.close() + + +class TestWSGIServer(unittest.TestCase): + """WSGI server tests.""" + + def test_no_app(self): + server = nova.wsgi.Server("test_app", None) + self.assertEquals("test_app", server.name) + + def test_start_random_port(self): + server = nova.wsgi.Server("test_random", None) + server.start("127.0.0.1", 0) + self.assertNotEqual(0, server.port) + server.stop() + server.wait() diff --git a/nova/utils.py b/nova/utils.py index 22d0ce940..691134ada 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -238,8 +238,6 @@ def default_flagfile(filename='nova.conf'): filename = "./nova.conf" if not os.path.exists(filename): filename = '/etc/nova/nova.conf' - if not os.path.exists(filename): - return flagfile = ['--flagfile=%s' % filename] sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] diff --git a/nova/wsgi.py b/nova/wsgi.py index ce0caf65b..8a7d38252 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -21,16 +21,16 @@ import os import sys + from xml.dom import minidom import eventlet import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True, time=True) -import routes +import greenlet import routes.middleware -import webob import webob.dec import webob.exc + from paste import deploy from nova import exception @@ -39,6 +39,9 @@ from nova import log from nova import utils +eventlet.patcher.monkey_patch(all=False, socket=True, time=True) + + FLAGS = flags.FLAGS LOG = log.getLogger('nova.wsgi') @@ -95,7 +98,7 @@ class Server(object): :returns: None """ - LOG.debug(_("Stopping WSGI server.")) + LOG.info(_("Stopping WSGI server.")) self._server.kill() def wait(self): @@ -106,8 +109,11 @@ class Server(object): :returns: None """ - LOG.debug(_("Waiting for WSGI server to stop.")) - self._server.wait() + LOG.info(_("Waiting for WSGI server to stop.")) + try: + self._server.wait() + except greenlet.GreenletExit: + LOG.info(_("WSGI server has stopped.")) class Request(webob.Request): diff --git a/tools/pip-requires b/tools/pip-requires index 7849dbea9..13523d56e 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -8,7 +8,7 @@ amqplib==0.6.1 anyjson==0.2.4 boto==1.9b carrot==0.10.5 -eventlet==0.9.12 +eventlet lockfile==0.8 python-novaclient==2.5.3 python-daemon==1.5.5 -- cgit From 91050cc49e61b46f55722d8fe7e342c2f8ac926b Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 10:39:17 -0400 Subject: Fix objectstore test. --- nova/tests/test_objectstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_objectstore.py b/nova/tests/test_objectstore.py index c35955c9c..be197a219 100644 --- a/nova/tests/test_objectstore.py +++ b/nova/tests/test_objectstore.py @@ -70,8 +70,8 @@ class S3APITestCase(test.TestCase): os.mkdir(FLAGS.buckets_path) router = s3server.S3Application(FLAGS.buckets_path) - self.server = wsgi.Server("s3api", router, FLAGS.s3_host, FLAGS.s3_port) - self.server.start() + self.server = wsgi.Server("s3api", router) + self.server.start(FLAGS.s3_host, FLAGS.s3_port) if not boto.config.has_section('Boto'): boto.config.add_section('Boto') -- cgit From c178b3ce44d89b662c5925b7b65aab9c2540cf37 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 14:54:53 -0400 Subject: pep8 fixes --- bin/nova-api | 2 -- nova/tests/integrated/integrated_helpers.py | 1 - nova/tests/test_wsgi.py | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 89112a159..2345b3f2c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -51,8 +51,6 @@ def main(): nova.flags.FLAGS(sys.argv) nova.utils.default_flagfile() - -# launch("osapi") pool = multiprocessing.Pool(2) pool.map_async(launch, ["ec2", "osapi"]) diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py index 26de86e74..47bd8c1e4 100644 --- a/nova/tests/integrated/integrated_helpers.py +++ b/nova/tests/integrated/integrated_helpers.py @@ -176,7 +176,6 @@ class _IntegratedTestBase(test.TestCase): self.auth_url = 'http://%s:%s/v1.1' % (osapi.host, osapi.port) LOG.warn(self.auth_url) - def tearDown(self): self.context.cleanup() super(_IntegratedTestBase, self).tearDown() diff --git a/nova/tests/test_wsgi.py b/nova/tests/test_wsgi.py index c19bd3ef6..be18baa95 100644 --- a/nova/tests/test_wsgi.py +++ b/nova/tests/test_wsgi.py @@ -62,7 +62,7 @@ document_root = /tmp self.loader = nova.wsgi.Loader(self.config.name) def test_config_found(self): - self.assertEquals(self.config.name, self.loader.config_path) + self.assertEquals(self.config.name, self.loader.config_path) def test_app_not_found(self): self.assertRaises( -- cgit From e849aa7112dcf24357d46f39195cfefce828a91a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 19:32:18 -0400 Subject: Removed logging logic from __init__, added concept of Launcher...no tests for it yet. --- bin/nova-api | 40 +++++++++----------------------- nova/__init__.py | 10 +------- nova/service.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- nova/wsgi.py | 1 - 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 2345b3f2c..563d7c090 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -24,44 +24,26 @@ Starts both the EC2 and OpenStack APIs in separate processes. """ import sys -import multiprocessing -import nova.flags import nova.log import nova.service import nova.version -import nova.utils - - -def launch(service_name): - """Launch WSGI service with name matching 'paste' config file section.""" - service = nova.service.WSGIService(service_name) - service.start() - try: - service.wait() - except KeyboardInterrupt: - service.stop() def main(): - """Begin process of launching both EC2 and OSAPI services.""" - version = nova.version.version_string_with_vcs() - logger = nova.log.getLogger("nova.api") - logger.audit(_("Starting nova-api node (version %s)") % version) - - nova.flags.FLAGS(sys.argv) - nova.utils.default_flagfile() - - pool = multiprocessing.Pool(2) - pool.map_async(launch, ["ec2", "osapi"]) - pool.close() - + """Launch EC2 and OSAPI services.""" + ec2 = nova.service.WSGIService("ec2") + osapi = nova.service.WSGIService("osapi") + + launcher = nova.service.Launcher(sys.argv) + launcher.launch_service(ec2) + launcher.launch_service(osapi) + try: - pool.join() + launcher.wait() except KeyboardInterrupt: - logger.audit(_("Exiting...")) - pool.terminate() - + launcher.stop() + if __name__ == '__main__': sys.exit(main()) diff --git a/nova/__init__.py b/nova/__init__.py index 886eaee20..884c4a713 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -33,13 +33,5 @@ import gettext -import log as logging - -def initialize(): - gettext.install("nova", unicode=1) - logging.setup() - logging.debug(_("Initialized logging.")) - - -initialize() +gettext.install("nova", unicode=1) diff --git a/nova/service.py b/nova/service.py index bb38cddb6..d896de1fe 100644 --- a/nova/service.py +++ b/nova/service.py @@ -19,10 +19,12 @@ """Generic Node baseclass for all workers that run on hosts.""" -import greenlet import inspect +import multiprocessing import os +import greenlet + from eventlet import greenthread from nova import context @@ -53,6 +55,72 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini", 'File name for the paste.deploy config for nova-api') +class Launcher(object): + """Launch one or more services and wait for them to complete.""" + + def __init__(self, _flags=None): + """Initialize the service launcher. + + :param _flags: Flags to use for the services we're going to load. + :returns: None + + """ + self._services = [] + self._version = version.version_string_with_vcs() + logging.setup() + logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) + FLAGS(_flags) + utils.default_flagfile() + + + @staticmethod + def run_service(service): + """Start and wait for a service to finish. + + :param service: Service to run and wait for. + :returns: None + + """ + service.start() + try: + service.wait() + except KeyboardInterrupt: + service.stop() + + def launch_service(self, service): + """Load and start the given service. + + :param service: The service you would like to start. + :returns: None + + """ + process = multiprocessing.Process(target=self.run_service, + args=(service,)) + process.start() + self._services.append(process) + + def stop(self): + """Stop all services which are currently running. + + :returns: None + + """ + for service in self._services: + if service.is_alive(): + service.terminate() + + def wait(self): + """Waits until all services have been stopped, and then returns. + + :returns: None + + """ + for service in self._services: + service.join() + logging.info("Process exited with %d" % service.exitcode) + + + class Service(object): """Base class for workers that run on hosts.""" diff --git a/nova/wsgi.py b/nova/wsgi.py index 8a7d38252..097fa4734 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -109,7 +109,6 @@ class Server(object): :returns: None """ - LOG.info(_("Waiting for WSGI server to stop.")) try: self._server.wait() except greenlet.GreenletExit: -- cgit From c17c73b3d0f07046c677711853e1b93768526e47 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:21:33 -0400 Subject: Tests for WSGI/Launcher --- bin/nova-api | 4 ++-- nova/service.py | 4 +--- nova/tests/test_service.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 563d7c090..20ad4bfa5 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -38,12 +38,12 @@ def main(): launcher = nova.service.Launcher(sys.argv) launcher.launch_service(ec2) launcher.launch_service(osapi) - + try: launcher.wait() except KeyboardInterrupt: launcher.stop() - + if __name__ == '__main__': sys.exit(main()) diff --git a/nova/service.py b/nova/service.py index d896de1fe..08071e80c 100644 --- a/nova/service.py +++ b/nova/service.py @@ -69,9 +69,8 @@ class Launcher(object): self._version = version.version_string_with_vcs() logging.setup() logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) - FLAGS(_flags) utils.default_flagfile() - + FLAGS(_flags or []) @staticmethod def run_service(service): @@ -120,7 +119,6 @@ class Launcher(object): logging.info("Process exited with %d" % service.exitcode) - class Service(object): """Base class for workers that run on hosts.""" diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index d1cc8bd61..70ef80147 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -21,6 +21,7 @@ Unit Tests for remote procedure calls using queue """ import mox +import unittest2 as unittest from nova import context from nova import db @@ -30,6 +31,7 @@ from nova import rpc from nova import test from nova import service from nova import manager +from nova import wsgi from nova.compute import manager as compute_manager FLAGS = flags.FLAGS @@ -349,3 +351,31 @@ class ServiceTestCase(test.TestCase): serv.stop() db.service_destroy(ctxt, service_ref['id']) + + +class TestWSGIService(test.TestCase): + + def setUp(self): + super(TestWSGIService, self).setUp() + self.stubs.Set(wsgi.Loader, "load_app", mox.MockAnything()) + + def test_service_random_port(self): + test_service = service.WSGIService("test_service") + self.assertEquals(0, test_service.port) + test_service.start() + self.assertNotEqual(0, test_service.port) + test_service.stop() + +class TestLauncher(test.TestCase): + + def setUp(self): + super(TestLauncher, self).setUp() + self.stubs.Set(wsgi.Loader, "load_app", mox.MockAnything()) + self.service = service.WSGIService("test_service") + + def test_launch_app(self): + self.assertEquals(0, self.service.port) + launcher = service.Launcher() + launcher.launch_service(self.service) + self.assertEquals(0, self.service.port) + launcher.stop() -- cgit From e821b96feb49492c7b20afaa7ae0be5143dd4879 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:32:31 -0400 Subject: Removed unneeded import. --- bin/nova-api | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 20ad4bfa5..b94928c7b 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -27,7 +27,6 @@ import sys import nova.log import nova.service -import nova.version def main(): -- cgit From 679b00759ab2f183c3372465baa7daab1abeb25e Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:35:45 -0400 Subject: Removed debugging and switched eventlet to monkey patch everything. --- nova/service.py | 1 - nova/wsgi.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/service.py b/nova/service.py index 08071e80c..4cf372377 100644 --- a/nova/service.py +++ b/nova/service.py @@ -116,7 +116,6 @@ class Launcher(object): """ for service in self._services: service.join() - logging.info("Process exited with %d" % service.exitcode) class Service(object): diff --git a/nova/wsgi.py b/nova/wsgi.py index 097fa4734..3c4f8a5da 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -39,7 +39,7 @@ from nova import log from nova import utils -eventlet.patcher.monkey_patch(all=False, socket=True, time=True) +eventlet.patcher.monkey_patch() FLAGS = flags.FLAGS -- cgit From c80dbac0b9563fb7afdc1a9ec3f0a851e2673236 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:54:14 -0400 Subject: log -> logging to keep with convention --- nova/wsgi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 3c4f8a5da..2991d1529 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -35,7 +35,7 @@ from paste import deploy from nova import exception from nova import flags -from nova import log +from nova import log as logging from nova import utils @@ -43,7 +43,7 @@ eventlet.patcher.monkey_patch() FLAGS = flags.FLAGS -LOG = log.getLogger('nova.wsgi') +LOG = logging.getLogger('nova.wsgi') class Server(object): @@ -72,7 +72,7 @@ class Server(object): """ pool = eventlet.GreenPool(self.pool_size) - logger = log.WritableLogger(log.getLogger("eventlet.wsgi.server")) + logger = logging.WritableLogger(logging.getLogger("eventlet.wsgi")) eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) def start(self, host, port, backlog=128): -- cgit From 821f597228ed206564931b6693b134d04ef29e42 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 09:46:00 -0400 Subject: Monkey patching 'os' kills multiprocessing's .join() functionality. Also, messed up the name of the eventlet WSGI logger. --- nova/wsgi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 2991d1529..0c869cfb8 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -39,7 +39,7 @@ from nova import log as logging from nova import utils -eventlet.patcher.monkey_patch() +eventlet.patcher.monkey_patch(socket=True, time=True) FLAGS = flags.FLAGS @@ -72,7 +72,8 @@ class Server(object): """ pool = eventlet.GreenPool(self.pool_size) - logger = logging.WritableLogger(logging.getLogger("eventlet.wsgi")) + log_name = "eventlet.wsgi.server" + logger = logging.WritableLogger(logging.getLogger(log_name)) eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) def start(self, host, port, backlog=128): -- cgit From afff25800521e7085ddff7e910195ef5a1f98732 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 09:57:40 -0400 Subject: pep8 fix --- nova/tests/test_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 70ef80147..350dc62ee 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -366,6 +366,7 @@ class TestWSGIService(test.TestCase): self.assertNotEqual(0, test_service.port) test_service.stop() + class TestLauncher(test.TestCase): def setUp(self): -- cgit From 8a0d287631b5773a9868ae0e3cce6e2aef1ea501 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 10:35:50 -0400 Subject: Oops, I broke --help on nova-api, fixed now. --- nova/service.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nova/service.py b/nova/service.py index 4cf372377..f82c64d9c 100644 --- a/nova/service.py +++ b/nova/service.py @@ -67,10 +67,31 @@ class Launcher(object): """ self._services = [] self._version = version.version_string_with_vcs() + self._flags = _flags + self._setup_logging() + self._setup_flags() + + def _setup_logging(self): + """Logic to ensure logging is going to work correctly for services. + + :returns: None + + """ logging.setup() logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) + + def _setup_flags(self): + """Logic to ensure flags/configuration are correctly set. + + :returns: None + + """ utils.default_flagfile() - FLAGS(_flags or []) + FLAGS(self._flags or []) + flags.DEFINE_flag(flags.HelpFlag()) + flags.DEFINE_flag(flags.HelpshortFlag()) + flags.DEFINE_flag(flags.HelpXMLFlag()) + FLAGS.ParseNewFlags() @staticmethod def run_service(service): -- cgit From 742c21e4e79ce5a26975b31486ded3956a846c55 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 11:25:44 -0400 Subject: Very small alterations, switched from using start() to pass host/port, to just defining them up front in init. Doesn't make sense to set them in start because we can't start more than once any way. Also, unbroke binaries. --- bin/nova-ajax-console-proxy | 5 +++-- bin/nova-direct-api | 7 +++++-- bin/nova-objectstore | 7 +++++-- bin/nova-vncproxy | 7 +++++-- nova/service.py | 13 ++++++++----- nova/tests/test_objectstore.py | 4 ++-- nova/tests/test_wsgi.py | 5 +++-- nova/wsgi.py | 39 ++++++++++++++++++++++----------------- 8 files changed, 53 insertions(+), 34 deletions(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index d88f59e40..21cf68007 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -137,8 +137,9 @@ if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) logging.setup() - server = wsgi.Server() + acp_port = FLAGS.ajax_console_proxy_port acp = AjaxConsoleProxy() acp.register_listeners() - server.start(acp, FLAGS.ajax_console_proxy_port, host='0.0.0.0') + server = wsgi.Server("AJAX Console Proxy", acp, port=acp_port) + server.start() server.wait() diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 83ec72722..5d63eb87f 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -93,6 +93,9 @@ if __name__ == '__main__': with_req = direct.PostParamsMiddleware(with_json) with_auth = direct.DelegatedAuthMiddleware(with_req) - server = wsgi.Server() - server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host) + server = wsgi.Server("Direct API", + with_auth, + host=FLAGS.direct_host, + port=FLAGS.direct_port) + server.start() server.wait() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 6ef841b85..aa0dc063f 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -50,6 +50,9 @@ if __name__ == '__main__': FLAGS(sys.argv) logging.setup() router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server() - server.start(router, FLAGS.s3_port, host=FLAGS.s3_host) + server = wsgi.Server("S3 Objectstore", + router, + port=FLAGS.s3_port, + host=FLAGS.s3_host) + server.start() server.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index ccb97e3a3..62d3b948c 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -96,6 +96,9 @@ if __name__ == "__main__": service.serve() - server = wsgi.Server() - server.start(with_auth, FLAGS.vncproxy_port, host=FLAGS.vncproxy_host) + server = wsgi.Server("VNC Proxy", + with_auth, + host=FLAGS.vncproxy_host, + port=FLAGS.vncproxy_port) + server.start() server.wait() diff --git a/nova/service.py b/nova/service.py index f82c64d9c..854b4572d 100644 --- a/nova/service.py +++ b/nova/service.py @@ -331,10 +331,13 @@ class WSGIService(object): """ self.name = name self.loader = loader or wsgi.Loader() - self.application = self.loader.load_app(name) - self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") - self.port = getattr(FLAGS, '%s_listen_port' % name, 0) - self.server = wsgi.Server(name, self.application) + self.app = self.loader.load_app(name) + self.host = getattr(FLAGS, '%s_listen' % name, None) + self.port = getattr(FLAGS, '%s_listen_port' % name, None) + self.server = wsgi.Server(name, + self.app, + host=self.host, + port=self.port) def start(self): """Start serving this service using loaded configuration. @@ -345,7 +348,7 @@ class WSGIService(object): :returns: None """ - self.server.start(self.host, self.port) + self.server.start() self.port = self.server.port def stop(self): diff --git a/nova/tests/test_objectstore.py b/nova/tests/test_objectstore.py index be197a219..7e69565f3 100644 --- a/nova/tests/test_objectstore.py +++ b/nova/tests/test_objectstore.py @@ -70,8 +70,8 @@ class S3APITestCase(test.TestCase): os.mkdir(FLAGS.buckets_path) router = s3server.S3Application(FLAGS.buckets_path) - self.server = wsgi.Server("s3api", router) - self.server.start(FLAGS.s3_host, FLAGS.s3_port) + self.server = wsgi.Server() + self.server.start(router, host=FLAGS.s3_host, port=FLAGS.s3_port) if not boto.config.has_section('Boto'): boto.config.add_section('Boto') diff --git a/nova/tests/test_wsgi.py b/nova/tests/test_wsgi.py index be18baa95..010fb819e 100644 --- a/nova/tests/test_wsgi.py +++ b/nova/tests/test_wsgi.py @@ -87,8 +87,9 @@ class TestWSGIServer(unittest.TestCase): self.assertEquals("test_app", server.name) def test_start_random_port(self): - server = nova.wsgi.Server("test_random", None) - server.start("127.0.0.1", 0) + server = nova.wsgi.Server("test_random_port", None, host="127.0.0.1") + self.assertEqual(0, server.port) + server.start() self.assertNotEqual(0, server.port) server.stop() server.wait() diff --git a/nova/wsgi.py b/nova/wsgi.py index 0c869cfb8..23d29079f 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -47,47 +47,52 @@ LOG = logging.getLogger('nova.wsgi') class Server(object): - """Server class to manage multiple WSGI sockets and applications.""" + """Server class to manage a WSGI server, serving a WSGI application.""" default_pool_size = 1000 - def __init__(self, name, app, pool_size=None): + def __init__(self, name, app, host=None, port=None, pool_size=None): """Initialize, but do not start, a WSGI server. - :param name: The name to use for logging and other human purposes. + :param name: Pretty name for logging. :param app: The WSGI application to serve. + :param host: IP address to serve the application. + :param port: Port number to server the application. :param pool_size: Maximum number of eventlets to spawn concurrently. :returns: None """ self.name = name self.app = app - self.pool_size = pool_size or self.default_pool_size - - def _start(self, socket): + self.host = host or "0.0.0.0" + self.port = port or 0 + self._server = None + self._socket = None + self._pool = eventlet.GreenPool(pool_size or self.default_pool_size) + self._logger = logging.getLogger("eventlet.wsgi.server") + self._wsgi_logger = logging.WritableLogger(self._logger) + + def _start(self): """Run the blocking eventlet WSGI server. - :param socket: The socket where the WSGI server will serve it's app. :returns: None """ - pool = eventlet.GreenPool(self.pool_size) - log_name = "eventlet.wsgi.server" - logger = logging.WritableLogger(logging.getLogger(log_name)) - eventlet.wsgi.server(socket, self.app, custom_pool=pool, log=logger) + eventlet.wsgi.server(self._socket, + self.app, + custom_pool=self._pool, + log=self._wsgi_logger) - def start(self, host, port, backlog=128): + def start(self, backlog=128): """Start serving a WSGI application. - :param host: IP address to serve the application. - :param port: Port number to server the application. :param backlog: Maximum number of queued connections. :returns: None """ - socket = eventlet.listen((host, port), backlog=backlog) - self._server = eventlet.spawn(self._start, socket) - (self.host, self.port) = socket.getsockname() + self._socket = eventlet.listen((self.host, self.port), backlog=backlog) + self._server = eventlet.spawn(self._start) + (self.host, self.port) = self._socket.getsockname() LOG.info(_("Started %(name)s on %(host)s:%(port)s") % self.__dict__) def stop(self): -- cgit From 7c846ea890f3c7143fd5e158931fc415e53a9bf0 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 11:50:28 -0400 Subject: Fixed objectstore test. --- nova/service.py | 4 ++-- nova/tests/test_objectstore.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nova/service.py b/nova/service.py index 854b4572d..ca247c0f9 100644 --- a/nova/service.py +++ b/nova/service.py @@ -332,8 +332,8 @@ class WSGIService(object): self.name = name self.loader = loader or wsgi.Loader() self.app = self.loader.load_app(name) - self.host = getattr(FLAGS, '%s_listen' % name, None) - self.port = getattr(FLAGS, '%s_listen_port' % name, None) + self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0") + self.port = getattr(FLAGS, '%s_listen_port' % name, 0) self.server = wsgi.Server(name, self.app, host=self.host, diff --git a/nova/tests/test_objectstore.py b/nova/tests/test_objectstore.py index 7e69565f3..39b4e18d7 100644 --- a/nova/tests/test_objectstore.py +++ b/nova/tests/test_objectstore.py @@ -70,8 +70,11 @@ class S3APITestCase(test.TestCase): os.mkdir(FLAGS.buckets_path) router = s3server.S3Application(FLAGS.buckets_path) - self.server = wsgi.Server() - self.server.start(router, host=FLAGS.s3_host, port=FLAGS.s3_port) + self.server = wsgi.Server("S3 Objectstore", + router, + host=FLAGS.s3_host, + port=FLAGS.s3_port) + self.server.start() if not boto.config.has_section('Boto'): boto.config.add_section('Boto') -- cgit From 186598a819c4e9c4b1b76aad61e7df56cdddd5be Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 12:03:27 -0400 Subject: Removed whitespace. --- bin/nova-direct-api | 4 ++-- bin/nova-objectstore | 6 +++--- bin/nova-vncproxy | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 5d63eb87f..c6cf9b2ff 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -93,8 +93,8 @@ if __name__ == '__main__': with_req = direct.PostParamsMiddleware(with_json) with_auth = direct.DelegatedAuthMiddleware(with_req) - server = wsgi.Server("Direct API", - with_auth, + server = wsgi.Server("Direct API", + with_auth, host=FLAGS.direct_host, port=FLAGS.direct_port) server.start() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index aa0dc063f..1aef3a255 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -50,9 +50,9 @@ if __name__ == '__main__': FLAGS(sys.argv) logging.setup() router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server("S3 Objectstore", - router, - port=FLAGS.s3_port, + server = wsgi.Server("S3 Objectstore", + router, + port=FLAGS.s3_port, host=FLAGS.s3_host) server.start() server.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index 62d3b948c..72271df3a 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -96,7 +96,7 @@ if __name__ == "__main__": service.serve() - server = wsgi.Server("VNC Proxy", + server = wsgi.Server("VNC Proxy", with_auth, host=FLAGS.vncproxy_host, port=FLAGS.vncproxy_port) -- cgit From 13a51049ce5e76fd679b3dee978edce58db21d09 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 22 Jun 2011 09:33:46 -0400 Subject: fix some issues with flags and logging --- nova/service.py | 19 ++++++++++++++----- nova/utils.py | 10 ++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/nova/service.py b/nova/service.py index ca247c0f9..41c6551e0 100644 --- a/nova/service.py +++ b/nova/service.py @@ -38,6 +38,8 @@ from nova import version from nova import wsgi +LOG = logging.getLogger('nova.service') + FLAGS = flags.FLAGS flags.DEFINE_integer('report_interval', 10, 'seconds between nodes reporting state to datastore', @@ -58,18 +60,19 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini", class Launcher(object): """Launch one or more services and wait for them to complete.""" - def __init__(self, _flags=None): + def __init__(self, flags=None): """Initialize the service launcher. - :param _flags: Flags to use for the services we're going to load. + :param flags: Flags to use for the services we're going to load. :returns: None """ self._services = [] self._version = version.version_string_with_vcs() - self._flags = _flags - self._setup_logging() + self._flags = flags self._setup_flags() + self._setup_logging() + self._log_flags() def _setup_logging(self): """Logic to ensure logging is going to work correctly for services. @@ -86,13 +89,19 @@ class Launcher(object): :returns: None """ - utils.default_flagfile() + utils.default_flagfile(args=self._flags) FLAGS(self._flags or []) flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) FLAGS.ParseNewFlags() + def _log_flags(self): + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) + @staticmethod def run_service(service): """Start and wait for a service to finish. diff --git a/nova/utils.py b/nova/utils.py index e2ac16f31..a9b0f3128 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -226,8 +226,10 @@ def novadir(): return os.path.abspath(nova.__file__).split('nova/__init__.pyc')[0] -def default_flagfile(filename='nova.conf'): - for arg in sys.argv: +def default_flagfile(filename='nova.conf', args=None): + if args is None: + args = sys.argv + for arg in args: if arg.find('flagfile') != -1: break else: @@ -239,8 +241,8 @@ def default_flagfile(filename='nova.conf'): filename = "./nova.conf" if not os.path.exists(filename): filename = '/etc/nova/nova.conf' - flagfile = ['--flagfile=%s' % filename] - sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] + flagfile = '--flagfile=%s' % filename + args.insert(1, flagfile) def debug(arg): -- cgit From 2059a683e11169a35b35819575926fc6cbc1a3f1 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 22 Jun 2011 23:27:49 -0400 Subject: run launcher first since it initializes global flags and logging --- bin/nova-api | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index b94928c7b..121f6f9a0 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -31,10 +31,11 @@ import nova.service def main(): """Launch EC2 and OSAPI services.""" + launcher = nova.service.Launcher(sys.argv) + ec2 = nova.service.WSGIService("ec2") osapi = nova.service.WSGIService("osapi") - launcher = nova.service.Launcher(sys.argv) launcher.launch_service(ec2) launcher.launch_service(osapi) -- cgit From 655a783d5a0ef2ddadcf119793cd34513a45fe27 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Thu, 23 Jun 2011 21:31:00 -0400 Subject: Created Bootstrapper to handle Nova bootstrapping logic. --- bin/nova-api | 5 +++-- nova/service.py | 36 +----------------------------------- nova/utils.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 37 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 121f6f9a0..ea99a1b48 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -25,17 +25,18 @@ Starts both the EC2 and OpenStack APIs in separate processes. import sys -import nova.log import nova.service +import nova.utils def main(): """Launch EC2 and OSAPI services.""" - launcher = nova.service.Launcher(sys.argv) + nova.utils.Bootstrapper.bootstrap_binary(sys.argv) ec2 = nova.service.WSGIService("ec2") osapi = nova.service.WSGIService("osapi") + launcher = nova.service.Launcher() launcher.launch_service(ec2) launcher.launch_service(osapi) diff --git a/nova/service.py b/nova/service.py index 41c6551e0..00e4f61e5 100644 --- a/nova/service.py +++ b/nova/service.py @@ -60,47 +60,13 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini", class Launcher(object): """Launch one or more services and wait for them to complete.""" - def __init__(self, flags=None): + def __init__(self): """Initialize the service launcher. - :param flags: Flags to use for the services we're going to load. :returns: None """ self._services = [] - self._version = version.version_string_with_vcs() - self._flags = flags - self._setup_flags() - self._setup_logging() - self._log_flags() - - def _setup_logging(self): - """Logic to ensure logging is going to work correctly for services. - - :returns: None - - """ - logging.setup() - logging.audit(_("Nova Version (%(_version)s)") % self.__dict__) - - def _setup_flags(self): - """Logic to ensure flags/configuration are correctly set. - - :returns: None - - """ - utils.default_flagfile(args=self._flags) - FLAGS(self._flags or []) - flags.DEFINE_flag(flags.HelpFlag()) - flags.DEFINE_flag(flags.HelpshortFlag()) - flags.DEFINE_flag(flags.HelpXMLFlag()) - FLAGS.ParseNewFlags() - - def _log_flags(self): - LOG.debug(_("Full set of FLAGS:")) - for flag in FLAGS: - flag_get = FLAGS.get(flag, None) - LOG.debug("%(flag)s : %(flag_get)s" % locals()) @staticmethod def run_service(service): diff --git a/nova/utils.py b/nova/utils.py index a9b0f3128..a6b8d4cbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -743,3 +743,40 @@ def is_uuid_like(val): if not isinstance(val, basestring): return False return (len(val) == 36) and (val.count('-') == 4) + + +class Bootstrapper(object): + """Provides environment bootstrapping capabilities for entry points.""" + + @staticmethod + def bootstrap_binary(argv): + """Initialize the Nova environment using command line arguments.""" + Bootstrapper.setup_flags(argv) + Bootstrapper.setup_logging() + Bootstrapper.log_flags() + + @staticmethod + def setup_logging(): + """Initialize logging and log a message indicating the Nova version.""" + logging.setup() + logging.audit(_("Nova Version (%s)") % + version.version_string_with_vcs()) + + @staticmethod + def setup_flags(input_flags): + """Initialize flags, load flag file, and print help if needed.""" + default_flagfile(args=input_flags) + FLAGS(input_flags or []) + flags.DEFINE_flag(flags.HelpFlag()) + flags.DEFINE_flag(flags.HelpshortFlag()) + flags.DEFINE_flag(flags.HelpXMLFlag()) + FLAGS.ParseNewFlags() + + @staticmethod + def log_flags(): + """Log the list of all active flags being used.""" + logging.audit(_("Currently active flags:")) + for key in FLAGS: + value = FLAGS.get(key, None) + logging.audit(_("%(key)s : %(value)s" % locals())) + -- cgit From 9855acc214a02d5181a0d8a735e57a89146127db Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 27 Jun 2011 15:49:24 -0400 Subject: Added nova.version to utils.py --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/utils.py b/nova/utils.py index a6b8d4cbe..badd67599 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -46,6 +46,7 @@ from eventlet.green import subprocess from nova import exception from nova import flags from nova import log as logging +from nova import version LOG = logging.getLogger("nova.utils") -- cgit From f6390090ff48258078113b2e6d9dd5fbf49bea3a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 28 Jun 2011 10:39:04 -0400 Subject: I accidently the whole unittest2 --- nova/tests/test_service.py | 1 - nova/tests/test_wsgi.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 350dc62ee..f45f76b73 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -21,7 +21,6 @@ Unit Tests for remote procedure calls using queue """ import mox -import unittest2 as unittest from nova import context from nova import db diff --git a/nova/tests/test_wsgi.py b/nova/tests/test_wsgi.py index 010fb819e..b71e8d418 100644 --- a/nova/tests/test_wsgi.py +++ b/nova/tests/test_wsgi.py @@ -21,7 +21,7 @@ import os.path import tempfile -import unittest2 as unittest +import unittest import nova.exception import nova.test -- cgit From 4ec4ec7e4008adabf051651e3c55137c9954139f Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 28 Jun 2011 10:43:25 -0400 Subject: pep8 fix --- nova/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index caa1df22a..918541224 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -790,4 +790,3 @@ class Bootstrapper(object): for key in FLAGS: value = FLAGS.get(key, None) logging.audit(_("%(key)s : %(value)s" % locals())) - -- cgit