diff options
41 files changed, 409 insertions, 479 deletions
@@ -41,7 +41,8 @@ <matt.dietz@rackspace.com> <mdietz@openstack> <mordred@inaugust.com> <mordred@hudson> <naveedm9@gmail.com> <naveed.massjouni@rackspace.com> -<nirmal.ranganathan@rackspace.com> <nirmal.ranganathan@rackspace.coom> +<rnirmal@gmail.com> <nirmal.ranganathan@rackspace.com> +<rnirmal@gmail.com> <nirmal.ranganathan@rackspace.coom> <paul@openstack.org> <paul.voccio@rackspace.com> <paul@openstack.org> <pvoccio@castor.local> <paul@openstack.org> <paul@substation9.com> @@ -123,7 +123,7 @@ Muneyuki Noguchi <noguchimn@nttdata.co.jp> Nachi Ueno <ueno.nachi@lab.ntt.co.jp> Naveed Massjouni <naveedm9@gmail.com> Nikolay Sokolov <nsokolov@griddynamics.com> -Nirmal Ranganathan <nirmal.ranganathan@rackspace.com> +Nirmal Ranganathan <rnirmal@gmail.com> Ollie Leahy <oliver.leahy@hp.com> Pádraig Brady <pbrady@redhat.com> Paul Voccio <paul@openstack.org> diff --git a/doc/source/devref/development.environment.rst b/doc/source/devref/development.environment.rst index ce5b721b7..2b2bacb5b 100644 --- a/doc/source/devref/development.environment.rst +++ b/doc/source/devref/development.environment.rst @@ -54,22 +54,16 @@ Install the prerequisite packages. On Ubuntu:: - sudo apt-get install python-dev swig libssl-dev python-pip git-core + sudo apt-get install python-dev libssl-dev python-pip git-core On Fedora-based distributions (e.g., Fedora/RHEL/CentOS/Scientific Linux):: - sudo yum install python-devel swig openssl-devel python-pip git + sudo yum install python-devel openssl-devel python-pip git Mac OS X Systems ---------------- -Install swig, which is needed to build the M2Crypto Python package. If you are -using the `homebrew <http://mxcl.github.com/homebrew/>`_, package manager, -install swig by doing:: - - brew install swig - Install virtualenv:: sudo easy_install virtualenv @@ -120,7 +114,7 @@ You can manually install the virtual environment instead of having This will install all of the Python packages listed in the ``tools/pip-requires`` file into your virtualenv. There will also be some -additional packages (pip, distribute, greenlet, M2Crypto) that are installed +additional packages (pip, distribute, greenlet) that are installed by the ``tools/install_venv.py`` file into the virutalenv. If all goes well, you should get a message something like this:: diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index e48f5c686..135a8c75b 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -70,3 +70,72 @@ class ProjectMapper(APIMapper): routes.Mapper.resource(self, member_name, collection_name, **kwargs) + + +class APIRouter(base_wsgi.Router): + """ + Routes requests on the OpenStack API to the appropriate controller + and method. + """ + ExtensionManager = None # override in subclasses + + @classmethod + def factory(cls, global_config, **local_config): + """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" + return cls() + + def __init__(self, ext_mgr=None): + if ext_mgr is None: + if self.ExtensionManager: + ext_mgr = self.ExtensionManager() + else: + raise Exception(_("Must specify an ExtensionManager class")) + + mapper = ProjectMapper() + self.resources = {} + self._setup_routes(mapper) + self._setup_ext_routes(mapper, ext_mgr) + self._setup_extensions(ext_mgr) + super(APIRouter, self).__init__(mapper) + + def _setup_ext_routes(self, mapper, ext_mgr): + for resource in ext_mgr.get_resources(): + LOG.debug(_('Extended resource: %s'), + resource.collection) + + wsgi_resource = wsgi.Resource(resource.controller) + self.resources[resource.collection] = wsgi_resource + kargs = dict( + controller=wsgi_resource, + collection=resource.collection_actions, + member=resource.member_actions) + + if resource.parent: + kargs['parent_resource'] = resource.parent + + mapper.resource(resource.collection, resource.collection, **kargs) + + if resource.custom_routes_fn: + resource.custom_routes_fn(mapper, wsgi_resource) + + def _setup_extensions(self, ext_mgr): + for extension in ext_mgr.get_controller_extensions(): + ext_name = extension.extension.name + collection = extension.collection + controller = extension.controller + + if collection not in self.resources: + LOG.warning(_('Extension %(ext_name)s: Cannot extend ' + 'resource %(collection)s: No such resource') % + locals()) + continue + + LOG.debug(_('Extension %(ext_name)s extending resource: ' + '%(collection)s') % locals()) + + resource = self.resources[collection] + resource.register_actions(controller) + resource.register_extensions(controller) + + def _setup_routes(self, mapper): + raise NotImplementedError diff --git a/nova/api/openstack/compute/__init__.py b/nova/api/openstack/compute/__init__.py index 0dd3484c2..d035173ed 100644 --- a/nova/api/openstack/compute/__init__.py +++ b/nova/api/openstack/compute/__init__.py @@ -17,12 +17,9 @@ # under the License. """ -WSGI middleware for OpenStack API controllers. +WSGI middleware for OpenStack Compute API. """ -import webob.dec -import webob.exc - import nova.api.openstack from nova.api.openstack.compute import consoles from nova.api.openstack.compute import extensions @@ -34,11 +31,9 @@ from nova.api.openstack.compute import limits from nova.api.openstack.compute import servers from nova.api.openstack.compute import server_metadata from nova.api.openstack.compute import versions -from nova.api.openstack import wsgi from nova.common import cfg from nova import flags from nova import log as logging -from nova import wsgi as base_wsgi LOG = logging.getLogger('nova.api.openstack.compute') @@ -52,66 +47,12 @@ FLAGS = flags.FLAGS FLAGS.add_option(allow_instance_snapshots_opt) -class APIRouter(base_wsgi.Router): +class APIRouter(nova.api.openstack.APIRouter): """ Routes requests on the OpenStack API to the appropriate controller and method. """ - - @classmethod - def factory(cls, global_config, **local_config): - """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" - return cls() - - def __init__(self, ext_mgr=None): - if ext_mgr is None: - ext_mgr = extensions.ExtensionManager() - - mapper = nova.api.openstack.ProjectMapper() - self.resources = {} - self._setup_routes(mapper) - self._setup_ext_routes(mapper, ext_mgr) - self._setup_extensions(ext_mgr) - super(APIRouter, self).__init__(mapper) - - def _setup_ext_routes(self, mapper, ext_mgr): - for resource in ext_mgr.get_resources(): - LOG.debug(_('Extended resource: %s'), - resource.collection) - - wsgi_resource = wsgi.Resource(resource.controller) - self.resources[resource.collection] = wsgi_resource - kargs = dict( - controller=wsgi_resource, - collection=resource.collection_actions, - member=resource.member_actions) - - if resource.parent: - kargs['parent_resource'] = resource.parent - - mapper.resource(resource.collection, resource.collection, **kargs) - - if resource.custom_routes_fn: - resource.custom_routes_fn(mapper, wsgi_resource) - - def _setup_extensions(self, ext_mgr): - for extension in ext_mgr.get_controller_extensions(): - ext_name = extension.extension.name - collection = extension.collection - controller = extension.controller - - if collection not in self.resources: - LOG.warning(_('Extension %(ext_name)s: Cannot extend ' - 'resource %(collection)s: No such resource') % - locals()) - continue - - LOG.debug(_('Extension %(ext_name)s extending resource: ' - '%(collection)s') % locals()) - - resource = self.resources[collection] - resource.register_actions(controller) - resource.register_extensions(controller) + ExtensionManager = extensions.ExtensionManager def _setup_routes(self, mapper): self.resources['versions'] = versions.create_resource() diff --git a/nova/api/openstack/compute/contrib/keypairs.py b/nova/api/openstack/compute/contrib/keypairs.py index 57ac48dbc..0e8a4bb06 100644 --- a/nova/api/openstack/compute/contrib/keypairs.py +++ b/nova/api/openstack/compute/contrib/keypairs.py @@ -87,7 +87,8 @@ class KeypairController(object): # NOTE(ja): generation is slow, so shortcut invalid name exception try: db.key_pair_get(context, context.user_id, name) - raise exception.KeyPairExists(key_name=name) + msg = _("Key pair '%s' already exists.") % name + raise webob.exc.HTTPConflict(explanation=msg) except exception.NotFound: pass diff --git a/nova/api/openstack/compute/contrib/virtual_interfaces.py b/nova/api/openstack/compute/contrib/virtual_interfaces.py index e8da23c00..9cf31e669 100644 --- a/nova/api/openstack/compute/contrib/virtual_interfaces.py +++ b/nova/api/openstack/compute/contrib/virtual_interfaces.py @@ -19,6 +19,7 @@ from nova.api.openstack import common from nova.api.openstack import extensions from nova.api.openstack import wsgi from nova.api.openstack import xmlutil +from nova import compute from nova import log as logging from nova import network @@ -54,6 +55,7 @@ class ServerVirtualInterfaceController(object): """ def __init__(self): + self.compute_api = compute.API() self.network_api = network.API() super(ServerVirtualInterfaceController, self).__init__() @@ -61,7 +63,8 @@ class ServerVirtualInterfaceController(object): """Returns a list of VIFs, transformed through entity_maker.""" context = req.environ['nova.context'] - vifs = self.network_api.get_vifs_by_instance(context, server_id) + instance = self.compute_api.get(context, server_id) + vifs = self.network_api.get_vifs_by_instance(context, instance) limited_list = common.limited(vifs, req) res = [entity_maker(context, vif) for vif in limited_list] return {'virtual_interfaces': res} diff --git a/nova/api/openstack/volume/__init__.py b/nova/api/openstack/volume/__init__.py index fb1a97f73..8dbae267b 100644 --- a/nova/api/openstack/volume/__init__.py +++ b/nova/api/openstack/volume/__init__.py @@ -20,85 +20,24 @@ WSGI middleware for OpenStack Volume API. """ -import webob.dec -import webob.exc - import nova.api.openstack from nova.api.openstack.volume import extensions from nova.api.openstack.volume import snapshots from nova.api.openstack.volume import types from nova.api.openstack.volume import volumes from nova.api.openstack.volume import versions -from nova.api.openstack import wsgi -from nova import flags from nova import log as logging -from nova import wsgi as base_wsgi LOG = logging.getLogger('nova.api.openstack.volume') -FLAGS = flags.FLAGS -class APIRouter(base_wsgi.Router): +class APIRouter(nova.api.openstack.APIRouter): """ Routes requests on the OpenStack API to the appropriate controller and method. """ - - @classmethod - def factory(cls, global_config, **local_config): - """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" - return cls() - - def __init__(self, ext_mgr=None): - if ext_mgr is None: - ext_mgr = extensions.ExtensionManager() - - mapper = nova.api.openstack.ProjectMapper() - self.resources = {} - self._setup_routes(mapper) - self._setup_ext_routes(mapper, ext_mgr) - self._setup_extensions(ext_mgr) - super(APIRouter, self).__init__(mapper) - - def _setup_ext_routes(self, mapper, ext_mgr): - for resource in ext_mgr.get_resources(): - LOG.debug(_('Extended resource: %s'), - resource.collection) - - wsgi_resource = wsgi.Resource(resource.controller) - self.resources[resource.collection] = wsgi_resource - kargs = dict( - controller=wsgi_resource, - collection=resource.collection_actions, - member=resource.member_actions) - - if resource.parent: - kargs['parent_resource'] = resource.parent - - mapper.resource(resource.collection, resource.collection, **kargs) - - if resource.custom_routes_fn: - resource.custom_routes_fn(mapper, wsgi_resource) - - def _setup_extensions(self, ext_mgr): - for extension in ext_mgr.get_controller_extensions(): - ext_name = extension.extension.name - collection = extension.collection - controller = extension.controller - - if collection not in self.resources: - LOG.warning(_('Extension %(ext_name)s: Cannot extend ' - 'resource %(collection)s: No such resource') % - locals()) - continue - - LOG.debug(_('Extension %(ext_name)s extending resource: ' - '%(collection)s') % locals()) - - resource = self.resources[collection] - resource.register_actions(controller) - resource.register_extensions(controller) + ExtensionManager = extensions.ExtensionManager def _setup_routes(self, mapper): self.resources['versions'] = versions.create_resource() diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 8fc4f16a1..57026d635 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1181,6 +1181,7 @@ class ComputeManager(manager.SchedulerDependentManager): instance_ref = self.db.instance_get_by_uuid(context, instance_uuid) + compute_utils.notify_usage_exists(instance_ref, current_period=True) self._notify_about_instance_usage(instance_ref, "resize.prep.start") same_host = instance_ref['host'] == FLAGS.host diff --git a/nova/crypto.py b/nova/crypto.py index f2b522d09..a8b10de9b 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -28,12 +28,10 @@ import hashlib import os import shutil import string -import struct import tempfile -import time import utils -import M2Crypto +import Crypto.Cipher.AES gettext.install('nova', unicode=1) @@ -154,33 +152,10 @@ def generate_key_pair(bits=1024): public_key = open(keyfile + '.pub').read() shutil.rmtree(tmpdir) - # code below returns public key in pem format - # key = M2Crypto.RSA.gen_key(bits, 65537, callback=lambda: None) - # private_key = key.as_pem(cipher=None) - # bio = M2Crypto.BIO.MemoryBuffer() - # key.save_pub_key_bio(bio) - # public_key = bio.read() - # public_key, err = execute('ssh-keygen', '-y', '-f', - # '/dev/stdin', private_key) return (private_key, public_key, fingerprint) -def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'): - buf = M2Crypto.BIO.MemoryBuffer(ssl_public_key) - rsa_key = M2Crypto.RSA.load_pub_key_bio(buf) - e, n = rsa_key.pub() - - key_type = 'ssh-rsa' - - key_data = struct.pack('>I', len(key_type)) - key_data += key_type - key_data += '%s%s' % (e, n) - - b64_blob = base64.b64encode(key_data) - return '%s %s %s@%s\n' % (key_type, b64_blob, name, suffix) - - def fetch_crl(project_id): """Get crl file for project.""" if not FLAGS.use_project_ca: @@ -335,72 +310,22 @@ def _sign_csr(csr_text, ca_folder): return (serial, crtfile.read()) -def mkreq(bits, subject='foo', ca=0): - pk = M2Crypto.EVP.PKey() - req = M2Crypto.X509.Request() - rsa = M2Crypto.RSA.gen_key(bits, 65537, callback=lambda: None) - pk.assign_rsa(rsa) - rsa = None # should not be freed here - req.set_pubkey(pk) - req.set_subject(subject) - req.sign(pk, 'sha512') - assert req.verify(pk) - pk2 = req.get_pubkey() - assert req.verify(pk2) - return req, pk - - -def mkcacert(subject='nova', years=1): - req, pk = mkreq(2048, subject, ca=1) - pkey = req.get_pubkey() - sub = req.get_subject() - cert = M2Crypto.X509.X509() - cert.set_serial_number(1) - cert.set_version(2) - # FIXME subject is not set in mkreq yet - cert.set_subject(sub) - t = long(time.time()) + time.timezone - now = M2Crypto.ASN1.ASN1_UTCTIME() - now.set_time(t) - nowPlusYear = M2Crypto.ASN1.ASN1_UTCTIME() - nowPlusYear.set_time(t + (years * 60 * 60 * 24 * 365)) - cert.set_not_before(now) - cert.set_not_after(nowPlusYear) - issuer = M2Crypto.X509.X509_Name() - issuer.C = 'US' - issuer.CN = subject - cert.set_issuer(issuer) - cert.set_pubkey(pkey) - ext = M2Crypto.X509.new_extension('basicConstraints', 'CA:TRUE') - cert.add_ext(ext) - cert.sign(pk, 'sha512') - - # print 'cert', dir(cert) - print cert.as_pem() - print pk.get_rsa().as_pem() - - return cert, pk, pkey - - -def _build_cipher(key, iv, encode=True): +def _build_cipher(key, iv): """Make a 128bit AES CBC encode/decode Cipher object. Padding is handled internally.""" - operation = 1 if encode else 0 - return M2Crypto.EVP.Cipher(alg='aes_128_cbc', key=key, iv=iv, op=operation) + return Crypto.Cipher.AES.new(key, IV=iv) -def encryptor(key, iv=None): +def encryptor(key): """Simple symmetric key encryption.""" key = base64.b64decode(key) - if iv is None: - iv = '\0' * 16 - else: - iv = base64.b64decode(iv) + iv = '\0' * 16 def encrypt(data): - cipher = _build_cipher(key, iv, encode=True) - v = cipher.update(data) - v = v + cipher.final() + cipher = _build_cipher(key, iv) + # Must pad string to multiple of 16 chars + padding = (16 - len(data) % 16) * " " + v = cipher.encrypt(data + padding) del cipher v = base64.b64encode(v) return v @@ -408,19 +333,15 @@ def encryptor(key, iv=None): return encrypt -def decryptor(key, iv=None): +def decryptor(key): """Simple symmetric key decryption.""" key = base64.b64decode(key) - if iv is None: - iv = '\0' * 16 - else: - iv = base64.b64decode(iv) + iv = '\0' * 16 def decrypt(data): data = base64.b64decode(data) - cipher = _build_cipher(key, iv, encode=False) - v = cipher.update(data) - v = v + cipher.final() + cipher = _build_cipher(key, iv) + v = cipher.decrypt(data).rstrip() del cipher return v diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index c61f3ce69..7a0f9b200 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1063,7 +1063,7 @@ def fixed_ip_disassociate(context, address): fixed_ip_ref = fixed_ip_get_by_address(context, address, session=session) - fixed_ip_ref.instance = None + fixed_ip_ref['instance_id'] = None fixed_ip_ref.save(session=session) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/035_secondary_dns.py b/nova/db/sqlalchemy/migrate_repo/versions/035_secondary_dns.py index ce1c3d880..9aa4d3531 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/035_secondary_dns.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/035_secondary_dns.py @@ -26,7 +26,7 @@ def upgrade(migrate_engine): meta.bind = migrate_engine networks = Table('networks', meta, autoload=True) - networks.c.dns.alter(Column('dns1', String(255))) + networks.c.dns.alter(name='dns1') networks.create_column(dns2) @@ -34,5 +34,5 @@ def downgrade(migrate_engine): meta.bind = migrate_engine networks = Table('networks', meta, autoload=True) - networks.c.dns1.alter(Column('dns', String(255))) + networks.c.dns1.alter(name='dns') networks.drop_column(dns2) diff --git a/nova/network/api.py b/nova/network/api.py index 365a1d37b..257c642a9 100644 --- a/nova/network/api.py +++ b/nova/network/api.py @@ -91,11 +91,13 @@ class API(base.Base): {'method': 'get_floating_ips_by_fixed_address', 'args': {'fixed_address': fixed_address}}) - def get_vifs_by_instance(self, context, instance_id): + def get_vifs_by_instance(self, context, instance): + # NOTE(vish): When the db calls are converted to store network + # data by instance_uuid, this should pass uuid instead. return rpc.call(context, FLAGS.network_topic, {'method': 'get_vifs_by_instance', - 'args': {'instance_id': instance_id}}) + 'args': {'instance_id': instance['id']}}) def allocate_floating_ip(self, context, pool=None): """Adds a floating ip to a project from a pool. (allocates)""" diff --git a/nova/network/manager.py b/nova/network/manager.py index 350c01a69..c6d3122b6 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -1704,6 +1704,7 @@ class VlanManager(RPCAllocateFixedIP, FloatingIP, NetworkManager): network_ref['vpn_public_port'], network_ref['vpn_private_address']) if not FLAGS.fake_network: + dev = self.driver.get_dev(network_ref) self.driver.update_dhcp(context, dev, network_ref) if(FLAGS.use_ipv6): self.driver.update_ra(context, dev, network_ref) diff --git a/nova/rpc/__init__.py b/nova/rpc/__init__.py index 1fbd9aead..a6067432e 100644 --- a/nova/rpc/__init__.py +++ b/nova/rpc/__init__.py @@ -48,7 +48,7 @@ def create_connection(new=True): return _get_impl().create_connection(new=new) -def call(context, topic, msg): +def call(context, topic, msg, timeout=None): """Invoke a remote method that returns something. :param context: Information that identifies the user that has made this @@ -59,10 +59,15 @@ def call(context, topic, msg): when the consumer was created with fanout=False. :param msg: This is a dict in the form { "method" : "method_to_invoke", "args" : dict_of_kwargs } + :param timeout: int, number of seconds to use for a response timeout. + If set, this overrides the rpc_response_timeout option. :returns: A dict from the remote method. + + :raises: nova.rpc.common.Timeout if a complete response is not received + before the timeout is reached. """ - return _get_impl().call(context, topic, msg) + return _get_impl().call(context, topic, msg, timeout) def cast(context, topic, msg): @@ -102,7 +107,7 @@ def fanout_cast(context, topic, msg): return _get_impl().fanout_cast(context, topic, msg) -def multicall(context, topic, msg): +def multicall(context, topic, msg, timeout=None): """Invoke a remote method and get back an iterator. In this case, the remote method will be returning multiple values in @@ -117,13 +122,18 @@ def multicall(context, topic, msg): when the consumer was created with fanout=False. :param msg: This is a dict in the form { "method" : "method_to_invoke", "args" : dict_of_kwargs } + :param timeout: int, number of seconds to use for a response timeout. + If set, this overrides the rpc_response_timeout option. :returns: An iterator. The iterator will yield a tuple (N, X) where N is an index that starts at 0 and increases by one for each value returned and X is the Nth value that was returned by the remote method. + + :raises: nova.rpc.common.Timeout if a complete response is not received + before the timeout is reached. """ - return _get_impl().multicall(context, topic, msg) + return _get_impl().multicall(context, topic, msg, timeout) def notify(context, topic, msg): diff --git a/nova/rpc/amqp.py b/nova/rpc/amqp.py index 483100806..0995d9ab8 100644 --- a/nova/rpc/amqp.py +++ b/nova/rpc/amqp.py @@ -262,9 +262,10 @@ class ProxyCallback(object): class MulticallWaiter(object): - def __init__(self, connection): + def __init__(self, connection, timeout): self._connection = connection - self._iterator = connection.iterconsume() + self._iterator = connection.iterconsume( + timeout=timeout or FLAGS.rpc_response_timeout) self._result = None self._done = False self._got_ending = False @@ -307,7 +308,7 @@ def create_connection(new=True): return ConnectionContext(pooled=not new) -def multicall(context, topic, msg): +def multicall(context, topic, msg, timeout): """Make a call that returns multiple times.""" # Can't use 'with' for multicall, as it returns an iterator # that will continue to use the connection. When it's done, @@ -320,15 +321,15 @@ def multicall(context, topic, msg): pack_context(msg, context) conn = ConnectionContext() - wait_msg = MulticallWaiter(conn) + wait_msg = MulticallWaiter(conn, timeout) conn.declare_direct_consumer(msg_id, wait_msg) conn.topic_send(topic, msg) return wait_msg -def call(context, topic, msg): +def call(context, topic, msg, timeout): """Sends a message on a topic and wait for a response.""" - rv = multicall(context, topic, msg) + rv = multicall(context, topic, msg, timeout) # NOTE(vish): return the last result from the multicall rv = list(rv) if not rv: diff --git a/nova/rpc/common.py b/nova/rpc/common.py index ff0577011..70d5d07ba 100644 --- a/nova/rpc/common.py +++ b/nova/rpc/common.py @@ -34,6 +34,9 @@ rpc_opts = [ cfg.IntOpt('rpc_conn_pool_size', default=30, help='Size of RPC connection pool'), + cfg.IntOpt('rpc_response_timeout', + default=3600, + help='Seconds to wait for a response from call or multicall'), ] flags.FLAGS.add_options(rpc_opts) @@ -59,6 +62,15 @@ class RemoteError(exception.NovaException): traceback=traceback) +class Timeout(exception.NovaException): + """Signifies that a timeout has occurred. + + This exception is raised if the rpc_response_timeout is reached while + waiting for a response from the remote side. + """ + message = _("Timeout while waiting on RPC response.") + + class Connection(object): """A connection, returned by rpc.create_connection(). diff --git a/nova/rpc/impl_carrot.py b/nova/rpc/impl_carrot.py index 5750e5989..7ce377941 100644 --- a/nova/rpc/impl_carrot.py +++ b/nova/rpc/impl_carrot.py @@ -522,8 +522,9 @@ class RpcContext(context.RequestContext): self.msg_id = None -def multicall(context, topic, msg): +def multicall(context, topic, msg, timeout=None): """Make a call that returns multiple times.""" + # NOTE(russellb): carrot doesn't support timeouts LOG.debug(_('Making asynchronous call on %s ...'), topic) msg_id = uuid.uuid4().hex msg.update({'_msg_id': msg_id}) @@ -594,9 +595,9 @@ def create_connection(new=True): return Connection.instance(new=new) -def call(context, topic, msg): +def call(context, topic, msg, timeout=None): """Sends a message on a topic and wait for a response.""" - rv = multicall(context, topic, msg) + rv = multicall(context, topic, msg, timeout) # NOTE(vish): return the last result from the multicall rv = list(rv) if not rv: diff --git a/nova/rpc/impl_fake.py b/nova/rpc/impl_fake.py index dc30522b8..6e4d2f6ec 100644 --- a/nova/rpc/impl_fake.py +++ b/nova/rpc/impl_fake.py @@ -18,14 +18,21 @@ queues. Casts will block, but this is very useful for tests. """ import inspect +import signal import sys +import time import traceback +import eventlet + from nova import context +from nova import flags from nova.rpc import common as rpc_common CONSUMERS = {} +FLAGS = flags.FLAGS + class RpcContext(context.RequestContext): def __init__(self, *args, **kwargs): @@ -45,31 +52,49 @@ class Consumer(object): self.topic = topic self.proxy = proxy - def call(self, context, method, args): + def call(self, context, method, args, timeout): node_func = getattr(self.proxy, method) node_args = dict((str(k), v) for k, v in args.iteritems()) - - ctxt = RpcContext.from_dict(context.to_dict()) - try: - rval = node_func(context=ctxt, **node_args) - # Caller might have called ctxt.reply() manually - for (reply, failure) in ctxt._response: - if failure: - raise failure[0], failure[1], failure[2] - yield reply - # if ending not 'sent'...we might have more data to - # return from the function itself - if not ctxt._done: - if inspect.isgenerator(rval): - for val in rval: - yield val - else: - yield rval - except Exception: - exc_info = sys.exc_info() - raise rpc_common.RemoteError(exc_info[0].__name__, - str(exc_info[1]), - ''.join(traceback.format_exception(*exc_info))) + done = eventlet.event.Event() + + def _inner(): + ctxt = RpcContext.from_dict(context.to_dict()) + try: + rval = node_func(context=ctxt, **node_args) + res = [] + # Caller might have called ctxt.reply() manually + for (reply, failure) in ctxt._response: + if failure: + raise failure[0], failure[1], failure[2] + res.append(reply) + # if ending not 'sent'...we might have more data to + # return from the function itself + if not ctxt._done: + if inspect.isgenerator(rval): + for val in rval: + res.append(val) + else: + res.append(rval) + done.send(res) + except Exception: + exc_info = sys.exc_info() + done.send_exception( + rpc_common.RemoteError(exc_info[0].__name__, + str(exc_info[1]), + ''.join(traceback.format_exception(*exc_info)))) + + thread = eventlet.greenthread.spawn(_inner) + + if timeout: + start_time = time.time() + while not done.ready(): + eventlet.greenthread.sleep(1) + cur_time = time.time() + if (cur_time - start_time) > timeout: + thread.kill() + raise rpc_common.Timeout() + + return done.wait() class Connection(object): @@ -99,7 +124,7 @@ def create_connection(new=True): return Connection() -def multicall(context, topic, msg): +def multicall(context, topic, msg, timeout=None): """Make a call that returns multiple times.""" method = msg.get('method') @@ -112,12 +137,12 @@ def multicall(context, topic, msg): except (KeyError, IndexError): return iter([None]) else: - return consumer.call(context, method, args) + return consumer.call(context, method, args, timeout) -def call(context, topic, msg): +def call(context, topic, msg, timeout=None): """Sends a message on a topic and wait for a response.""" - rv = multicall(context, topic, msg) + rv = multicall(context, topic, msg, timeout) # NOTE(vish): return the last result from the multicall rv = list(rv) if not rv: diff --git a/nova/rpc/impl_kombu.py b/nova/rpc/impl_kombu.py index e2c0b9036..50459e5ad 100644 --- a/nova/rpc/impl_kombu.py +++ b/nova/rpc/impl_kombu.py @@ -15,6 +15,7 @@ # under the License. import itertools +import socket import sys import time import uuid @@ -425,7 +426,7 @@ class Connection(object): while True: try: return method(*args, **kwargs) - except self.connection_errors, e: + except (self.connection_errors, socket.timeout), e: pass except Exception, e: # NOTE(comstud): Unfortunately it's possible for amqplib @@ -478,15 +479,20 @@ class Connection(object): return self.ensure(_connect_error, _declare_consumer) - def iterconsume(self, limit=None): + def iterconsume(self, limit=None, timeout=None): """Return an iterator that will consume from all queues/consumers""" info = {'do_consume': True} def _error_callback(exc): - LOG.exception(_('Failed to consume message from queue: %s') % - str(exc)) - info['do_consume'] = True + if isinstance(exc, socket.timeout): + LOG.exception(_('Timed out waiting for RPC response: %s') % + str(exc)) + raise rpc_common.Timeout() + else: + LOG.exception(_('Failed to consume message from queue: %s') % + str(exc)) + info['do_consume'] = True def _consume(): if info['do_consume']: @@ -496,7 +502,7 @@ class Connection(object): queue.consume(nowait=True) queues_tail.consume(nowait=False) info['do_consume'] = False - return self.connection.drain_events() + return self.connection.drain_events(timeout=timeout) for iteration in itertools.count(0): if limit and iteration >= limit: @@ -595,14 +601,14 @@ def create_connection(new=True): return rpc_amqp.create_connection(new) -def multicall(context, topic, msg): +def multicall(context, topic, msg, timeout=None): """Make a call that returns multiple times.""" - return rpc_amqp.multicall(context, topic, msg) + return rpc_amqp.multicall(context, topic, msg, timeout) -def call(context, topic, msg): +def call(context, topic, msg, timeout=None): """Sends a message on a topic and wait for a response.""" - return rpc_amqp.call(context, topic, msg) + return rpc_amqp.call(context, topic, msg, timeout) def cast(context, topic, msg): diff --git a/nova/rpc/impl_qpid.py b/nova/rpc/impl_qpid.py index f4b6b9ffa..353c7e502 100644 --- a/nova/rpc/impl_qpid.py +++ b/nova/rpc/impl_qpid.py @@ -28,6 +28,7 @@ import qpid.messaging.exceptions from nova.common import cfg from nova import flags from nova.rpc import amqp as rpc_amqp +from nova.rpc import common as rpc_common from nova.rpc.common import LOG @@ -338,7 +339,8 @@ class Connection(object): while True: try: return method(*args, **kwargs) - except qpid.messaging.exceptions.ConnectionError, e: + except (qpid.messaging.exceptions.Empty, + qpid.messaging.exceptions.ConnectionError), e: if error_callback: error_callback(e) self.reconnect() @@ -372,15 +374,20 @@ class Connection(object): return self.ensure(_connect_error, _declare_consumer) - def iterconsume(self, limit=None): + def iterconsume(self, limit=None, timeout=None): """Return an iterator that will consume from all queues/consumers""" def _error_callback(exc): - LOG.exception(_('Failed to consume message from queue: %s') % - str(exc)) + if isinstance(exc, qpid.messaging.exceptions.Empty): + LOG.exception(_('Timed out waiting for RPC response: %s') % + str(exc)) + raise rpc_common.Timeout() + else: + LOG.exception(_('Failed to consume message from queue: %s') % + str(exc)) def _consume(): - nxt_receiver = self.session.next_receiver() + nxt_receiver = self.session.next_receiver(timeout=timeout) self._lookup_consumer(nxt_receiver).consume() for iteration in itertools.count(0): @@ -483,14 +490,14 @@ def create_connection(new=True): return rpc_amqp.create_connection(new) -def multicall(context, topic, msg): +def multicall(context, topic, msg, timeout=None): """Make a call that returns multiple times.""" - return rpc_amqp.multicall(context, topic, msg) + return rpc_amqp.multicall(context, topic, msg, timeout) -def call(context, topic, msg): +def call(context, topic, msg, timeout=None): """Sends a message on a topic and wait for a response.""" - return rpc_amqp.call(context, topic, msg) + return rpc_amqp.call(context, topic, msg, timeout) def cast(context, topic, msg): diff --git a/nova/scheduler/distributed_scheduler.py b/nova/scheduler/distributed_scheduler.py index 16e688e6f..2dd7ea619 100644 --- a/nova/scheduler/distributed_scheduler.py +++ b/nova/scheduler/distributed_scheduler.py @@ -22,8 +22,6 @@ Weighing Functions. import json import operator -import M2Crypto - from novaclient import v1_1 as novaclient from novaclient import exceptions as novaclient_exceptions from nova import crypto @@ -43,11 +41,6 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.distributed_scheduler') -class InvalidBlob(exception.NovaException): - message = _("Ill-formed or incorrectly routed 'blob' data sent " - "to instance create request.") - - class DistributedScheduler(driver.Scheduler): """Scheduler that can work across any nova deployment, from simple deployments to multiple nested zones. @@ -185,19 +178,16 @@ class DistributedScheduler(driver.Scheduler): or None if invalid. Broken out for testing. """ decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key) - try: - json_entry = decryptor(blob) - # Extract our WeightedHost values - wh_dict = json.loads(json_entry) - host = wh_dict.get('host', None) - blob = wh_dict.get('blob', None) - zone = wh_dict.get('zone', None) - return least_cost.WeightedHost(wh_dict['weight'], - host_state=host_manager.HostState(host, 'compute'), - blob=blob, zone=zone) - - except M2Crypto.EVP.EVPError: - raise InvalidBlob() + json_entry = decryptor(blob) + + # Extract our WeightedHost values + wh_dict = json.loads(json_entry) + host = wh_dict.get('host', None) + blob = wh_dict.get('blob', None) + zone = wh_dict.get('zone', None) + return least_cost.WeightedHost(wh_dict['weight'], + host_state=host_manager.HostState(host, 'compute'), + blob=blob, zone=zone) def _ask_child_zone_to_create_instance(self, context, weighted_host, request_spec, kwargs): diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 676518a48..02bc77898 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -37,6 +37,7 @@ setattr(__builtin__, '_', lambda x: x) def setup(): + import mox # Fail fast if you don't have mox. Workaround for bug 810424 import os import shutil diff --git a/nova/tests/api/ec2/test_cloud.py b/nova/tests/api/ec2/test_cloud.py index 63cb750d7..ef9d0e8c7 100644 --- a/nova/tests/api/ec2/test_cloud.py +++ b/nova/tests/api/ec2/test_cloud.py @@ -20,18 +20,15 @@ import base64 import copy import functools +import tempfile import os -from M2Crypto import BIO -from M2Crypto import RSA - from nova.api.ec2 import cloud from nova.api.ec2 import ec2utils from nova.api.ec2 import inst_state from nova.compute import power_state from nova.compute import vm_states from nova import context -from nova import crypto from nova import db from nova import exception from nova import flags @@ -1188,16 +1185,21 @@ class CloudTestCase(test.TestCase): def test_key_generation(self): result = self._create_key('test') private_key = result['private_key'] - key = RSA.load_key_string(private_key, callback=lambda: None) - bio = BIO.MemoryBuffer() - public_key = db.key_pair_get(self.context, + + expected = db.key_pair_get(self.context, self.context.user_id, 'test')['public_key'] - key.save_pub_key_bio(bio) - converted = crypto.ssl_pub_to_ssh_pub(bio.read()) + + (fd, fname) = tempfile.mkstemp() + os.write(fd, private_key) + + public_key, err = utils.execute('ssh-keygen', '-e', '-f', fname) + + os.unlink(fname) + # assert key fields are equal - self.assertEqual(public_key.split(" ")[1].strip(), - converted.split(" ")[1].strip()) + self.assertEqual(''.join(public_key.split("\n")[2:-2]), + expected.split(" ")[1].strip()) def test_describe_key_pairs(self): self._create_key('test1') diff --git a/nova/tests/api/openstack/compute/contrib/test_cloudpipe.py b/nova/tests/api/openstack/compute/contrib/test_cloudpipe.py index 27341f199..939957471 100644 --- a/nova/tests/api/openstack/compute/contrib/test_cloudpipe.py +++ b/nova/tests/api/openstack/compute/contrib/test_cloudpipe.py @@ -21,7 +21,7 @@ from lxml import etree from nova.api import auth from nova.api.openstack import compute -from nova.api.openstack.compute import wsgi +from nova.api.openstack import wsgi from nova.api.openstack.compute.contrib import cloudpipe from nova import context from nova import db diff --git a/nova/tests/api/openstack/compute/contrib/test_keypairs.py b/nova/tests/api/openstack/compute/contrib/test_keypairs.py index ff59e174b..fa04e5efb 100644 --- a/nova/tests/api/openstack/compute/contrib/test_keypairs.py +++ b/nova/tests/api/openstack/compute/contrib/test_keypairs.py @@ -46,6 +46,10 @@ def db_key_pair_destroy(context, user_id, name): raise Exception() +def db_key_pair_get(context, user_id, name): + pass + + class KeypairsTest(test.TestCase): def setUp(self): @@ -130,6 +134,16 @@ class KeypairsTest(test.TestCase): self.assertTrue(len(res_dict['keypair']['fingerprint']) > 0) self.assertFalse('private_key' in res_dict['keypair']) + def test_keypair_create_duplicate(self): + self.stubs.Set(db, "key_pair_get", db_key_pair_get) + body = {'keypair': {'name': 'create_duplicate'}} + req = webob.Request.blank('/v2/fake/os-keypairs') + req.method = 'POST' + req.body = json.dumps(body) + req.headers['Content-Type'] = 'application/json' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 409) + def test_keypair_import_bad_key(self): body = { 'keypair': { diff --git a/nova/tests/api/openstack/compute/contrib/test_virtual_interfaces.py b/nova/tests/api/openstack/compute/contrib/test_virtual_interfaces.py index d9f41f07b..6885e929f 100644 --- a/nova/tests/api/openstack/compute/contrib/test_virtual_interfaces.py +++ b/nova/tests/api/openstack/compute/contrib/test_virtual_interfaces.py @@ -20,12 +20,20 @@ import webob from nova.api.openstack.compute.contrib import virtual_interfaces from nova.api.openstack import wsgi +from nova import compute from nova import network from nova import test from nova.tests.api.openstack import fakes -def get_vifs_by_instance(self, context, server_id): +FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + + +def compute_api_get(self, context, instance_id): + return dict(uuid=FAKE_UUID, id=instance_id, instance_type_id=1, host='bob') + + +def get_vifs_by_instance(self, context, instance_id): return [{'uuid': '00000000-0000-0000-0000-00000000000000000', 'address': '00-00-00-00-00-00'}, {'uuid': '11111111-1111-1111-1111-11111111111111111', @@ -37,6 +45,8 @@ class ServerVirtualInterfaceTest(test.TestCase): def setUp(self): super(ServerVirtualInterfaceTest, self).setUp() self.controller = virtual_interfaces.ServerVirtualInterfaceController() + self.stubs.Set(compute.api.API, "get", + compute_api_get) self.stubs.Set(network.api.API, "get_vifs_by_instance", get_vifs_by_instance) diff --git a/nova/tests/api/openstack/compute/test_api.py b/nova/tests/api/openstack/compute/test_api.py index da23ee711..dbb187c40 100644 --- a/nova/tests/api/openstack/compute/test_api.py +++ b/nova/tests/api/openstack/compute/test_api.py @@ -24,7 +24,7 @@ from webob import Request from nova import test from nova.api import openstack as openstack_api -from nova.api.openstack.compute import wsgi +from nova.api.openstack import wsgi from nova.tests.api.openstack import fakes diff --git a/nova/tests/rpc/common.py b/nova/tests/rpc/common.py index dc8aafcfe..c41375ace 100644 --- a/nova/tests/rpc/common.py +++ b/nova/tests/rpc/common.py @@ -19,9 +19,13 @@ Unit Tests for remote procedure calls shared between all implementations """ +import time + +import nose + from nova import context from nova import log as logging -from nova.rpc.common import RemoteError +from nova.rpc.common import RemoteError, Timeout from nova import test @@ -29,13 +33,14 @@ LOG = logging.getLogger('nova.tests.rpc') class _BaseRpcTestCase(test.TestCase): - def setUp(self): + def setUp(self, supports_timeouts=True): super(_BaseRpcTestCase, self).setUp() self.conn = self.rpc.create_connection(True) self.receiver = TestReceiver() self.conn.create_consumer('test', self.receiver, False) self.conn.consume_in_thread() self.context = context.get_admin_context() + self.supports_timeouts = supports_timeouts def tearDown(self): self.conn.close() @@ -162,6 +167,28 @@ class _BaseRpcTestCase(test.TestCase): conn.close() self.assertEqual(value, result) + def test_call_timeout(self): + """Make sure rpc.call will time out""" + if not self.supports_timeouts: + raise nose.SkipTest(_("RPC backend does not support timeouts")) + + value = 42 + self.assertRaises(Timeout, + self.rpc.call, + self.context, + 'test', + {"method": "block", + "args": {"value": value}}, timeout=1) + try: + self.rpc.call(self.context, + 'test', + {"method": "block", + "args": {"value": value}}, + timeout=1) + self.fail("should have thrown Timeout") + except Timeout as exc: + pass + class TestReceiver(object): """Simple Proxy class so the consumer has methods to call. @@ -205,3 +232,7 @@ class TestReceiver(object): def fail(context, value): """Raises an exception with the value sent in.""" raise Exception(value) + + @staticmethod + def block(context, value): + time.sleep(2) diff --git a/nova/tests/rpc/test_carrot.py b/nova/tests/rpc/test_carrot.py index 2523810d8..153747da2 100644 --- a/nova/tests/rpc/test_carrot.py +++ b/nova/tests/rpc/test_carrot.py @@ -30,7 +30,7 @@ LOG = logging.getLogger('nova.tests.rpc') class RpcCarrotTestCase(common._BaseRpcTestCase): def setUp(self): self.rpc = impl_carrot - super(RpcCarrotTestCase, self).setUp() + super(RpcCarrotTestCase, self).setUp(supports_timeouts=False) def tearDown(self): super(RpcCarrotTestCase, self).tearDown() diff --git a/nova/tests/rpc/test_qpid.py b/nova/tests/rpc/test_qpid.py index 0417674b8..9e318fbfd 100644 --- a/nova/tests/rpc/test_qpid.py +++ b/nova/tests/rpc/test_qpid.py @@ -221,21 +221,25 @@ class RpcQpidTestCase(test.TestCase): self.mock_session.sender(send_addr).AndReturn(self.mock_sender) self.mock_sender.send(mox.IgnoreArg()) - self.mock_session.next_receiver().AndReturn(self.mock_receiver) + self.mock_session.next_receiver(timeout=mox.IsA(int)).AndReturn( + self.mock_receiver) self.mock_receiver.fetch().AndReturn(qpid.messaging.Message( {"result": "foo", "failure": False, "ending": False})) if multi: - self.mock_session.next_receiver().AndReturn(self.mock_receiver) + self.mock_session.next_receiver(timeout=mox.IsA(int)).AndReturn( + self.mock_receiver) self.mock_receiver.fetch().AndReturn( qpid.messaging.Message( {"result": "bar", "failure": False, "ending": False})) - self.mock_session.next_receiver().AndReturn(self.mock_receiver) + self.mock_session.next_receiver(timeout=mox.IsA(int)).AndReturn( + self.mock_receiver) self.mock_receiver.fetch().AndReturn( qpid.messaging.Message( {"result": "baz", "failure": False, "ending": False})) - self.mock_session.next_receiver().AndReturn(self.mock_receiver) + self.mock_session.next_receiver(timeout=mox.IsA(int)).AndReturn( + self.mock_receiver) self.mock_receiver.fetch().AndReturn(qpid.messaging.Message( {"failure": False, "ending": True})) self.mock_session.close() diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 0ce9a4ea5..400e6949f 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -171,12 +171,12 @@ class BaseTestCase(test.TestCase): context = self.context.elevated() inst = {} inst['name'] = 'm1.small' - inst['memory_mb'] = '1024' - inst['vcpus'] = '1' - inst['root_gb'] = '20' - inst['ephemeral_gb'] = '10' + inst['memory_mb'] = 1024 + inst['vcpus'] = 1 + inst['root_gb'] = 20 + inst['ephemeral_gb'] = 10 inst['flavorid'] = '1' - inst['swap'] = '2048' + inst['swap'] = 2048 inst['rxtx_factor'] = 1 inst.update(params) return db.instance_type_create(context, inst)['id'] @@ -1036,12 +1036,15 @@ class ComputeTestCase(BaseTestCase): instance_uuid, 'pre-migrating') - self.assertEquals(len(test_notifier.NOTIFICATIONS), 2) + self.assertEquals(len(test_notifier.NOTIFICATIONS), 3) msg = test_notifier.NOTIFICATIONS[0] self.assertEquals(msg['event_type'], - 'compute.instance.resize.prep.start') + 'compute.instance.exists') msg = test_notifier.NOTIFICATIONS[1] self.assertEquals(msg['event_type'], + 'compute.instance.resize.prep.start') + msg = test_notifier.NOTIFICATIONS[2] + self.assertEquals(msg['event_type'], 'compute.instance.resize.prep.end') self.assertEquals(msg['priority'], 'INFO') payload = msg['payload'] diff --git a/nova/tests/test_crypto.py b/nova/tests/test_crypto.py index b5f70ea72..2bfb345b9 100644 --- a/nova/tests/test_crypto.py +++ b/nova/tests/test_crypto.py @@ -21,7 +21,6 @@ import shutil import tempfile import mox -from M2Crypto import X509 from nova import crypto from nova import db @@ -48,17 +47,6 @@ class SymmetricKeyTestCase(test.TestCase): self.assertEquals(plain_text, plain) - # IV supplied ... - iv = '562e17996d093d28ddb3ba695a2e6f58' - encrypt = crypto.encryptor(key, iv) - cipher_text = encrypt(plain_text) - self.assertNotEquals(plain_text, cipher_text) - - decrypt = crypto.decryptor(key, iv) - plain = decrypt(cipher_text) - - self.assertEquals(plain_text, plain) - class X509Test(test.TestCase): def test_can_generate_x509(self): @@ -69,18 +57,19 @@ class X509Test(test.TestCase): _key, cert_str = crypto.generate_x509_cert('fake', 'fake') project_cert = crypto.fetch_ca(project_id='fake') - cloud_cert = crypto.fetch_ca() - # TODO(vish): This will need to be replaced with something else - # when we remove M2Crypto - signed_cert = X509.load_cert_string(cert_str) - project_cert = X509.load_cert_string(project_cert) - cloud_cert = X509.load_cert_string(cloud_cert) - self.assertTrue(signed_cert.verify(project_cert.get_pubkey())) - - if not FLAGS.use_project_ca: - self.assertTrue(signed_cert.verify(cloud_cert.get_pubkey())) - else: - self.assertFalse(signed_cert.verify(cloud_cert.get_pubkey())) + + signed_cert_file = os.path.join(tmpdir, "signed") + with open(signed_cert_file, 'w') as keyfile: + keyfile.write(cert_str) + + project_cert_file = os.path.join(tmpdir, "project") + with open(project_cert_file, 'w') as keyfile: + keyfile.write(project_cert) + + enc, err = utils.execute('openssl', 'verify', '-CAfile', + project_cert_file, '-verbose', signed_cert_file) + self.assertFalse(err) + finally: shutil.rmtree(tmpdir) diff --git a/nova/utils.py b/nova/utils.py index 0d3067fd3..d1968ac4d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -45,6 +45,7 @@ from eventlet import semaphore from eventlet.green import subprocess import netaddr +from nova.common import cfg from nova import exception from nova import flags from nova import log as logging @@ -56,6 +57,11 @@ PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" FLAGS = flags.FLAGS +FLAGS.add_option( + cfg.BoolOpt('disable_process_locking', default=False, + help='Whether to disable inter-process locks')) + + def import_class(import_str): """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') @@ -764,14 +770,6 @@ else: _semaphores = {} -class _NoopContextManager(object): - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def synchronized(name, external=False): """Synchronization decorator. @@ -816,21 +814,19 @@ def synchronized(name, external=False): LOG.debug(_('Got semaphore "%(lock)s" for method ' '"%(method)s"...' % {'lock': name, 'method': f.__name__})) - if external: + if external and not FLAGS.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...' % {'lock': name, 'method': f.__name__})) lock_file_path = os.path.join(FLAGS.lock_path, 'nova-%s' % name) lock = lockfile.FileLock(lock_file_path) - else: - lock = _NoopContextManager() - - with lock: - if external: + with lock: LOG.debug(_('Got file lock "%(lock)s" for ' 'method "%(method)s"...' % {'lock': name, 'method': f.__name__})) + retval = f(*args, **kwargs) + else: retval = f(*args, **kwargs) # If no-one else is waiting for it, delete it. diff --git a/nova/virt/disk/api.py b/nova/virt/disk/api.py index 8f477bcea..223f08a2f 100644 --- a/nova/virt/disk/api.py +++ b/nova/virt/disk/api.py @@ -109,12 +109,10 @@ def extend(image, size): class _DiskImage(object): """Provide operations on a disk image file.""" - def __init__(self, image, partition=None, use_cow=False, - disable_auto_fsck=False, mount_dir=None): + def __init__(self, image, partition=None, use_cow=False, mount_dir=None): # These passed to each mounter self.image = image self.partition = partition - self.disable_auto_fsck = disable_auto_fsck self.mount_dir = mount_dir # Internal @@ -164,7 +162,6 @@ class _DiskImage(object): mounter_cls = self._handler_class(h) mounter = mounter_cls(image=self.image, partition=self.partition, - disable_auto_fsck=self.disable_auto_fsck, mount_dir=self.mount_dir) if mounter.do_mount(): self._mounter = mounter @@ -191,7 +188,7 @@ class _DiskImage(object): # Public module functions def inject_data(image, key=None, net=None, metadata=None, - partition=None, use_cow=False, disable_auto_fsck=True): + partition=None, use_cow=False): """Injects a ssh key and optionally net data into a disk image. it will mount the image as a fully partitioned disk and attempt to inject @@ -200,8 +197,7 @@ def inject_data(image, key=None, net=None, metadata=None, If partition is not specified it mounts the image as a single partition. """ - img = _DiskImage(image=image, partition=partition, use_cow=use_cow, - disable_auto_fsck=disable_auto_fsck) + img = _DiskImage(image=image, partition=partition, use_cow=use_cow) if img.mount(): try: inject_data_into_fs(img.mount_dir, key, net, metadata, @@ -212,11 +208,9 @@ def inject_data(image, key=None, net=None, metadata=None, raise exception.Error(img.errors) -def inject_files(image, files, partition=None, use_cow=False, - disable_auto_fsck=True): +def inject_files(image, files, partition=None, use_cow=False): """Injects arbitrary files into a disk image""" - img = _DiskImage(image=image, partition=partition, use_cow=use_cow, - disable_auto_fsck=disable_auto_fsck) + img = _DiskImage(image=image, partition=partition, use_cow=use_cow) if img.mount(): try: for (path, contents) in files: @@ -300,8 +294,15 @@ def _inject_key_into_fs(key, fs, execute=None): utils.execute('chown', 'root', sshdir, run_as_root=True) utils.execute('chmod', '700', sshdir, run_as_root=True) keyfile = os.path.join(sshdir, 'authorized_keys') + key_data = [ + '\n', + '# The following ssh key was injected by Nova', + '\n', + key.strip(), + '\n', + ] utils.execute('tee', '-a', keyfile, - process_input='\n' + key.strip() + '\n', run_as_root=True) + process_input=''.join(key_data), run_as_root=True) def _inject_net_into_fs(net, fs, execute=None): diff --git a/nova/virt/disk/mount.py b/nova/virt/disk/mount.py index a7ea77097..3971e7ef3 100644 --- a/nova/virt/disk/mount.py +++ b/nova/virt/disk/mount.py @@ -30,13 +30,11 @@ class Mount(object): to be called in that order. """ - def __init__(self, image, mount_dir, partition=None, - disable_auto_fsck=False): + def __init__(self, image, mount_dir, partition=None): # Input self.image = image self.partition = partition - self.disable_auto_fsck = disable_auto_fsck self.mount_dir = mount_dir # Output @@ -84,16 +82,6 @@ class Mount(object): self.mapped_device = self.device self.mapped = True - # This is an orthogonal operation - # which only needs to be done once - if self.disable_auto_fsck and self.mapped: - self.disable_auto_fsck = False - # attempt to set ext[234] so that it doesn't auto-fsck - _out, err = utils.trycmd('tune2fs', '-c', 0, '-i', 0, - self.mapped_device, run_as_root=True) - if err: - LOG.info(_('Failed to disable fs check: %s') % err) - return self.mapped def unmap_dev(self): diff --git a/nova/virt/interfaces.template b/nova/virt/interfaces.template index e527cf35c..ad8c1bcdb 100644 --- a/nova/virt/interfaces.template +++ b/nova/virt/interfaces.template @@ -1,3 +1,5 @@ +# Injected by Nova on instance boot +# # This file describes the network interfaces available on your system # and how to activate them. For more information, see interfaces(5). diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py index 905480f18..4455dacec 100644 --- a/nova/virt/libvirt/connection.py +++ b/nova/virt/libvirt/connection.py @@ -942,7 +942,7 @@ class LibvirtConnection(driver.ComputeDriver): inst_type_id = inst['instance_type_id'] inst_type = instance_types.get_instance_type(inst_type_id) - if inst_type['name'] == 'm1.tiny' or suffix == '.rescue': + if size == 0 or suffix == '.rescue': size = None root_fname += "_sm" else: @@ -1076,11 +1076,9 @@ class LibvirtConnection(driver.ComputeDriver): if config_drive: # Should be True or None by now. injection_path = basepath('disk.config') img_id = 'config-drive' - disable_auto_fsck = False else: injection_path = basepath('disk') img_id = inst.image_ref - disable_auto_fsck = True for injection in ('metadata', 'key', 'net'): if locals()[injection]: @@ -1090,8 +1088,7 @@ class LibvirtConnection(driver.ComputeDriver): try: disk.inject_data(injection_path, key, net, metadata, partition=target_partition, - use_cow=FLAGS.use_cow_images, - disable_auto_fsck=disable_auto_fsck) + use_cow=FLAGS.use_cow_images) except Exception as e: # This could be a windows image, or a vmdk format disk diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index f3ff9a22d..c7f713ea2 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -20,7 +20,9 @@ Management class for VM-related functions (spawn, reboot, etc). """ import base64 +import binascii import json +import os import pickle import random import sys @@ -28,7 +30,6 @@ import time import uuid from eventlet import greenthread -import M2Crypto from nova.common import cfg from nova.compute import api as compute @@ -250,6 +251,14 @@ class VMOps(object): """Spawn a rescue instance.""" self.spawn(context, instance, image_meta, network_info) + def _generate_hostname(self, instance): + """Generate the instance's hostname.""" + hostname = instance["hostname"] + if getattr(instance, "_rescue", False): + hostname = "RESCUE-%s" % hostname + + return hostname + def _create_vm(self, context, instance, vdis, network_info, image_meta): """Create VM instance.""" instance_name = instance.name @@ -338,7 +347,9 @@ class VMOps(object): self.create_vifs(vm_ref, instance, network_info) self.inject_network_info(instance, network_info, vm_ref) - self.inject_hostname(instance, vm_ref, instance['hostname']) + + hostname = self._generate_hostname(instance) + self.inject_hostname(instance, vm_ref, hostname) return vm_ref @@ -501,7 +512,7 @@ class VMOps(object): # if the guest agent is not available, configure the # instance, but skip the admin password configuration - no_agent = version is None or agent_build is None + no_agent = version is None self._configure_instance(ctx, instance, vm_ref, no_agent) def _handle_spawn_error(self, vdis, spawn_error): @@ -1763,7 +1774,6 @@ class VMOps(object): """Removes filters for each VIF of the specified instance.""" self.firewall_driver.unfilter_instance(instance_ref, network_info=network_info) - ######################################################################## class SimpleDH(object): @@ -1776,51 +1786,35 @@ class SimpleDH(object): as it uses that to handle the encryption and decryption. If openssl is not available, a RuntimeError will be raised. """ - def __init__(self, prime=None, base=None, secret=None): - """ - You can specify the values for prime and base if you wish; - otherwise, reasonable default values will be used. - """ - if prime is None: - self._prime = 162259276829213363391578010288127 - else: - self._prime = prime - if base is None: - self._base = 5 - else: - self._base = base - self._shared = self._public = None + def __init__(self): + self._prime = 162259276829213363391578010288127 + self._base = 5 + self._public = None + self._shared = None + self.generate_private() - self._dh = M2Crypto.DH.set_params( - self.dec_to_mpi(self._prime), - self.dec_to_mpi(self._base)) - self._dh.gen_key() - self._public = self.mpi_to_dec(self._dh.pub) + def generate_private(self): + self._private = int(binascii.hexlify(os.urandom(10)), 16) + return self._private def get_public(self): + self._public = self.mod_exp(self._base, self._private, self._prime) return self._public def compute_shared(self, other): - self._shared = self.bin_to_dec( - self._dh.compute_key(self.dec_to_mpi(other))) + self._shared = self.mod_exp(other, self._private, self._prime) return self._shared - def mpi_to_dec(self, mpi): - bn = M2Crypto.m2.mpi_to_bn(mpi) - hexval = M2Crypto.m2.bn_to_hex(bn) - dec = int(hexval, 16) - return dec - - def bin_to_dec(self, binval): - bn = M2Crypto.m2.bin_to_bn(binval) - hexval = M2Crypto.m2.bn_to_hex(bn) - dec = int(hexval, 16) - return dec - - def dec_to_mpi(self, dec): - bn = M2Crypto.m2.dec_to_bn('%s' % dec) - mpi = M2Crypto.m2.bn_to_mpi(bn) - return mpi + @staticmethod + def mod_exp(num, exp, mod): + """Efficient implementation of (num ** exp) % mod""" + result = 1 + while exp > 0: + if (exp & 1) == 1: + result = (result * num) % mod + exp = exp >> 1 + num = (num * num) % mod + return result def _run_ssl(self, text, decrypt=False): cmd = ['openssl', 'aes-128-cbc', '-A', '-a', '-pass', diff --git a/tools/install_venv.py b/tools/install_venv.py index 0a9fe8252..90b4e60aa 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -27,7 +27,6 @@ import optparse import os import subprocess import sys -import platform ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -88,9 +87,6 @@ class Distro(object): ' requires virtualenv, please install it using your' ' favorite package management tool') - def install_m2crypto(self): - pip_install('M2Crypto') - def post_process(self): """Any distribution-specific post-processing gets done here. @@ -99,18 +95,6 @@ class Distro(object): pass -class UbuntuOneiric(Distro): - """Oneiric specific installation steps""" - - def install_m2crypto(self): - """ - The pip installed version of m2crypto has problems on oneiric - """ - print "Attempting to install 'python-m2crypto' via apt-get" - run_command(['sudo', 'apt-get', 'install', '-y', - "python-m2crypto"]) - - class Fedora(Distro): """This covers all Fedora-based distributions. @@ -136,14 +120,6 @@ class Fedora(Distro): super(Fedora, self).install_virtualenv() - # - # pip install M2Crypto fails on Fedora because of - # weird differences with OpenSSL headers - # - def install_m2crypto(self): - if not self.check_pkg('m2crypto'): - self.yum_install('m2crypto') - def post_process(self): """Workaround for a bug in eventlet. @@ -169,8 +145,6 @@ def get_distro(): if os.path.exists('/etc/fedora-release') or \ os.path.exists('/etc/redhat-release'): return Fedora() - elif platform.linux_distribution()[2] == 'oneiric': - return UbuntuOneiric() else: return Distro() @@ -215,8 +189,6 @@ def install_dependencies(venv=VENV): pip_install('-r', PIP_REQUIRES) - get_distro().install_m2crypto() - # Tell the virtual env how to "import nova" pthfile = os.path.join(venv, "lib", PY_VERSION, "site-packages", "nova.pth") diff --git a/tools/pip-requires b/tools/pip-requires index 8a590c74a..bc7d057bc 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -31,3 +31,4 @@ coverage nosexcover paramiko feedparser +pycrypto |