From ceaa125915c4f1432ba802396a84a6204a6678df Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 10:30:06 -0500 Subject: added bin/nova-listinstances, which is mostly just a duplication of euca-describe-instances but doesn't go through the API. --- bin/nova-listinstances | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 bin/nova-listinstances diff --git a/bin/nova-listinstances b/bin/nova-listinstances new file mode 100755 index 000000000..2f8ff28f9 --- /dev/null +++ b/bin/nova-listinstances @@ -0,0 +1,19 @@ +#!/usr/bin/python + +# +# Duplicates the functionality of euca-describe-instances, but doesn't require +# going through the API. Does a direct query to the datastore. This is +# mostly a test program written for the scheduler +# + +from nova.compute import model + +data_needed = ['image_id', 'memory_kb', 'local_gb', 'node_name', 'vcpus'] + +instances = model.InstanceDirectory().all + +for instance in instances: + print 'Instance: %s' % instance['instance_id'] + for x in data_needed: + print ' %s: %s' % (x, instance[x]) + -- cgit From 5f41e9c764d2d064590e61018e655b9da8b17e9c Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 12:52:55 -0500 Subject: compute nodes should store total memory and disk space available for VMs --- nova/compute/model.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nova/compute/model.py b/nova/compute/model.py index 212830d3c..3913d8738 100644 --- a/nova/compute/model.py +++ b/nova/compute/model.py @@ -53,10 +53,14 @@ from nova import utils FLAGS = flags.FLAGS - +flags.DEFINE_integer('total_memory_mb', 1000, + 'amount of memory a node has for VMs in MB') +flags.DEFINE_integer('total_disk_gb', 1000, + 'amount of disk space a node has for VMs in GB') # TODO(todd): Implement this at the class level for Instance class InstanceDirectory(object): + """an api for interacting with the global state of instances""" def get(self, instance_id): @@ -200,6 +204,8 @@ class Daemon(datastore.BasicModel): def default_state(self): return {"hostname": self.hostname, "binary": self.binary, + "total_memory_mb": FLAGS.total_memory_mb, + "total_disk_gb": FLAGS.total_disk_gb, "updated_at": utils.isotime() } -- cgit From bf0ea2deaf24419d85cae684e0700241e4c03f8c Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 12:54:13 -0500 Subject: remove extra line accidentally added --- nova/compute/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/compute/model.py b/nova/compute/model.py index 3913d8738..edd49a5c0 100644 --- a/nova/compute/model.py +++ b/nova/compute/model.py @@ -60,7 +60,6 @@ flags.DEFINE_integer('total_disk_gb', 1000, # TODO(todd): Implement this at the class level for Instance class InstanceDirectory(object): - """an api for interacting with the global state of instances""" def get(self, instance_id): -- cgit From f42be0875d06a5d3ec0d5304d2f01a41b1f6a477 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 16:11:59 -0500 Subject: almost there on random scheduler. not pushing to correct compute node topic, yet, apparently... --- bin/nova-scheduler | 32 +++++++++++++++++ nova/endpoint/cloud.py | 2 +- nova/flags.py | 1 + nova/scheduler/__init__.py | 33 ++++++++++++++++++ nova/scheduler/service.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100755 bin/nova-scheduler create mode 100644 nova/scheduler/__init__.py create mode 100644 nova/scheduler/service.py diff --git a/bin/nova-scheduler b/bin/nova-scheduler new file mode 100755 index 000000000..1ad41bbd3 --- /dev/null +++ b/bin/nova-scheduler @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" + Twistd daemon for the nova scheduler nodes. +""" + +from nova import twistd +from nova.scheduler import service + + +if __name__ == '__main__': + twistd.serve(__file__) + +if __name__ == '__builtin__': + application = service.SchedulerService.create() diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 0ee278f84..a808e54c3 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -576,7 +576,7 @@ class CloudController(object): inst['private_dns_name'] = str(address) # TODO: allocate expresses on the router node inst.save() - rpc.cast(FLAGS.compute_topic, + rpc.cast(FLAGS.scheduler_topic, {"method": "run_instance", "args": {"instance_id" : inst.instance_id}}) logging.debug("Casting to node for %s's instance with IP of %s" % diff --git a/nova/flags.py b/nova/flags.py index f35f5fa10..7f92e3f70 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -41,6 +41,7 @@ DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') #DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') +DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on') DEFINE_string('network_topic', 'network', 'the topic network nodes listen on') diff --git a/nova/scheduler/__init__.py b/nova/scheduler/__init__.py new file mode 100644 index 000000000..516ea61bc --- /dev/null +++ b/nova/scheduler/__init__.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`nova.scheduler` -- Scheduler Nodes +===================================================== + +.. automodule:: nova.scheduler + :platform: Unix + :synopsis: Daemon that picks a host for a VM instance. +.. moduleauthor:: Jesse Andrews +.. moduleauthor:: Devin Carlen +.. moduleauthor:: Vishvananda Ishaya +.. moduleauthor:: Joshua McKenty +.. moduleauthor:: Manish Singh +.. moduleauthor:: Andy Smith +.. moduleauthor:: Chris Behrens +""" diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py new file mode 100644 index 000000000..aca5b5db6 --- /dev/null +++ b/nova/scheduler/service.py @@ -0,0 +1,87 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Scheduler Service +""" + +import logging +import random +import sys +from twisted.internet import defer +from twisted.internet import task + +from nova import exception +from nova import flags +from nova import process +from nova import rpc +from nova import service +from nova import utils +from nova.compute import model +from nova.datastore import Redis + +FLAGS = flags.FLAGS + + +class SchedulerService(service.Service): + """ + Manages the running instances. + """ + def __init__(self): + super(SchedulerService, self).__init__() + self.instdir = model.InstanceDirectory() + + def noop(self): + """ simple test of an AMQP message call """ + return defer.succeed('PONG') + + @defer.inlineCallbacks + def report_state(self, nodename, daemon): + # TODO(termie): make this pattern be more elegant. -todd + try: + record = model.Daemon(nodename, daemon) + record.heartbeat() + if getattr(self, "model_disconnected", False): + self.model_disconnected = False + logging.error("Recovered model server connection!") + + except model.ConnectionError, ex: + if not getattr(self, "model_disconnected", False): + self.model_disconnected = True + logging.exception("model server went away") + yield + + @property + def compute_identifiers(self): + return [identifier for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] + + def pick_node(self, instance_id, **_kwargs): + identifiers = self.compute_identifiers + return identifiers[int(random.random() * len(identifiers))].split(':')[0] + + @exception.wrap_exception + def run_instance(self, instance_id, **_kwargs): + node = self.pick_node(instance_id, **_kwargs) + + rpc.cast('%s:%s' % (FLAGS.compute_topic, node), + {"method": "run_instance", + "args": {"instance_id" : instance_id}}) + logging.debug("Casting to node %s for instance %s" % + (node, instance_id)) + + -- cgit From fd5000e70a724d9bea69754d4e7b99630d2d5ea2 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 16:19:21 -0500 Subject: compute topic for a node is compute.node not compute:node! --- nova/scheduler/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index aca5b5db6..2875dd554 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -78,7 +78,7 @@ class SchedulerService(service.Service): def run_instance(self, instance_id, **_kwargs): node = self.pick_node(instance_id, **_kwargs) - rpc.cast('%s:%s' % (FLAGS.compute_topic, node), + rpc.cast('%s.%s' % (FLAGS.compute_topic, node), {"method": "run_instance", "args": {"instance_id" : instance_id}}) logging.debug("Casting to node %s for instance %s" % -- cgit From c7e5faf0aa97ae8f0894b19a9f851d3868e578c3 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 5 Aug 2010 15:10:56 -0700 Subject: fixed doc string --- nova/scheduler/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 2875dd554..46e541a4f 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -40,7 +40,7 @@ FLAGS = flags.FLAGS class SchedulerService(service.Service): """ - Manages the running instances. + Picks nodes for instances to run. """ def __init__(self): super(SchedulerService, self).__init__() -- cgit From 869f33c9bf4a70e2a4ca4d1034114890d458f983 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 6 Aug 2010 14:40:24 -0500 Subject: Start breaking out scheduler classes... --- nova/scheduler/service.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 46e541a4f..3a226322f 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -23,6 +23,7 @@ Scheduler Service import logging import random import sys +import time from twisted.internet import defer from twisted.internet import task @@ -36,11 +37,12 @@ from nova.compute import model from nova.datastore import Redis FLAGS = flags.FLAGS - +flags.DEFINE_integer('node_down_time', 60, + 'seconds without heartbeat that determines a compute node to be down') class SchedulerService(service.Service): """ - Picks nodes for instances to run. + Manages the running instances. """ def __init__(self): super(SchedulerService, self).__init__() @@ -67,12 +69,20 @@ class SchedulerService(service.Service): yield @property - def compute_identifiers(self): - return [identifier for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] + def compute_nodes(self): + return [identifier.split(':')[0] for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] + + def compute_node_is_up(self, node): + time_str = Redis.instance().hget('%s:%s:%s' % ('daemon', node, 'nova-compute'), 'updated_at') + return(time_str and + (time.time() - (int(time.mktime(time.strptime(time_str.replace('Z', 'UTC'), '%Y-%m-%dT%H:%M:%S%Z'))) - time.timezone) < FLAGS.node_down_time)) + + def compute_nodes_up(self): + return [node for node in self.compute_nodes if self.compute_node_is_up(node)] def pick_node(self, instance_id, **_kwargs): - identifiers = self.compute_identifiers - return identifiers[int(random.random() * len(identifiers))].split(':')[0] + """You DEFINITELY want to define this in your subclass""" + raise NotImplementedError("Your subclass should define pick_node") @exception.wrap_exception def run_instance(self, instance_id, **_kwargs): @@ -84,4 +94,16 @@ class SchedulerService(service.Service): logging.debug("Casting to node %s for instance %s" % (node, instance_id)) +class RandomService(SchedulerService): + """ + Implements SchedulerService as a random node selector + """ + + def __init__(self): + super(RandomService, self).__init__() + + def pick_node(self, instance_id, **_kwargs): + nodes = self.compute_nodes_up() + return nodes[int(random.random() * len(nodes))] + -- cgit From 6c4e257b6df94b8c8e0745e8c3d0701293ae588e Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 6 Aug 2010 17:40:10 -0500 Subject: Moved Scheduler classes into scheduler.py. Created a way to specify scheduler class that the SchedulerService uses... --- nova/scheduler/scheduler.py | 82 +++++++++++++++++++++++++++++++++++++++++++++ nova/scheduler/service.py | 52 ++++++++-------------------- 2 files changed, 96 insertions(+), 38 deletions(-) create mode 100644 nova/scheduler/scheduler.py diff --git a/nova/scheduler/scheduler.py b/nova/scheduler/scheduler.py new file mode 100644 index 000000000..0da7b95cf --- /dev/null +++ b/nova/scheduler/scheduler.py @@ -0,0 +1,82 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Scheduler Classes +""" + +import logging +import random +import sys +import time + +from nova import exception +from nova import flags +from nova.datastore import Redis + +FLAGS = flags.FLAGS +flags.DEFINE_integer('node_down_time', + 60, + 'seconds without heartbeat that determines a compute node to be down') + + +class Scheduler(object): + """ + The base class that all Scheduler clases should inherit from + """ + + @property + def compute_nodes(self): + return [identifier.split(':')[0] for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] + + def compute_node_is_up(self, node): + time_str = Redis.instance().hget('%s:%s:%s' % ('daemon', node, 'nova-compute'), 'updated_at') + # Would be a lot easier if we stored heartbeat time in epoch :) + return(time_str and + (time.time() - (int(time.mktime(time.strptime(time_str.replace('Z', 'UTC'), '%Y-%m-%dT%H:%M:%S%Z'))) - time.timezone) < FLAGS.node_down_time)) + + def compute_nodes_up(self): + return [node for node in self.compute_nodes if self.compute_node_is_up(node)] + + def pick_node(self, instance_id, **_kwargs): + """You DEFINITELY want to define this in your subclass""" + raise NotImplementedError("Your subclass should define pick_node") + +class RandomScheduler(Scheduler): + """ + Implements Scheduler as a random node selector + """ + + def __init__(self): + super(RandomScheduler, self).__init__() + + def pick_node(self, instance_id, **_kwargs): + nodes = self.compute_nodes_up() + return nodes[int(random.random() * len(nodes))] + +class BestFitScheduler(Scheduler): + """ + Implements Scheduler as a best-fit node selector + """ + + def __init__(self): + super(BestFitScheduler, self).__init__() + + def pick_node(self, instance_id, **_kwargs): + raise NotImplementedError("BestFitScheduler is not done yet") + diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 3a226322f..3a86cefbe 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -21,32 +21,34 @@ Scheduler Service """ import logging -import random -import sys -import time from twisted.internet import defer -from twisted.internet import task from nova import exception from nova import flags -from nova import process from nova import rpc from nova import service -from nova import utils from nova.compute import model -from nova.datastore import Redis +from nova.scheduler import scheduler FLAGS = flags.FLAGS -flags.DEFINE_integer('node_down_time', 60, - 'seconds without heartbeat that determines a compute node to be down') - +flags.DEFINE_string('scheduler_type', + 'random', + 'the scheduler to use') + +scheduler_classes = { + 'random': scheduler.RandomScheduler, + 'bestfit': scheduler.BestFitScheduler + } + class SchedulerService(service.Service): """ Manages the running instances. """ def __init__(self): super(SchedulerService, self).__init__() - self.instdir = model.InstanceDirectory() + if (FLAGS.scheduler_type not in scheduler_classes): + raise exception.Error("Scheduler '%s' does not exist" % FLAGS.scheduler_type) + self._scheduler_class = scheduler_classes[FLAGS.scheduler_type] def noop(self): """ simple test of an AMQP message call """ @@ -68,21 +70,8 @@ class SchedulerService(service.Service): logging.exception("model server went away") yield - @property - def compute_nodes(self): - return [identifier.split(':')[0] for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] - - def compute_node_is_up(self, node): - time_str = Redis.instance().hget('%s:%s:%s' % ('daemon', node, 'nova-compute'), 'updated_at') - return(time_str and - (time.time() - (int(time.mktime(time.strptime(time_str.replace('Z', 'UTC'), '%Y-%m-%dT%H:%M:%S%Z'))) - time.timezone) < FLAGS.node_down_time)) - - def compute_nodes_up(self): - return [node for node in self.compute_nodes if self.compute_node_is_up(node)] - def pick_node(self, instance_id, **_kwargs): - """You DEFINITELY want to define this in your subclass""" - raise NotImplementedError("Your subclass should define pick_node") + return self._scheduler_class().pick_node(instance_id, **_kwargs) @exception.wrap_exception def run_instance(self, instance_id, **_kwargs): @@ -94,16 +83,3 @@ class SchedulerService(service.Service): logging.debug("Casting to node %s for instance %s" % (node, instance_id)) -class RandomService(SchedulerService): - """ - Implements SchedulerService as a random node selector - """ - - def __init__(self): - super(RandomService, self).__init__() - - def pick_node(self, instance_id, **_kwargs): - nodes = self.compute_nodes_up() - return nodes[int(random.random() * len(nodes))] - - -- cgit From 094d64334e419d86a550c913ea4f0b8f086777bd Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 6 Aug 2010 18:10:41 -0500 Subject: fix copyrights for new files, etc --- bin/nova-listinstances | 21 ++++++++++++++++++++- nova/scheduler/__init__.py | 12 ++---------- nova/scheduler/scheduler.py | 4 +--- nova/scheduler/service.py | 4 +--- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/bin/nova-listinstances b/bin/nova-listinstances index 2f8ff28f9..386283d2f 100755 --- a/bin/nova-listinstances +++ b/bin/nova-listinstances @@ -1,4 +1,19 @@ -#!/usr/bin/python +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # # Duplicates the functionality of euca-describe-instances, but doesn't require @@ -6,6 +21,10 @@ # mostly a test program written for the scheduler # +""" +List instances by doing a direct query to the datastore +""" + from nova.compute import model data_needed = ['image_id', 'memory_kb', 'local_gb', 'node_name', 'vcpus'] diff --git a/nova/scheduler/__init__.py b/nova/scheduler/__init__.py index 516ea61bc..8359a7aeb 100644 --- a/nova/scheduler/__init__.py +++ b/nova/scheduler/__init__.py @@ -1,8 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. +# Copyright (c) 2010 Openstack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -22,12 +20,6 @@ .. automodule:: nova.scheduler :platform: Unix - :synopsis: Daemon that picks a host for a VM instance. -.. moduleauthor:: Jesse Andrews -.. moduleauthor:: Devin Carlen -.. moduleauthor:: Vishvananda Ishaya -.. moduleauthor:: Joshua McKenty -.. moduleauthor:: Manish Singh -.. moduleauthor:: Andy Smith + :synopsis: Module that picks a compute node to run a VM instance. .. moduleauthor:: Chris Behrens """ diff --git a/nova/scheduler/scheduler.py b/nova/scheduler/scheduler.py index 0da7b95cf..79ed9dc06 100644 --- a/nova/scheduler/scheduler.py +++ b/nova/scheduler/scheduler.py @@ -1,8 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. +# Copyright (c) 2010 Openstack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 3a86cefbe..39bfd6e07 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -1,8 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. +# Copyright (c) 2010 Openstack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain -- cgit From ba3b5ac30d9cd72e1cb757919ea76843112b307e Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 6 Aug 2010 18:54:45 -0500 Subject: pep8 and pylint cleanups --- nova/scheduler/scheduler.py | 29 +++++++++++++++++++---------- nova/scheduler/service.py | 18 +++++++++--------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/nova/scheduler/scheduler.py b/nova/scheduler/scheduler.py index 79ed9dc06..49ef40f06 100644 --- a/nova/scheduler/scheduler.py +++ b/nova/scheduler/scheduler.py @@ -18,19 +18,17 @@ Scheduler Classes """ -import logging import random -import sys import time -from nova import exception from nova import flags from nova.datastore import Redis FLAGS = flags.FLAGS flags.DEFINE_integer('node_down_time', 60, - 'seconds without heartbeat that determines a compute node to be down') + 'seconds without heartbeat that determines a ' + 'compute node to be down') class Scheduler(object): @@ -40,21 +38,32 @@ class Scheduler(object): @property def compute_nodes(self): - return [identifier.split(':')[0] for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] + return [identifier.split(':')[0] + for identifier in Redis.instance().smembers("daemons") + if (identifier.split(':')[1] == "nova-compute")] def compute_node_is_up(self, node): - time_str = Redis.instance().hget('%s:%s:%s' % ('daemon', node, 'nova-compute'), 'updated_at') + time_str = Redis.instance().hget('%s:%s:%s' % + ('daemon', node, 'nova-compute'), + 'updated_at') + if not time_str: + return False + # Would be a lot easier if we stored heartbeat time in epoch :) - return(time_str and - (time.time() - (int(time.mktime(time.strptime(time_str.replace('Z', 'UTC'), '%Y-%m-%dT%H:%M:%S%Z'))) - time.timezone) < FLAGS.node_down_time)) + time_str = time_str.replace('Z', 'UTC') + time_split = time.strptime(time_str, '%Y-%m-%dT%H:%M:%S%Z') + epoch_time = int(time.mktime(time_split)) - time.timezone + return (time.time() - epoch_time) < FLAGS.node_down_time def compute_nodes_up(self): - return [node for node in self.compute_nodes if self.compute_node_is_up(node)] + return [node for node in self.compute_nodes + if self.compute_node_is_up(node)] def pick_node(self, instance_id, **_kwargs): """You DEFINITELY want to define this in your subclass""" raise NotImplementedError("Your subclass should define pick_node") + class RandomScheduler(Scheduler): """ Implements Scheduler as a random node selector @@ -67,6 +76,7 @@ class RandomScheduler(Scheduler): nodes = self.compute_nodes_up() return nodes[int(random.random() * len(nodes))] + class BestFitScheduler(Scheduler): """ Implements Scheduler as a best-fit node selector @@ -77,4 +87,3 @@ class BestFitScheduler(Scheduler): def pick_node(self, instance_id, **_kwargs): raise NotImplementedError("BestFitScheduler is not done yet") - diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 39bfd6e07..1246b6e72 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -33,19 +33,20 @@ flags.DEFINE_string('scheduler_type', 'random', 'the scheduler to use') -scheduler_classes = { - 'random': scheduler.RandomScheduler, - 'bestfit': scheduler.BestFitScheduler - } - +scheduler_classes = {'random': scheduler.RandomScheduler, + 'bestfit': scheduler.BestFitScheduler} + + class SchedulerService(service.Service): """ Manages the running instances. """ + def __init__(self): super(SchedulerService, self).__init__() if (FLAGS.scheduler_type not in scheduler_classes): - raise exception.Error("Scheduler '%s' does not exist" % FLAGS.scheduler_type) + raise exception.Error("Scheduler '%s' does not exist" % + FLAGS.scheduler_type) self._scheduler_class = scheduler_classes[FLAGS.scheduler_type] def noop(self): @@ -77,7 +78,6 @@ class SchedulerService(service.Service): rpc.cast('%s.%s' % (FLAGS.compute_topic, node), {"method": "run_instance", - "args": {"instance_id" : instance_id}}) - logging.debug("Casting to node %s for instance %s" % + "args": {"instance_id": instance_id}}) + logging.debug("Casting to node %s for running instance %s" % (node, instance_id)) - -- cgit From 795b32fc66f243239d05a5434f939a76800c0052 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 9 Aug 2010 09:37:50 -0500 Subject: remove duplicated report_state that exists in the base class more pylint fixes --- nova/scheduler/service.py | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 1246b6e72..9d2d35f13 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -25,7 +25,6 @@ from nova import exception from nova import flags from nova import rpc from nova import service -from nova.compute import model from nova.scheduler import scheduler FLAGS = flags.FLAGS @@ -33,7 +32,7 @@ flags.DEFINE_string('scheduler_type', 'random', 'the scheduler to use') -scheduler_classes = {'random': scheduler.RandomScheduler, +SCHEDULER_CLASSES = {'random': scheduler.RandomScheduler, 'bestfit': scheduler.BestFitScheduler} @@ -44,40 +43,33 @@ class SchedulerService(service.Service): def __init__(self): super(SchedulerService, self).__init__() - if (FLAGS.scheduler_type not in scheduler_classes): + if (FLAGS.scheduler_type not in SCHEDULER_CLASSES): raise exception.Error("Scheduler '%s' does not exist" % FLAGS.scheduler_type) - self._scheduler_class = scheduler_classes[FLAGS.scheduler_type] + self._scheduler_class = SCHEDULER_CLASSES[FLAGS.scheduler_type] def noop(self): """ simple test of an AMQP message call """ return defer.succeed('PONG') - @defer.inlineCallbacks - def report_state(self, nodename, daemon): - # TODO(termie): make this pattern be more elegant. -todd - try: - record = model.Daemon(nodename, daemon) - record.heartbeat() - if getattr(self, "model_disconnected", False): - self.model_disconnected = False - logging.error("Recovered model server connection!") - - except model.ConnectionError, ex: - if not getattr(self, "model_disconnected", False): - self.model_disconnected = True - logging.exception("model server went away") - yield - def pick_node(self, instance_id, **_kwargs): + """ + Return a node to use based on the selected Scheduler + """ + return self._scheduler_class().pick_node(instance_id, **_kwargs) @exception.wrap_exception def run_instance(self, instance_id, **_kwargs): + """ + Picks a node for a running VM and casts the run_instance request + """ + node = self.pick_node(instance_id, **_kwargs) rpc.cast('%s.%s' % (FLAGS.compute_topic, node), {"method": "run_instance", "args": {"instance_id": instance_id}}) - logging.debug("Casting to node %s for running instance %s" % - (node, instance_id)) + logging.debug("Casting to node %s for running instance %s", + node, instance_id) + -- cgit From 3e01acd4e70f9e850487c5ac4067ab2c2c1a18eb Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 9 Aug 2010 12:56:32 -0500 Subject: separated scheduler types into own modules --- nova/scheduler/base.py | 65 +++++++++++++++++++++++++++++++++ nova/scheduler/bestfit.py | 30 +++++++++++++++ nova/scheduler/chance.py | 33 +++++++++++++++++ nova/scheduler/scheduler.py | 89 --------------------------------------------- nova/scheduler/service.py | 10 ++--- 5 files changed, 133 insertions(+), 94 deletions(-) create mode 100644 nova/scheduler/base.py create mode 100644 nova/scheduler/bestfit.py create mode 100644 nova/scheduler/chance.py delete mode 100644 nova/scheduler/scheduler.py diff --git a/nova/scheduler/base.py b/nova/scheduler/base.py new file mode 100644 index 000000000..5c359943e --- /dev/null +++ b/nova/scheduler/base.py @@ -0,0 +1,65 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Scheduler base class that all Schedulers should inherit from +""" + +import time + +from nova import flags +from nova.datastore import Redis + +FLAGS = flags.FLAGS +flags.DEFINE_integer('node_down_time', + 60, + 'seconds without heartbeat that determines a ' + 'compute node to be down') + + +class Scheduler(object): + """ + The base class that all Scheduler clases should inherit from + """ + + @property + def compute_nodes(self): + return [identifier.split(':')[0] + for identifier in Redis.instance().smembers("daemons") + if (identifier.split(':')[1] == "nova-compute")] + + def compute_node_is_up(self, node): + time_str = Redis.instance().hget('%s:%s:%s' % + ('daemon', node, 'nova-compute'), + 'updated_at') + if not time_str: + return False + + # Would be a lot easier if we stored heartbeat time in epoch :) + + # The 'str()' here is to get rid of a pylint error + time_str = str(time_str).replace('Z', 'UTC') + time_split = time.strptime(time_str, '%Y-%m-%dT%H:%M:%S%Z') + epoch_time = int(time.mktime(time_split)) - time.timezone + return (time.time() - epoch_time) < FLAGS.node_down_time + + def compute_nodes_up(self): + return [node for node in self.compute_nodes + if self.compute_node_is_up(node)] + + def pick_node(self, instance_id, **_kwargs): + """You DEFINITELY want to define this in your subclass""" + raise NotImplementedError("Your subclass should define pick_node") diff --git a/nova/scheduler/bestfit.py b/nova/scheduler/bestfit.py new file mode 100644 index 000000000..1bd24456a --- /dev/null +++ b/nova/scheduler/bestfit.py @@ -0,0 +1,30 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Best Fit Scheduler +""" + +from nova.scheduler.base import Scheduler + + +class BestFitScheduler(Scheduler): + """ + Implements Scheduler as a best-fit node selector + """ + + def pick_node(self, instance_id, **_kwargs): + raise NotImplementedError("BestFitScheduler is not done yet") diff --git a/nova/scheduler/chance.py b/nova/scheduler/chance.py new file mode 100644 index 000000000..c57c346f5 --- /dev/null +++ b/nova/scheduler/chance.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Chance (Random) Scheduler implementation +""" + +import random + +from nova.scheduler.base import Scheduler + + +class ChanceScheduler(Scheduler): + """ + Implements Scheduler as a random node selector + """ + + def pick_node(self, instance_id, **_kwargs): + nodes = self.compute_nodes_up() + return nodes[int(random.random() * len(nodes))] diff --git a/nova/scheduler/scheduler.py b/nova/scheduler/scheduler.py deleted file mode 100644 index 49ef40f06..000000000 --- a/nova/scheduler/scheduler.py +++ /dev/null @@ -1,89 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Openstack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Scheduler Classes -""" - -import random -import time - -from nova import flags -from nova.datastore import Redis - -FLAGS = flags.FLAGS -flags.DEFINE_integer('node_down_time', - 60, - 'seconds without heartbeat that determines a ' - 'compute node to be down') - - -class Scheduler(object): - """ - The base class that all Scheduler clases should inherit from - """ - - @property - def compute_nodes(self): - return [identifier.split(':')[0] - for identifier in Redis.instance().smembers("daemons") - if (identifier.split(':')[1] == "nova-compute")] - - def compute_node_is_up(self, node): - time_str = Redis.instance().hget('%s:%s:%s' % - ('daemon', node, 'nova-compute'), - 'updated_at') - if not time_str: - return False - - # Would be a lot easier if we stored heartbeat time in epoch :) - time_str = time_str.replace('Z', 'UTC') - time_split = time.strptime(time_str, '%Y-%m-%dT%H:%M:%S%Z') - epoch_time = int(time.mktime(time_split)) - time.timezone - return (time.time() - epoch_time) < FLAGS.node_down_time - - def compute_nodes_up(self): - return [node for node in self.compute_nodes - if self.compute_node_is_up(node)] - - def pick_node(self, instance_id, **_kwargs): - """You DEFINITELY want to define this in your subclass""" - raise NotImplementedError("Your subclass should define pick_node") - - -class RandomScheduler(Scheduler): - """ - Implements Scheduler as a random node selector - """ - - def __init__(self): - super(RandomScheduler, self).__init__() - - def pick_node(self, instance_id, **_kwargs): - nodes = self.compute_nodes_up() - return nodes[int(random.random() * len(nodes))] - - -class BestFitScheduler(Scheduler): - """ - Implements Scheduler as a best-fit node selector - """ - - def __init__(self): - super(BestFitScheduler, self).__init__() - - def pick_node(self, instance_id, **_kwargs): - raise NotImplementedError("BestFitScheduler is not done yet") diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 9d2d35f13..44b30ecb5 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -25,15 +25,16 @@ from nova import exception from nova import flags from nova import rpc from nova import service -from nova.scheduler import scheduler +from nova.scheduler import chance +from nova.scheduler import bestfit FLAGS = flags.FLAGS flags.DEFINE_string('scheduler_type', - 'random', + 'chance', 'the scheduler to use') -SCHEDULER_CLASSES = {'random': scheduler.RandomScheduler, - 'bestfit': scheduler.BestFitScheduler} +SCHEDULER_CLASSES = {'chance': chance.ChanceScheduler, + 'bestfit': bestfit.BestFitScheduler} class SchedulerService(service.Service): @@ -72,4 +73,3 @@ class SchedulerService(service.Service): "args": {"instance_id": instance_id}}) logging.debug("Casting to node %s for running instance %s", node, instance_id) - -- cgit From d1982a50561f7b35ffc76ce5d45aaec11e76a23c Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Tue, 10 Aug 2010 18:48:33 -0500 Subject: more pylint fixes --- nova/scheduler/base.py | 23 +++++++++++++++++++---- nova/scheduler/bestfit.py | 4 ++++ nova/scheduler/chance.py | 4 ++++ nova/scheduler/service.py | 3 ++- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/base.py b/nova/scheduler/base.py index 5c359943e..2872ae6fe 100644 --- a/nova/scheduler/base.py +++ b/nova/scheduler/base.py @@ -35,13 +35,23 @@ class Scheduler(object): The base class that all Scheduler clases should inherit from """ - @property - def compute_nodes(self): + @staticmethod + def compute_nodes(): + """ + Return a list of compute nodes + """ + return [identifier.split(':')[0] for identifier in Redis.instance().smembers("daemons") if (identifier.split(':')[1] == "nova-compute")] - def compute_node_is_up(self, node): + @staticmethod + def compute_node_is_up(node): + """ + Given a node name, return whether the node is considered 'up' by + if it's sent a heartbeat recently + """ + time_str = Redis.instance().hget('%s:%s:%s' % ('daemon', node, 'nova-compute'), 'updated_at') @@ -57,9 +67,14 @@ class Scheduler(object): return (time.time() - epoch_time) < FLAGS.node_down_time def compute_nodes_up(self): - return [node for node in self.compute_nodes + """ + Return the list of compute nodes that are considered 'up' + """ + + return [node for node in self.compute_nodes() if self.compute_node_is_up(node)] def pick_node(self, instance_id, **_kwargs): """You DEFINITELY want to define this in your subclass""" + raise NotImplementedError("Your subclass should define pick_node") diff --git a/nova/scheduler/bestfit.py b/nova/scheduler/bestfit.py index 1bd24456a..bdd4fcbdc 100644 --- a/nova/scheduler/bestfit.py +++ b/nova/scheduler/bestfit.py @@ -27,4 +27,8 @@ class BestFitScheduler(Scheduler): """ def pick_node(self, instance_id, **_kwargs): + """ + Picks a node that is up and is a best fit for the new instance + """ + raise NotImplementedError("BestFitScheduler is not done yet") diff --git a/nova/scheduler/chance.py b/nova/scheduler/chance.py index c57c346f5..719c37674 100644 --- a/nova/scheduler/chance.py +++ b/nova/scheduler/chance.py @@ -29,5 +29,9 @@ class ChanceScheduler(Scheduler): """ def pick_node(self, instance_id, **_kwargs): + """ + Picks a node that is up at random + """ + nodes = self.compute_nodes_up() return nodes[int(random.random() * len(nodes))] diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py index 44b30ecb5..136f262c2 100644 --- a/nova/scheduler/service.py +++ b/nova/scheduler/service.py @@ -49,7 +49,8 @@ class SchedulerService(service.Service): FLAGS.scheduler_type) self._scheduler_class = SCHEDULER_CLASSES[FLAGS.scheduler_type] - def noop(self): + @staticmethod + def noop(): """ simple test of an AMQP message call """ return defer.succeed('PONG') -- cgit From 6ae66c595d4f85802045734ed1b230a292f9c953 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 20 Aug 2010 13:26:24 +0100 Subject: Better error message on subprocess spawn fail, and it's a ProcessExecutionException irrespective of how the process is run. --- nova/process.py | 65 +++++++++++++++++++++++++++------------------------------ nova/utils.py | 17 +++++++++++++-- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/nova/process.py b/nova/process.py index 425d9f162..259e3f92e 100644 --- a/nova/process.py +++ b/nova/process.py @@ -29,28 +29,12 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags +from nova.utils import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') - -# NOTE(termie): this is copied from twisted.internet.utils but since -# they don't export it I've copied and modified -class UnexpectedErrorOutput(IOError): - """ - Standard error data was received where it was not expected. This is a - subclass of L{IOError} to preserve backward compatibility with the previous - error behavior of L{getProcessOutput}. - - @ivar processEnded: A L{Deferred} which will fire when the process which - produced the data on stderr has ended (exited and all file descriptors - closed). - """ - def __init__(self, stdout=None, stderr=None): - IOError.__init__(self, "got stdout: %r\nstderr: %r" % (stdout, stderr)) - - # This is based on _BackRelay from twister.internal.utils, but modified to # capture both stdout and stderr, without odd stderr handling, and also to # handle stdin @@ -62,22 +46,23 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_UnexpectedErrorOutput} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are received over stderr, this attribute will refer to a L{Deferred} which will be called back when the process ends. This C{Deferred} is also - associated with the L{_UnexpectedErrorOutput} which C{deferred} fires + associated with the L{_ProcessExecutionError} which C{deferred} fires with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, started_deferred=None, + def __init__(self, deferred, cmd, started_deferred=None, terminate_on_stderr=False, check_exit_code=True, process_input=None): self.deferred = deferred + self.cmd = cmd self.stdout = StringIO.StringIO() self.stderr = StringIO.StringIO() self.started_deferred = started_deferred @@ -85,14 +70,18 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + + def _build_execution_error(self, exit_code=None): + return ProcessExecutionError( cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) + def errReceived(self, text): self.stderr.write(text) if self.terminate_on_stderr and (self.deferred is not None): self.on_process_ended = defer.Deferred() - self.deferred.errback(UnexpectedErrorOutput( - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue())) + self.deferred.errback(self._build_execution_error()) self.deferred = None self.transport.loseConnection() @@ -102,15 +91,19 @@ class BackRelayWithInput(protocol.ProcessProtocol): def processEnded(self, reason): if self.deferred is not None: stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be - # called also. However, this is what the unit tests test for... - self.deferred.errback(UnexpectedErrorOutput(stdout, stderr)) + exit_code = reason.value.exitCode + if self.check_exit_code and exit_code <> 0: + self.deferred.errback(self._build_execution_error(exit_code)) + else: + try: + if self.check_exit_code: + reason.trap(error.ProcessDone) + self.deferred.callback((stdout, stderr)) + except: + # NOTE(justinsb): This logic is a little suspicious to me... + # If the callback throws an exception, then errback will be + # called also. However, this is what the unit tests test for... + self.deferred.errback(_build_execution_error(exit-code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) @@ -131,8 +124,12 @@ def get_process_output(executable, args=None, env=None, path=None, args = args and args or () env = env and env and {} deferred = defer.Deferred() + cmd = executable + if args: + cmd = cmd + " " + ' '.join(args) process_handler = BackRelayWithInput( - deferred, + deferred, + cmd, started_deferred=started_deferred, check_exit_code=check_exit_code, process_input=process_input, diff --git a/nova/utils.py b/nova/utils.py index dc3c626ec..b8abb5388 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -36,6 +36,16 @@ from nova import flags FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +class ProcessExecutionError(IOError): + def __init__( self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( + description, cmd, exit_code, stdout, stderr) + IOError.__init__(self, message) def import_class(import_str): """Returns a class from a string including module and class""" @@ -73,8 +83,11 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): if obj.returncode: logging.debug("Result was %s" % (obj.returncode)) if check_exit_code and obj.returncode <> 0: - raise Exception( "Unexpected exit code: %s. result=%s" - % (obj.returncode, result)) + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=cmd) return result -- cgit From 41864e2653286fd46c7b69ee992d4be492b014c6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 20 Aug 2010 14:50:43 +0100 Subject: Fixed typo --- nova/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/process.py b/nova/process.py index 259e3f92e..81262a506 100644 --- a/nova/process.py +++ b/nova/process.py @@ -103,7 +103,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): # NOTE(justinsb): This logic is a little suspicious to me... # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... - self.deferred.errback(_build_execution_error(exit-code)) + self.deferred.errback(_build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) -- cgit From c4bf107b7e4fd64376dab7ebe39e4531f64879c5 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 21 Aug 2010 11:54:03 +0100 Subject: Added missing "self." --- nova/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/process.py b/nova/process.py index 81262a506..069310802 100644 --- a/nova/process.py +++ b/nova/process.py @@ -103,7 +103,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): # NOTE(justinsb): This logic is a little suspicious to me... # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... - self.deferred.errback(_build_execution_error(exit_code)) + self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) -- cgit From 7f666230e37745b174998a485fe1d7626c4862ae Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 25 Aug 2010 16:45:59 +0000 Subject: A few small changes to install_venv to let venv builds work on the tarmac box. --- tools/install_venv.py | 4 ++++ tools/pip-requires | 1 + 2 files changed, 5 insertions(+) diff --git a/tools/install_venv.py b/tools/install_venv.py index 1f0fa3cc7..e764efff6 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -88,6 +88,10 @@ def create_virtualenv(venv=VENV): def install_dependencies(venv=VENV): print 'Installing dependencies with pip (this can take a while)...' + # Install greenlet by hand - just listing it in the requires file does not + # get it in stalled in the right order + run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, 'greenlet'], + redirect_output=False) run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, '-r', PIP_REQUIRES], redirect_output=False) run_command(['tools/with_venv.sh', 'pip', 'install', '-E', venv, TWISTED_NOVA], diff --git a/tools/pip-requires b/tools/pip-requires index 13e8e5f45..9853252dc 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -18,3 +18,4 @@ wsgiref==0.1.2 zope.interface==3.6.1 mox==0.5.0 -f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz +greenlet==0.3.1 -- cgit From a6bd6f8581b9c03da9aceed7d87f4664410d0998 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 26 Aug 2010 14:09:14 -0400 Subject: work endpoint/images.py into an S3ImageService. The translation isn't perfect, but it's a start. --- nova/endpoint/images.py | 30 +++++------------------------- nova/image/service.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index 2a88d66af..cfea4c20b 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -26,6 +26,7 @@ import urllib import boto.s3.connection +from nova import image from nova import flags from nova import utils from nova.auth import manager @@ -35,7 +36,7 @@ FLAGS = flags.FLAGS def modify(context, image_id, operation): - conn(context).make_request( + image.S3ImageService(context)._conn().make_request( method='POST', bucket='_images', query_args=qs({'image_id': image_id, 'operation': operation})) @@ -47,7 +48,7 @@ def register(context, image_location): """ rpc call to register a new image based from a manifest """ image_id = utils.generate_uid('ami') - conn(context).make_request( + image.S3ImageService(context)._conn().make_request( method='PUT', bucket='_images', query_args=qs({'image_location': image_location, @@ -61,12 +62,7 @@ def list(context, filter_list=[]): optionally filtered by a list of image_id """ - # FIXME: send along the list of only_images to check for - response = conn(context).make_request( - method='GET', - bucket='_images') - - result = json.loads(response.read()) + result = image.S3ImageService(context).index().values() if not filter_list is None: return [i for i in result if i['imageId'] in filter_list] return result @@ -74,23 +70,7 @@ def list(context, filter_list=[]): def deregister(context, image_id): """ unregister an image """ - conn(context).make_request( - method='DELETE', - bucket='_images', - query_args=qs({'image_id': image_id})) - - -def conn(context): - access = manager.AuthManager().get_access_key(context.user, - context.project) - secret = str(context.user.secret) - calling = boto.s3.connection.OrdinaryCallingFormat() - return boto.s3.connection.S3Connection(aws_access_key_id=access, - aws_secret_access_key=secret, - is_secure=False, - calling_format=calling, - port=FLAGS.s3_port, - host=FLAGS.s3_host) + image.S3ImageService(context).delete(image_id) def qs(params): diff --git a/nova/image/service.py b/nova/image/service.py index 1a7a258b7..25e4bb675 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -38,6 +38,8 @@ class ImageService(object): def show(self, id): """ Returns a dict containing image data for the given opaque image id. + + Returns None if the id does not exist. """ @@ -88,3 +90,46 @@ class LocalImageService(ImageService): Delete the given image. Raises OSError if the image does not exist. """ os.unlink(self._path_to(image_id)) + + +# TODO(gundlach): before this can be loaded dynamically in ImageService.load(), +# we'll have to make __init__() not require a context. Right now it +# is only used by the AWS API, which hard-codes it, so that's OK. +class S3ImageService(ImageService): + """Service that stores images in an S3 provider.""" + + def __init__(self, context): + self._context = context + + def index(self): + response = self._conn().make_request( + method='GET', + bucket='_images') + items = json.loads(response.read()) + return dict((item['imageId'], item) for item in items) + + def show(self, id): + response = self._conn().make_request( + method='GET', + bucket='_images', + query_args=qs({'image_id': image_id})) + return json.loads(response.read()) + + def delete(self, image_id): + self._conn().make_request( + method='DELETE', + bucket='_images', + query_args=qs({'image_id': image_id})) + + def _conn(self): + """Return a boto S3Connection to the S3 store.""" + access = manager.AuthManager().get_access_key(self._context.user, + self._context.project) + secret = str(self._context.user.secret) + calling = boto.s3.connection.OrdinaryCallingFormat() + return boto.s3.connection.S3Connection(aws_access_key_id=access, + aws_secret_access_key=secret, + is_secure=False, + calling_format=calling, + port=FLAGS.s3_port, + host=FLAGS.s3_host) -- cgit From f0223b5135059ac6535739916a297654953751fc Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 30 Aug 2010 18:38:26 -0400 Subject: Notes for converting Tornado to Eventlet --- nova/endpoint/notes.txt | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 nova/endpoint/notes.txt diff --git a/nova/endpoint/notes.txt b/nova/endpoint/notes.txt new file mode 100644 index 000000000..c1d441de0 --- /dev/null +++ b/nova/endpoint/notes.txt @@ -0,0 +1,62 @@ +bin/nova-api: + somehow listens for 'cloud_topic' rpc messages and ties them to + the cloud controller (maybe so internal calls can hit the API + via Queuing instead of via HTTP?) + hands CloudController and AdminController to APIServerApplication + and hands that to Tornado. + + +api.py: + +APIServerApplication(tornado.web.Application) + maps routes to APIRequestHandler, CloudPipRequestHandler, MetadataRequestHandler, + RootRequestHandler(just lists versions) + (and to controllers which are passed to __init__) + magical twisted mapping to it + +APIRequestHandler + execute: + authenticates request + picks controller from APIServerApplication's list + picks action from incoming request arguments + dict = APIRequest(controller, action).send(Context(user, project)) + _write_callback(dict) + self.finish() + +APIRequest + send(context, **kwargs): + dict = controller.action(context, **kwargs) + return _render_response(dict) # turns into XML + + +CloudController and AdminController: + actions return dict (or True which is converted into dict(return=True)) + actions have @rbac.allow('list', 'of', 'roles', 'or', '"all"') + actions can have @defer.inlineCallbacks which is used for yield statements + can use rpc.cast and then defer a returnValue + + +==== STRATEGY TO CONVERT TO EVENTLET+WSGI ==== + +* Controllers: + move the @rbac.allow data into an auth WSGI that is right above the call + to the controller + verify @defer.inlineCallbacks is just to allow the yield statements, then + remove the yield statements (untangle from twisted) + +* nova-api: + verify that cloud_topic is going away which I seem to remember, so we can ignore rpc + +* apiserverapplication: + replace with a Router to a wsgi.Controller + root controller all goes into a "version" action + ??? dunno what cloudpipes or metadatarequesthandlers do... + apirequesthandler stuff goes into an "ec2" action + +* apirequesthandler + ec2() method on wsgi.Controller + - basically it's execute() from old APIRequestHandler + change to return data directly instead of _write_callback() and finish() + +* apirequest + doesn't need to change -- cgit From 909c24b9cd35d6752f9f051f4e9a80ce30eaee4d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 30 Aug 2010 19:04:51 -0400 Subject: Move class into its own file --- nova/api/ec2/apirequesthandler.py | 126 ++++++++++++++++++++++++++++++++++++++ nova/endpoint/api.py | 101 ------------------------------ 2 files changed, 126 insertions(+), 101 deletions(-) create mode 100644 nova/api/ec2/apirequesthandler.py diff --git a/nova/api/ec2/apirequesthandler.py b/nova/api/ec2/apirequesthandler.py new file mode 100644 index 000000000..bbba60c02 --- /dev/null +++ b/nova/api/ec2/apirequesthandler.py @@ -0,0 +1,126 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +APIRequestHandler, pulled unmodified out of nova.endpoint.api +""" + +import logging + +import tornado.web + +from nova import exception +from nova import utils +from nova.auth import manager + + +_log = logging.getLogger("api") +_log.setLevel(logging.DEBUG) + + +class APIRequestHandler(tornado.web.RequestHandler): + def get(self, controller_name): + self.execute(controller_name) + + @tornado.web.asynchronous + def execute(self, controller_name): + # Obtain the appropriate controller for this request. + try: + controller = self.application.controllers[controller_name] + except KeyError: + self._error('unhandled', 'no controller named %s' % controller_name) + return + + args = self.request.arguments + + # Read request signature. + try: + signature = args.pop('Signature')[0] + except: + raise tornado.web.HTTPError(400) + + # Make a copy of args for authentication and signature verification. + auth_params = {} + for key, value in args.items(): + auth_params[key] = value[0] + + # Get requested action and remove authentication args for final request. + try: + action = args.pop('Action')[0] + access = args.pop('AWSAccessKeyId')[0] + args.pop('SignatureMethod') + args.pop('SignatureVersion') + args.pop('Version') + args.pop('Timestamp') + except: + raise tornado.web.HTTPError(400) + + # Authenticate the request. + try: + (user, project) = manager.AuthManager().authenticate( + access, + signature, + auth_params, + self.request.method, + self.request.host, + self.request.path + ) + + except exception.Error, ex: + logging.debug("Authentication Failure: %s" % ex) + raise tornado.web.HTTPError(403) + + _log.debug('action: %s' % action) + + for key, value in args.items(): + _log.debug('arg: %s\t\tval: %s' % (key, value)) + + request = APIRequest(controller, action) + context = APIRequestContext(self, user, project) + d = request.send(context, **args) + # d.addCallback(utils.debug) + + # TODO: Wrap response in AWS XML format + d.addCallbacks(self._write_callback, self._error_callback) + + def _write_callback(self, data): + self.set_header('Content-Type', 'text/xml') + self.write(data) + self.finish() + + def _error_callback(self, failure): + try: + failure.raiseException() + except exception.ApiError as ex: + self._error(type(ex).__name__ + "." + ex.code, ex.message) + # TODO(vish): do something more useful with unknown exceptions + except Exception as ex: + self._error(type(ex).__name__, str(ex)) + raise + + def post(self, controller_name): + self.execute(controller_name) + + def _error(self, code, message): + self._status_code = 400 + self.set_header('Content-Type', 'text/xml') + self.write('\n') + self.write('%s' + '%s' + '?' % (code, message)) + self.finish() diff --git a/nova/endpoint/api.py b/nova/endpoint/api.py index 40be00bb7..25ed613b9 100755 --- a/nova/endpoint/api.py +++ b/nova/endpoint/api.py @@ -21,7 +21,6 @@ Tornado REST API Request Handlers for Nova functions Most calls are proxied into the responsible controller. """ -import logging import multiprocessing import random import re @@ -33,10 +32,7 @@ import tornado.web from twisted.internet import defer from nova import crypto -from nova import exception from nova import flags -from nova import utils -from nova.auth import manager import nova.cloudpipe.api from nova.endpoint import cloud @@ -45,10 +41,6 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -_log = logging.getLogger("api") -_log.setLevel(logging.DEBUG) - - _c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') @@ -230,99 +222,6 @@ class MetadataRequestHandler(tornado.web.RequestHandler): self.finish() -class APIRequestHandler(tornado.web.RequestHandler): - def get(self, controller_name): - self.execute(controller_name) - - @tornado.web.asynchronous - def execute(self, controller_name): - # Obtain the appropriate controller for this request. - try: - controller = self.application.controllers[controller_name] - except KeyError: - self._error('unhandled', 'no controller named %s' % controller_name) - return - - args = self.request.arguments - - # Read request signature. - try: - signature = args.pop('Signature')[0] - except: - raise tornado.web.HTTPError(400) - - # Make a copy of args for authentication and signature verification. - auth_params = {} - for key, value in args.items(): - auth_params[key] = value[0] - - # Get requested action and remove authentication args for final request. - try: - action = args.pop('Action')[0] - access = args.pop('AWSAccessKeyId')[0] - args.pop('SignatureMethod') - args.pop('SignatureVersion') - args.pop('Version') - args.pop('Timestamp') - except: - raise tornado.web.HTTPError(400) - - # Authenticate the request. - try: - (user, project) = manager.AuthManager().authenticate( - access, - signature, - auth_params, - self.request.method, - self.request.host, - self.request.path - ) - - except exception.Error, ex: - logging.debug("Authentication Failure: %s" % ex) - raise tornado.web.HTTPError(403) - - _log.debug('action: %s' % action) - - for key, value in args.items(): - _log.debug('arg: %s\t\tval: %s' % (key, value)) - - request = APIRequest(controller, action) - context = APIRequestContext(self, user, project) - d = request.send(context, **args) - # d.addCallback(utils.debug) - - # TODO: Wrap response in AWS XML format - d.addCallbacks(self._write_callback, self._error_callback) - - def _write_callback(self, data): - self.set_header('Content-Type', 'text/xml') - self.write(data) - self.finish() - - def _error_callback(self, failure): - try: - failure.raiseException() - except exception.ApiError as ex: - self._error(type(ex).__name__ + "." + ex.code, ex.message) - # TODO(vish): do something more useful with unknown exceptions - except Exception as ex: - self._error(type(ex).__name__, str(ex)) - raise - - def post(self, controller_name): - self.execute(controller_name) - - def _error(self, code, message): - self._status_code = 400 - self.set_header('Content-Type', 'text/xml') - self.write('\n') - self.write('%s' - '%s' - '?' % (code, message)) - self.finish() - - class APIServerApplication(tornado.web.Application): def __init__(self, controllers): tornado.web.Application.__init__(self, [ -- cgit From be2b529a987627bf454f7343df74d4e8ae670761 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 30 Aug 2010 19:08:22 -0400 Subject: Move APIRequest into its own file --- nova/api/ec2/apirequest.py | 132 +++++++++++++++++++++++++++++++++++++++++++++ nova/endpoint/api.py | 109 ------------------------------------- 2 files changed, 132 insertions(+), 109 deletions(-) create mode 100644 nova/api/ec2/apirequest.py diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py new file mode 100644 index 000000000..1fc84248b --- /dev/null +++ b/nova/api/ec2/apirequest.py @@ -0,0 +1,132 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +APIRequest class +""" + +# TODO(termie): replace minidom with etree +from xml.dom import minidom + +from twisted.internet import defer + + +_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') + + +def _camelcase_to_underscore(str): + return _c2u.sub(r'_\1', str).lower().strip('_') + + +def _underscore_to_camelcase(str): + return ''.join([x[:1].upper() + x[1:] for x in str.split('_')]) + + +def _underscore_to_xmlcase(str): + res = _underscore_to_camelcase(str) + return res[:1].lower() + res[1:] + + +class APIRequest(object): + def __init__(self, controller, action): + self.controller = controller + self.action = action + + def send(self, context, **kwargs): + + try: + method = getattr(self.controller, + _camelcase_to_underscore(self.action)) + except AttributeError: + _error = ('Unsupported API request: controller = %s,' + 'action = %s') % (self.controller, self.action) + _log.warning(_error) + # TODO: Raise custom exception, trap in apiserver, + # and reraise as 400 error. + raise Exception(_error) + + args = {} + for key, value in kwargs.items(): + parts = key.split(".") + key = _camelcase_to_underscore(parts[0]) + if len(parts) > 1: + d = args.get(key, {}) + d[parts[1]] = value[0] + value = d + else: + value = value[0] + args[key] = value + + for key in args.keys(): + if isinstance(args[key], dict): + if args[key] != {} and args[key].keys()[0].isdigit(): + s = args[key].items() + s.sort() + args[key] = [v for k, v in s] + + d = defer.maybeDeferred(method, context, **args) + d.addCallback(self._render_response, context.request_id) + return d + + def _render_response(self, response_data, request_id): + xml = minidom.Document() + + response_el = xml.createElement(self.action + 'Response') + response_el.setAttribute('xmlns', + 'http://ec2.amazonaws.com/doc/2009-11-30/') + request_id_el = xml.createElement('requestId') + request_id_el.appendChild(xml.createTextNode(request_id)) + response_el.appendChild(request_id_el) + if(response_data == True): + self._render_dict(xml, response_el, {'return': 'true'}) + else: + self._render_dict(xml, response_el, response_data) + + xml.appendChild(response_el) + + response = xml.toxml() + xml.unlink() + _log.debug(response) + return response + + def _render_dict(self, xml, el, data): + try: + for key in data.keys(): + val = data[key] + el.appendChild(self._render_data(xml, key, val)) + except: + _log.debug(data) + raise + + def _render_data(self, xml, el_name, data): + el_name = _underscore_to_xmlcase(el_name) + data_el = xml.createElement(el_name) + + if isinstance(data, list): + for item in data: + data_el.appendChild(self._render_data(xml, 'item', item)) + elif isinstance(data, dict): + self._render_dict(xml, data_el, data) + elif hasattr(data, '__dict__'): + self._render_dict(xml, data_el, data.__dict__) + elif isinstance(data, bool): + data_el.appendChild(xml.createTextNode(str(data).lower())) + elif data != None: + data_el.appendChild(xml.createTextNode(str(data))) + + return data_el diff --git a/nova/endpoint/api.py b/nova/endpoint/api.py index 25ed613b9..5a4e496a0 100755 --- a/nova/endpoint/api.py +++ b/nova/endpoint/api.py @@ -25,11 +25,8 @@ import multiprocessing import random import re import urllib -# TODO(termie): replace minidom with etree -from xml.dom import minidom import tornado.web -from twisted.internet import defer from nova import crypto from nova import flags @@ -41,22 +38,6 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') - - -def _camelcase_to_underscore(str): - return _c2u.sub(r'_\1', str).lower().strip('_') - - -def _underscore_to_camelcase(str): - return ''.join([x[:1].upper() + x[1:] for x in str.split('_')]) - - -def _underscore_to_xmlcase(str): - res = _underscore_to_camelcase(str) - return res[:1].lower() + res[1:] - - class APIRequestContext(object): def __init__(self, handler, user, project): self.handler = handler @@ -68,96 +49,6 @@ class APIRequestContext(object): ) -class APIRequest(object): - def __init__(self, controller, action): - self.controller = controller - self.action = action - - def send(self, context, **kwargs): - - try: - method = getattr(self.controller, - _camelcase_to_underscore(self.action)) - except AttributeError: - _error = ('Unsupported API request: controller = %s,' - 'action = %s') % (self.controller, self.action) - _log.warning(_error) - # TODO: Raise custom exception, trap in apiserver, - # and reraise as 400 error. - raise Exception(_error) - - args = {} - for key, value in kwargs.items(): - parts = key.split(".") - key = _camelcase_to_underscore(parts[0]) - if len(parts) > 1: - d = args.get(key, {}) - d[parts[1]] = value[0] - value = d - else: - value = value[0] - args[key] = value - - for key in args.keys(): - if isinstance(args[key], dict): - if args[key] != {} and args[key].keys()[0].isdigit(): - s = args[key].items() - s.sort() - args[key] = [v for k, v in s] - - d = defer.maybeDeferred(method, context, **args) - d.addCallback(self._render_response, context.request_id) - return d - - def _render_response(self, response_data, request_id): - xml = minidom.Document() - - response_el = xml.createElement(self.action + 'Response') - response_el.setAttribute('xmlns', - 'http://ec2.amazonaws.com/doc/2009-11-30/') - request_id_el = xml.createElement('requestId') - request_id_el.appendChild(xml.createTextNode(request_id)) - response_el.appendChild(request_id_el) - if(response_data == True): - self._render_dict(xml, response_el, {'return': 'true'}) - else: - self._render_dict(xml, response_el, response_data) - - xml.appendChild(response_el) - - response = xml.toxml() - xml.unlink() - _log.debug(response) - return response - - def _render_dict(self, xml, el, data): - try: - for key in data.keys(): - val = data[key] - el.appendChild(self._render_data(xml, key, val)) - except: - _log.debug(data) - raise - - def _render_data(self, xml, el_name, data): - el_name = _underscore_to_xmlcase(el_name) - data_el = xml.createElement(el_name) - - if isinstance(data, list): - for item in data: - data_el.appendChild(self._render_data(xml, 'item', item)) - elif isinstance(data, dict): - self._render_dict(xml, data_el, data) - elif hasattr(data, '__dict__'): - self._render_dict(xml, data_el, data.__dict__) - elif isinstance(data, bool): - data_el.appendChild(xml.createTextNode(str(data).lower())) - elif data != None: - data_el.appendChild(xml.createTextNode(str(data))) - - return data_el - - class RootRequestHandler(tornado.web.RequestHandler): def get(self): # available api versions -- cgit From 4bca41506c90e779a8d4a5defdca3add79073185 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 30 Aug 2010 19:10:17 -0400 Subject: Move APIRequestContext into its own file --- nova/api/ec2/apirequestcontext.py | 33 +++++++++++++++++++++++++++++++++ nova/endpoint/api.py | 12 ------------ 2 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 nova/api/ec2/apirequestcontext.py diff --git a/nova/api/ec2/apirequestcontext.py b/nova/api/ec2/apirequestcontext.py new file mode 100644 index 000000000..fb3118020 --- /dev/null +++ b/nova/api/ec2/apirequestcontext.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +APIRequestContext +""" + +import random + +class APIRequestContext(object): + def __init__(self, handler, user, project): + self.handler = handler + self.user = user + self.project = project + self.request_id = ''.join( + [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') + for x in xrange(20)] + ) diff --git a/nova/endpoint/api.py b/nova/endpoint/api.py index 5a4e496a0..311fb1880 100755 --- a/nova/endpoint/api.py +++ b/nova/endpoint/api.py @@ -22,7 +22,6 @@ Most calls are proxied into the responsible controller. """ import multiprocessing -import random import re import urllib @@ -38,17 +37,6 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -class APIRequestContext(object): - def __init__(self, handler, user, project): - self.handler = handler - self.user = user - self.project = project - self.request_id = ''.join( - [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') - for x in xrange(20)] - ) - - class RootRequestHandler(tornado.web.RequestHandler): def get(self): # available api versions -- cgit From 1ef59040aa1304a4682c6bcdaa3333372e7f8629 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 30 Aug 2010 19:12:31 -0400 Subject: Delete __init__.py in prep for turning apirequesthandler into __init__ --- nova/api/ec2/__init__.py | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 nova/api/ec2/__init__.py diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py deleted file mode 100644 index 6eec0abf7..000000000 --- a/nova/api/ec2/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -WSGI middleware for EC2 API controllers. -""" - -import routes -import webob.dec - -from nova import wsgi - - -class API(wsgi.Router): - """Routes EC2 requests to the appropriate controller.""" - - def __init__(self): - mapper = routes.Mapper() - mapper.connect(None, "{all:.*}", controller=self.dummy) - super(API, self).__init__(mapper) - - @staticmethod - @webob.dec.wsgify - def dummy(req): - """Temporary dummy controller.""" - msg = "dummy response -- please hook up __init__() to cloud.py instead" - return repr({'dummy': msg, - 'kwargs': repr(req.environ['wsgiorg.routing_args'][1])}) -- cgit From c54d6c3d1fcb0210e9f52097f1a1e85550c84bf6 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 31 Aug 2010 10:03:51 -0400 Subject: First steps in reworking EC2 APIRequestHandler into separate Authenticate() and Router() WSGI apps --- nova/api/__init__.py | 4 + nova/api/ec2/__init__.py | 151 ++++++++++++++++++++++++++++++++++++++ nova/api/ec2/apirequesthandler.py | 126 ------------------------------- nova/endpoint/notes.txt | 10 +-- 4 files changed, 160 insertions(+), 131 deletions(-) create mode 100644 nova/api/ec2/__init__.py delete mode 100644 nova/api/ec2/apirequesthandler.py diff --git a/nova/api/__init__.py b/nova/api/__init__.py index b9b9e3988..0166b7fc1 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -32,6 +32,10 @@ class API(wsgi.Router): def __init__(self): mapper = routes.Mapper() + # TODO(gundlach): EC2 RootController is replaced by this class; + # MetadataRequestHandlers isn't part of the EC2 API and thus can + # be dropped; and I'm leaving off CloudPipeRequestHandler until + # I hear that we need it. mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API()) mapper.connect("/ec2/{path_info:.*}", controller=ec2.API()) super(API, self).__init__(mapper) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py new file mode 100644 index 000000000..a4d9b95f9 --- /dev/null +++ b/nova/api/ec2/__init__.py @@ -0,0 +1,151 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Starting point for routing EC2 requests +""" + +import logging +import routes +import webob.exc +from webob.dec import wsgify + +from nova.api.ec2 import admin +from nova.api.ec2 import cloud +from nova import exception +from nova import utils +from nova.auth import manager + + +_log = logging.getLogger("api") +_log.setLevel(logging.DEBUG) + + +class API(wsgi.Middleware): + """Routing for all EC2 API requests.""" + + def __init__(self): + self.application = Authenticate(Router()) + +class Authenticate(wsgi.Middleware): + """Authenticates an EC2 request.""" + + @webob.dec.wsgify + def __call__(self, req): + #TODO(gundlach): where do arguments come from? + args = self.request.arguments + + # Read request signature. + try: + signature = args.pop('Signature')[0] + except: + raise webob.exc.HTTPBadRequest() + + # Make a copy of args for authentication and signature verification. + auth_params = {} + for key, value in args.items(): + auth_params[key] = value[0] + + # Get requested action and remove authentication args for final request. + try: + action = args.pop('Action')[0] + access = args.pop('AWSAccessKeyId')[0] + args.pop('SignatureMethod') + args.pop('SignatureVersion') + args.pop('Version') + args.pop('Timestamp') + except: + raise webob.exc.HTTPBadRequest() + + # Authenticate the request. + try: + (user, project) = manager.AuthManager().authenticate( + access, + signature, + auth_params, + req.method, + req.host, + req.path + ) + + except exception.Error, ex: + logging.debug("Authentication Failure: %s" % ex) + raise webob.exc.HTTPForbidden() + + _log.debug('action: %s' % action) + + for key, value in args.items(): + _log.debug('arg: %s\t\tval: %s' % (key, value)) + + # Authenticated! + req.environ['ec2.action'] = action + req.environ['ec2.context'] = APIRequestContext(user, project) + return self.application + + +class Router(wsgi.Application): + """ + Finds controller for a request, executes environ['ec2.action'] upon it, and + returns a response. + """ + def __init__(self): + self.map = routes.Mapper() + self.map.connect("/{controller_name}/") + self.controllers = dict(Cloud=cloud.CloudController(), + Admin=admin.AdminController()) + + def __call__(self, req): + # Obtain the appropriate controller for this request. + match = self.map.match(req.path) + if not match: + raise webob.exc.HTTPNotFound() + controller_name = match['controller_name'] + + try: + controller = self.controllers[controller_name] + except KeyError: + self._error('unhandled', 'no controller named %s' % controller_name) + return + + request = APIRequest(controller, req.environ['ec2.action']) + context = req.environ['ec2.context'] + try: + data = request.send(context, **args) + req.headers['Content-Type'] = 'text/xml' + return data + #TODO(gundlach) under what conditions would _error_callbock used to + #be called? What was 'failure' that you could call .raiseException + #on it? + except Exception, ex: + try: + #TODO + failure.raiseException() + except exception.ApiError as ex: + self._error(req, type(ex).__name__ + "." + ex.code, ex.message) + # TODO(vish): do something more useful with unknown exceptions + except Exception as ex: + self._error(type(ex).__name__, str(ex)) + + def _error(self, req, code, message): + req.status = 400 + req.headers['Content-Type'] = 'text/xml' + req.response = ('\n' + '%s' + '%s' + '?') % (code, message)) + diff --git a/nova/api/ec2/apirequesthandler.py b/nova/api/ec2/apirequesthandler.py deleted file mode 100644 index bbba60c02..000000000 --- a/nova/api/ec2/apirequesthandler.py +++ /dev/null @@ -1,126 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -APIRequestHandler, pulled unmodified out of nova.endpoint.api -""" - -import logging - -import tornado.web - -from nova import exception -from nova import utils -from nova.auth import manager - - -_log = logging.getLogger("api") -_log.setLevel(logging.DEBUG) - - -class APIRequestHandler(tornado.web.RequestHandler): - def get(self, controller_name): - self.execute(controller_name) - - @tornado.web.asynchronous - def execute(self, controller_name): - # Obtain the appropriate controller for this request. - try: - controller = self.application.controllers[controller_name] - except KeyError: - self._error('unhandled', 'no controller named %s' % controller_name) - return - - args = self.request.arguments - - # Read request signature. - try: - signature = args.pop('Signature')[0] - except: - raise tornado.web.HTTPError(400) - - # Make a copy of args for authentication and signature verification. - auth_params = {} - for key, value in args.items(): - auth_params[key] = value[0] - - # Get requested action and remove authentication args for final request. - try: - action = args.pop('Action')[0] - access = args.pop('AWSAccessKeyId')[0] - args.pop('SignatureMethod') - args.pop('SignatureVersion') - args.pop('Version') - args.pop('Timestamp') - except: - raise tornado.web.HTTPError(400) - - # Authenticate the request. - try: - (user, project) = manager.AuthManager().authenticate( - access, - signature, - auth_params, - self.request.method, - self.request.host, - self.request.path - ) - - except exception.Error, ex: - logging.debug("Authentication Failure: %s" % ex) - raise tornado.web.HTTPError(403) - - _log.debug('action: %s' % action) - - for key, value in args.items(): - _log.debug('arg: %s\t\tval: %s' % (key, value)) - - request = APIRequest(controller, action) - context = APIRequestContext(self, user, project) - d = request.send(context, **args) - # d.addCallback(utils.debug) - - # TODO: Wrap response in AWS XML format - d.addCallbacks(self._write_callback, self._error_callback) - - def _write_callback(self, data): - self.set_header('Content-Type', 'text/xml') - self.write(data) - self.finish() - - def _error_callback(self, failure): - try: - failure.raiseException() - except exception.ApiError as ex: - self._error(type(ex).__name__ + "." + ex.code, ex.message) - # TODO(vish): do something more useful with unknown exceptions - except Exception as ex: - self._error(type(ex).__name__, str(ex)) - raise - - def post(self, controller_name): - self.execute(controller_name) - - def _error(self, code, message): - self._status_code = 400 - self.set_header('Content-Type', 'text/xml') - self.write('\n') - self.write('%s' - '%s' - '?' % (code, message)) - self.finish() diff --git a/nova/endpoint/notes.txt b/nova/endpoint/notes.txt index c1d441de0..7a85cdc93 100644 --- a/nova/endpoint/notes.txt +++ b/nova/endpoint/notes.txt @@ -17,7 +17,9 @@ APIServerApplication(tornado.web.Application) APIRequestHandler execute: authenticates request - picks controller from APIServerApplication's list + picks controller from APIServerApplication's list based on name that was at the + start of the URL (e.g. /services/Cloud has /services mapped here via + APIServerApplication then Cloud is controller_name) picks action from incoming request arguments dict = APIRequest(controller, action).send(Context(user, project)) _write_callback(dict) @@ -49,12 +51,10 @@ CloudController and AdminController: * apiserverapplication: replace with a Router to a wsgi.Controller - root controller all goes into a "version" action - ??? dunno what cloudpipes or metadatarequesthandlers do... - apirequesthandler stuff goes into an "ec2" action + apirequesthandler stuff is just an entry in api.APIRouter * apirequesthandler - ec2() method on wsgi.Controller + wsgi.Controller pointed to by api.APIRouter - basically it's execute() from old APIRequestHandler change to return data directly instead of _write_callback() and finish() -- cgit From 070d87df264ca949b51131df9287fbcee373d480 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 31 Aug 2010 10:46:01 -0400 Subject: Get rid of some convoluted exception handling that we don't need in eventlet --- nova/api/ec2/__init__.py | 23 +++++++---------------- nova/api/ec2/apirequest.py | 7 ++++--- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index a4d9b95f9..46e543d0e 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -122,24 +122,15 @@ class Router(wsgi.Application): self._error('unhandled', 'no controller named %s' % controller_name) return - request = APIRequest(controller, req.environ['ec2.action']) + api_request = APIRequest(controller, req.environ['ec2.action']) context = req.environ['ec2.context'] try: - data = request.send(context, **args) - req.headers['Content-Type'] = 'text/xml' - return data - #TODO(gundlach) under what conditions would _error_callbock used to - #be called? What was 'failure' that you could call .raiseException - #on it? - except Exception, ex: - try: - #TODO - failure.raiseException() - except exception.ApiError as ex: - self._error(req, type(ex).__name__ + "." + ex.code, ex.message) - # TODO(vish): do something more useful with unknown exceptions - except Exception as ex: - self._error(type(ex).__name__, str(ex)) + return api_request.send(context, **args) + except exception.ApiError as ex: + self._error(req, type(ex).__name__ + "." + ex.code, ex.message) + # TODO(vish): do something more useful with unknown exceptions + except Exception as ex: + self._error(type(ex).__name__, str(ex)) def _error(self, req, code, message): req.status = 400 diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 1fc84248b..77f1a7759 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -79,9 +79,10 @@ class APIRequest(object): s.sort() args[key] = [v for k, v in s] - d = defer.maybeDeferred(method, context, **args) - d.addCallback(self._render_response, context.request_id) - return d + result = method(context, **args) + + req.headers['Content-Type'] = 'text/xml' + return self._render_response(result, context.request_id) def _render_response(self, response_data, request_id): xml = minidom.Document() -- cgit From cb55d65827170dd9d54dbd22f32e5c2171f8e1b1 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 31 Aug 2010 10:55:53 -0400 Subject: small import cleanup --- nova/api/ec2/__init__.py | 5 ++--- nova/api/ec2/apirequest.py | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 46e543d0e..7e345d297 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -22,13 +22,12 @@ Starting point for routing EC2 requests import logging import routes +import webob.dec import webob.exc -from webob.dec import wsgify from nova.api.ec2 import admin from nova.api.ec2 import cloud from nova import exception -from nova import utils from nova.auth import manager @@ -101,7 +100,7 @@ class Authenticate(wsgi.Middleware): class Router(wsgi.Application): """ Finds controller for a request, executes environ['ec2.action'] upon it, and - returns a response. + returns an XML response. If the action fails, returns a 400. """ def __init__(self): self.map = routes.Mapper() diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 77f1a7759..261346a09 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -23,8 +23,6 @@ APIRequest class # TODO(termie): replace minidom with etree from xml.dom import minidom -from twisted.internet import defer - _c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') -- cgit From ab43c28e583116c4885b19afc6448192aae10096 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 31 Aug 2010 12:15:29 -0400 Subject: Move cloudcontroller and admincontroller into new api --- nova/api/ec2/admin.py | 211 ++++++++++++++ nova/api/ec2/cloud.py | 739 +++++++++++++++++++++++++++++++++++++++++++++++++ nova/endpoint/admin.py | 211 -------------- nova/endpoint/cloud.py | 739 ------------------------------------------------- 4 files changed, 950 insertions(+), 950 deletions(-) create mode 100644 nova/api/ec2/admin.py create mode 100644 nova/api/ec2/cloud.py delete mode 100644 nova/endpoint/admin.py delete mode 100644 nova/endpoint/cloud.py diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py new file mode 100644 index 000000000..d6f622755 --- /dev/null +++ b/nova/api/ec2/admin.py @@ -0,0 +1,211 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Admin API controller, exposed through http via the api worker. +""" + +import base64 + +from nova.auth import manager +from nova.compute import model + + +def user_dict(user, base64_file=None): + """Convert the user object to a result dict""" + if user: + return { + 'username': user.id, + 'accesskey': user.access, + 'secretkey': user.secret, + 'file': base64_file} + else: + return {} + + +def project_dict(project): + """Convert the project object to a result dict""" + if project: + return { + 'projectname': project.id, + 'project_manager_id': project.project_manager_id, + 'description': project.description} + else: + return {} + + +def host_dict(host): + """Convert a host model object to a result dict""" + if host: + return host.state + else: + return {} + + +def admin_only(target): + """Decorator for admin-only API calls""" + def wrapper(*args, **kwargs): + """Internal wrapper method for admin-only API calls""" + context = args[1] + if context.user.is_admin(): + return target(*args, **kwargs) + else: + return {} + + return wrapper + + +class AdminController(object): + """ + API Controller for users, hosts, nodes, and workers. + Trivial admin_only wrapper will be replaced with RBAC, + allowing project managers to administer project users. + """ + + def __str__(self): + return 'AdminController' + + @admin_only + def describe_user(self, _context, name, **_kwargs): + """Returns user data, including access and secret keys.""" + return user_dict(manager.AuthManager().get_user(name)) + + @admin_only + def describe_users(self, _context, **_kwargs): + """Returns all users - should be changed to deal with a list.""" + return {'userSet': + [user_dict(u) for u in manager.AuthManager().get_users()] } + + @admin_only + def register_user(self, _context, name, **_kwargs): + """Creates a new user, and returns generated credentials.""" + return user_dict(manager.AuthManager().create_user(name)) + + @admin_only + def deregister_user(self, _context, name, **_kwargs): + """Deletes a single user (NOT undoable.) + Should throw an exception if the user has instances, + volumes, or buckets remaining. + """ + manager.AuthManager().delete_user(name) + + return True + + @admin_only + def describe_roles(self, context, project_roles=True, **kwargs): + """Returns a list of allowed roles.""" + roles = manager.AuthManager().get_roles(project_roles) + return { 'roles': [{'role': r} for r in roles]} + + @admin_only + def describe_user_roles(self, context, user, project=None, **kwargs): + """Returns a list of roles for the given user. + Omitting project will return any global roles that the user has. + Specifying project will return only project specific roles. + """ + roles = manager.AuthManager().get_user_roles(user, project=project) + return { 'roles': [{'role': r} for r in roles]} + + @admin_only + def modify_user_role(self, context, user, role, project=None, + operation='add', **kwargs): + """Add or remove a role for a user and project.""" + if operation == 'add': + manager.AuthManager().add_role(user, role, project) + elif operation == 'remove': + manager.AuthManager().remove_role(user, role, project) + else: + raise exception.ApiError('operation must be add or remove') + + return True + + @admin_only + def generate_x509_for_user(self, _context, name, project=None, **kwargs): + """Generates and returns an x509 certificate for a single user. + Is usually called from a client that will wrap this with + access and secret key info, and return a zip file. + """ + if project is None: + project = name + project = manager.AuthManager().get_project(project) + user = manager.AuthManager().get_user(name) + return user_dict(user, base64.b64encode(project.get_credentials(user))) + + @admin_only + def describe_project(self, context, name, **kwargs): + """Returns project data, including member ids.""" + return project_dict(manager.AuthManager().get_project(name)) + + @admin_only + def describe_projects(self, context, user=None, **kwargs): + """Returns all projects - should be changed to deal with a list.""" + return {'projectSet': + [project_dict(u) for u in + manager.AuthManager().get_projects(user=user)]} + + @admin_only + def register_project(self, context, name, manager_user, description=None, + member_users=None, **kwargs): + """Creates a new project""" + return project_dict( + manager.AuthManager().create_project( + name, + manager_user, + description=None, + member_users=None)) + + @admin_only + def deregister_project(self, context, name): + """Permanently deletes a project.""" + manager.AuthManager().delete_project(name) + return True + + @admin_only + def describe_project_members(self, context, name, **kwargs): + project = manager.AuthManager().get_project(name) + result = { + 'members': [{'member': m} for m in project.member_ids]} + return result + + @admin_only + def modify_project_member(self, context, user, project, operation, **kwargs): + """Add or remove a user from a project.""" + if operation =='add': + manager.AuthManager().add_to_project(user, project) + elif operation == 'remove': + manager.AuthManager().remove_from_project(user, project) + else: + raise exception.ApiError('operation must be add or remove') + return True + + @admin_only + def describe_hosts(self, _context, **_kwargs): + """Returns status info for all nodes. Includes: + * Disk Space + * Instance List + * RAM used + * CPU used + * DHCP servers running + * Iptables / bridges + """ + return {'hostSet': [host_dict(h) for h in model.Host.all()]} + + @admin_only + def describe_host(self, _context, name, **_kwargs): + """Returns status info for single node.""" + return host_dict(model.Host.lookup(name)) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py new file mode 100644 index 000000000..30634429d --- /dev/null +++ b/nova/api/ec2/cloud.py @@ -0,0 +1,739 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Cloud Controller: Implementation of EC2 REST API calls, which are +dispatched to other nodes via AMQP RPC. State is via distributed +datastore. +""" + +import base64 +import logging +import os +import time + +from twisted.internet import defer + +from nova import datastore +from nova import exception +from nova import flags +from nova import rpc +from nova import utils +from nova.auth import rbac +from nova.auth import manager +from nova.compute import model +from nova.compute.instance_types import INSTANCE_TYPES +from nova.endpoint import images +from nova.network import service as network_service +from nova.network import model as network_model +from nova.volume import service + + +FLAGS = flags.FLAGS +flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') + + +def _gen_key(user_id, key_name): + """ Tuck this into AuthManager """ + try: + mgr = manager.AuthManager() + private_key, fingerprint = mgr.generate_key_pair(user_id, key_name) + except Exception as ex: + return {'exception': ex} + return {'private_key': private_key, 'fingerprint': fingerprint} + + +class CloudController(object): + """ CloudController provides the critical dispatch between + inbound API calls through the endpoint and messages + sent to the other nodes. +""" + def __init__(self): + self.instdir = model.InstanceDirectory() + self.setup() + + @property + def instances(self): + """ All instances in the system, as dicts """ + return self.instdir.all + + @property + def volumes(self): + """ returns a list of all volumes """ + for volume_id in datastore.Redis.instance().smembers("volumes"): + volume = service.get_volume(volume_id) + yield volume + + def __str__(self): + return 'CloudController' + + def setup(self): + """ Ensure the keychains and folders exist. """ + # Create keys folder, if it doesn't exist + if not os.path.exists(FLAGS.keys_path): + os.makedirs(FLAGS.keys_path) + # Gen root CA, if we don't have one + root_ca_path = os.path.join(FLAGS.ca_path, FLAGS.ca_file) + if not os.path.exists(root_ca_path): + start = os.getcwd() + os.chdir(FLAGS.ca_path) + utils.runthis("Generating root CA: %s", "sh genrootca.sh") + os.chdir(start) + # TODO: Do this with M2Crypto instead + + def get_instance_by_ip(self, ip): + return self.instdir.by_ip(ip) + + def _get_mpi_data(self, project_id): + result = {} + for instance in self.instdir.all: + if instance['project_id'] == project_id: + line = '%s slots=%d' % (instance['private_dns_name'], + INSTANCE_TYPES[instance['instance_type']]['vcpus']) + if instance['key_name'] in result: + result[instance['key_name']].append(line) + else: + result[instance['key_name']] = [line] + return result + + def get_metadata(self, ipaddress): + i = self.get_instance_by_ip(ipaddress) + if i is None: + return None + mpi = self._get_mpi_data(i['project_id']) + if i['key_name']: + keys = { + '0': { + '_name': i['key_name'], + 'openssh-key': i['key_data'] + } + } + else: + keys = '' + + address_record = network_model.FixedIp(i['private_dns_name']) + if address_record: + hostname = address_record['hostname'] + else: + hostname = 'ip-%s' % i['private_dns_name'].replace('.', '-') + data = { + 'user-data': base64.b64decode(i['user_data']), + 'meta-data': { + 'ami-id': i['image_id'], + 'ami-launch-index': i['ami_launch_index'], + 'ami-manifest-path': 'FIXME', # image property + 'block-device-mapping': { # TODO: replace with real data + 'ami': 'sda1', + 'ephemeral0': 'sda2', + 'root': '/dev/sda1', + 'swap': 'sda3' + }, + 'hostname': hostname, + 'instance-action': 'none', + 'instance-id': i['instance_id'], + 'instance-type': i.get('instance_type', ''), + 'local-hostname': hostname, + 'local-ipv4': i['private_dns_name'], # TODO: switch to IP + 'kernel-id': i.get('kernel_id', ''), + 'placement': { + 'availaibility-zone': i.get('availability_zone', 'nova'), + }, + 'public-hostname': hostname, + 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP + 'public-keys': keys, + 'ramdisk-id': i.get('ramdisk_id', ''), + 'reservation-id': i['reservation_id'], + 'security-groups': i.get('groups', ''), + 'mpi': mpi + } + } + if False: # TODO: store ancestor ids + data['ancestor-ami-ids'] = [] + if i.get('product_codes', None): + data['product-codes'] = i['product_codes'] + return data + + @rbac.allow('all') + def describe_availability_zones(self, context, **kwargs): + return {'availabilityZoneInfo': [{'zoneName': 'nova', + 'zoneState': 'available'}]} + + @rbac.allow('all') + def describe_regions(self, context, region_name=None, **kwargs): + # TODO(vish): region_name is an array. Support filtering + return {'regionInfo': [{'regionName': 'nova', + 'regionUrl': FLAGS.ec2_url}]} + + @rbac.allow('all') + def describe_snapshots(self, + context, + snapshot_id=None, + owner=None, + restorable_by=None, + **kwargs): + return {'snapshotSet': [{'snapshotId': 'fixme', + 'volumeId': 'fixme', + 'status': 'fixme', + 'startTime': 'fixme', + 'progress': 'fixme', + 'ownerId': 'fixme', + 'volumeSize': 0, + 'description': 'fixme'}]} + + @rbac.allow('all') + def describe_key_pairs(self, context, key_name=None, **kwargs): + key_pairs = context.user.get_key_pairs() + if not key_name is None: + key_pairs = [x for x in key_pairs if x.name in key_name] + + result = [] + for key_pair in key_pairs: + # filter out the vpn keys + suffix = FLAGS.vpn_key_suffix + if context.user.is_admin() or not key_pair.name.endswith(suffix): + result.append({ + 'keyName': key_pair.name, + 'keyFingerprint': key_pair.fingerprint, + }) + + return {'keypairsSet': result} + + @rbac.allow('all') + def create_key_pair(self, context, key_name, **kwargs): + dcall = defer.Deferred() + pool = context.handler.application.settings.get('pool') + def _complete(kwargs): + if 'exception' in kwargs: + dcall.errback(kwargs['exception']) + return + dcall.callback({'keyName': key_name, + 'keyFingerprint': kwargs['fingerprint'], + 'keyMaterial': kwargs['private_key']}) + pool.apply_async(_gen_key, [context.user.id, key_name], + callback=_complete) + return dcall + + @rbac.allow('all') + def delete_key_pair(self, context, key_name, **kwargs): + context.user.delete_key_pair(key_name) + # aws returns true even if the key doens't exist + return True + + @rbac.allow('all') + def describe_security_groups(self, context, group_names, **kwargs): + groups = {'securityGroupSet': []} + + # Stubbed for now to unblock other things. + return groups + + @rbac.allow('netadmin') + def create_security_group(self, context, group_name, **kwargs): + return True + + @rbac.allow('netadmin') + def delete_security_group(self, context, group_name, **kwargs): + return True + + @rbac.allow('projectmanager', 'sysadmin') + def get_console_output(self, context, instance_id, **kwargs): + # instance_id is passed in as a list of instances + instance = self._get_instance(context, instance_id[0]) + return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), + {"method": "get_console_output", + "args": {"instance_id": instance_id[0]}}) + + def _get_user_id(self, context): + if context and context.user: + return context.user.id + else: + return None + + @rbac.allow('projectmanager', 'sysadmin') + def describe_volumes(self, context, **kwargs): + volumes = [] + for volume in self.volumes: + if context.user.is_admin() or volume['project_id'] == context.project.id: + v = self.format_volume(context, volume) + volumes.append(v) + return defer.succeed({'volumeSet': volumes}) + + def format_volume(self, context, volume): + v = {} + v['volumeId'] = volume['volume_id'] + v['status'] = volume['status'] + v['size'] = volume['size'] + v['availabilityZone'] = volume['availability_zone'] + v['createTime'] = volume['create_time'] + if context.user.is_admin(): + v['status'] = '%s (%s, %s, %s, %s)' % ( + volume.get('status', None), + volume.get('user_id', None), + volume.get('node_name', None), + volume.get('instance_id', ''), + volume.get('mountpoint', '')) + if volume['attach_status'] == 'attached': + v['attachmentSet'] = [{'attachTime': volume['attach_time'], + 'deleteOnTermination': volume['delete_on_termination'], + 'device': volume['mountpoint'], + 'instanceId': volume['instance_id'], + 'status': 'attached', + 'volume_id': volume['volume_id']}] + else: + v['attachmentSet'] = [{}] + return v + + @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks + def create_volume(self, context, size, **kwargs): + # TODO(vish): refactor this to create the volume object here and tell service to create it + result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", + "args": {"size": size, + "user_id": context.user.id, + "project_id": context.project.id}}) + # NOTE(vish): rpc returned value is in the result key in the dictionary + volume = self._get_volume(context, result) + defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) + + def _get_address(self, context, public_ip): + # FIXME(vish) this should move into network.py + address = network_model.ElasticIp.lookup(public_ip) + if address and (context.user.is_admin() or address['project_id'] == context.project.id): + return address + raise exception.NotFound("Address at ip %s not found" % public_ip) + + def _get_image(self, context, image_id): + """passes in context because + objectstore does its own authorization""" + result = images.list(context, [image_id]) + if not result: + raise exception.NotFound('Image %s could not be found' % image_id) + image = result[0] + return image + + def _get_instance(self, context, instance_id): + for instance in self.instdir.all: + if instance['instance_id'] == instance_id: + if context.user.is_admin() or instance['project_id'] == context.project.id: + return instance + raise exception.NotFound('Instance %s could not be found' % instance_id) + + def _get_volume(self, context, volume_id): + volume = service.get_volume(volume_id) + if context.user.is_admin() or volume['project_id'] == context.project.id: + return volume + raise exception.NotFound('Volume %s could not be found' % volume_id) + + @rbac.allow('projectmanager', 'sysadmin') + def attach_volume(self, context, volume_id, instance_id, device, **kwargs): + volume = self._get_volume(context, volume_id) + if volume['status'] == "attached": + raise exception.ApiError("Volume is already attached") + # TODO(vish): looping through all volumes is slow. We should probably maintain an index + for vol in self.volumes: + if vol['instance_id'] == instance_id and vol['mountpoint'] == device: + raise exception.ApiError("Volume %s is already attached to %s" % (vol['volume_id'], vol['mountpoint'])) + volume.start_attach(instance_id, device) + instance = self._get_instance(context, instance_id) + compute_node = instance['node_name'] + rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), + {"method": "attach_volume", + "args": {"volume_id": volume_id, + "instance_id": instance_id, + "mountpoint": device}}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) + + @rbac.allow('projectmanager', 'sysadmin') + def detach_volume(self, context, volume_id, **kwargs): + volume = self._get_volume(context, volume_id) + instance_id = volume.get('instance_id', None) + if not instance_id: + raise exception.Error("Volume isn't attached to anything!") + if volume['status'] == "available": + raise exception.Error("Volume is already detached") + try: + volume.start_detach() + instance = self._get_instance(context, instance_id) + rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), + {"method": "detach_volume", + "args": {"instance_id": instance_id, + "volume_id": volume_id}}) + except exception.NotFound: + # If the instance doesn't exist anymore, + # then we need to call detach blind + volume.finish_detach() + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) + + def _convert_to_set(self, lst, label): + if lst == None or lst == []: + return None + if not isinstance(lst, list): + lst = [lst] + return [{label: x} for x in lst] + + @rbac.allow('all') + def describe_instances(self, context, **kwargs): + return defer.succeed(self._format_describe_instances(context)) + + def _format_describe_instances(self, context): + return { 'reservationSet': self._format_instances(context) } + + def _format_run_instances(self, context, reservation_id): + i = self._format_instances(context, reservation_id) + assert len(i) == 1 + return i[0] + + def _format_instances(self, context, reservation_id = None): + reservations = {} + if context.user.is_admin(): + instgenerator = self.instdir.all + else: + instgenerator = self.instdir.by_project(context.project.id) + for instance in instgenerator: + res_id = instance.get('reservation_id', 'Unknown') + if reservation_id != None and reservation_id != res_id: + continue + if not context.user.is_admin(): + if instance['image_id'] == FLAGS.vpn_image_id: + continue + i = {} + i['instance_id'] = instance.get('instance_id', None) + i['image_id'] = instance.get('image_id', None) + i['instance_state'] = { + 'code': instance.get('state', 0), + 'name': instance.get('state_description', 'pending') + } + i['public_dns_name'] = network_model.get_public_ip_for_instance( + i['instance_id']) + i['private_dns_name'] = instance.get('private_dns_name', None) + if not i['public_dns_name']: + i['public_dns_name'] = i['private_dns_name'] + i['dns_name'] = instance.get('dns_name', None) + i['key_name'] = instance.get('key_name', None) + if context.user.is_admin(): + i['key_name'] = '%s (%s, %s)' % (i['key_name'], + instance.get('project_id', None), + instance.get('node_name', '')) + i['product_codes_set'] = self._convert_to_set( + instance.get('product_codes', None), 'product_code') + i['instance_type'] = instance.get('instance_type', None) + i['launch_time'] = instance.get('launch_time', None) + i['ami_launch_index'] = instance.get('ami_launch_index', + None) + if not reservations.has_key(res_id): + r = {} + r['reservation_id'] = res_id + r['owner_id'] = instance.get('project_id', None) + r['group_set'] = self._convert_to_set( + instance.get('groups', None), 'group_id') + r['instances_set'] = [] + reservations[res_id] = r + reservations[res_id]['instances_set'].append(i) + + return list(reservations.values()) + + @rbac.allow('all') + def describe_addresses(self, context, **kwargs): + return self.format_addresses(context) + + def format_addresses(self, context): + addresses = [] + for address in network_model.ElasticIp.all(): + # TODO(vish): implement a by_project iterator for addresses + if (context.user.is_admin() or + address['project_id'] == context.project.id): + address_rv = { + 'public_ip': address['address'], + 'instance_id': address.get('instance_id', 'free') + } + if context.user.is_admin(): + address_rv['instance_id'] = "%s (%s, %s)" % ( + address['instance_id'], + address['user_id'], + address['project_id'], + ) + addresses.append(address_rv) + return {'addressesSet': addresses} + + @rbac.allow('netadmin') + @defer.inlineCallbacks + def allocate_address(self, context, **kwargs): + network_topic = yield self._get_network_topic(context) + public_ip = yield rpc.call(network_topic, + {"method": "allocate_elastic_ip", + "args": {"user_id": context.user.id, + "project_id": context.project.id}}) + defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) + + @rbac.allow('netadmin') + @defer.inlineCallbacks + def release_address(self, context, public_ip, **kwargs): + # NOTE(vish): Should we make sure this works? + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, + {"method": "deallocate_elastic_ip", + "args": {"elastic_ip": public_ip}}) + defer.returnValue({'releaseResponse': ["Address released."]}) + + @rbac.allow('netadmin') + @defer.inlineCallbacks + def associate_address(self, context, instance_id, public_ip, **kwargs): + instance = self._get_instance(context, instance_id) + address = self._get_address(context, public_ip) + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, + {"method": "associate_elastic_ip", + "args": {"elastic_ip": address['address'], + "fixed_ip": instance['private_dns_name'], + "instance_id": instance['instance_id']}}) + defer.returnValue({'associateResponse': ["Address associated."]}) + + @rbac.allow('netadmin') + @defer.inlineCallbacks + def disassociate_address(self, context, public_ip, **kwargs): + address = self._get_address(context, public_ip) + network_topic = yield self._get_network_topic(context) + rpc.cast(network_topic, + {"method": "disassociate_elastic_ip", + "args": {"elastic_ip": address['address']}}) + defer.returnValue({'disassociateResponse': ["Address disassociated."]}) + + @defer.inlineCallbacks + def _get_network_topic(self, context): + """Retrieves the network host for a project""" + host = network_service.get_host_for_project(context.project.id) + if not host: + host = yield rpc.call(FLAGS.network_topic, + {"method": "set_network_host", + "args": {"user_id": context.user.id, + "project_id": context.project.id}}) + defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) + + @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks + def run_instances(self, context, **kwargs): + # make sure user can access the image + # vpn image is private so it doesn't show up on lists + if kwargs['image_id'] != FLAGS.vpn_image_id: + image = self._get_image(context, kwargs['image_id']) + + # FIXME(ja): if image is cloudpipe, this breaks + + # get defaults from imagestore + image_id = image['imageId'] + kernel_id = image.get('kernelId', FLAGS.default_kernel) + ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) + + # API parameters overrides of defaults + kernel_id = kwargs.get('kernel_id', kernel_id) + ramdisk_id = kwargs.get('ramdisk_id', ramdisk_id) + + # make sure we have access to kernel and ramdisk + self._get_image(context, kernel_id) + self._get_image(context, ramdisk_id) + + logging.debug("Going to run instances...") + reservation_id = utils.generate_uid('r') + launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + key_data = None + if kwargs.has_key('key_name'): + key_pair = context.user.get_key_pair(kwargs['key_name']) + if not key_pair: + raise exception.ApiError('Key Pair %s not found' % + kwargs['key_name']) + key_data = key_pair.public_key + network_topic = yield self._get_network_topic(context) + # TODO: Get the real security group of launch in here + security_group = "default" + for num in range(int(kwargs['max_count'])): + is_vpn = False + if image_id == FLAGS.vpn_image_id: + is_vpn = True + inst = self.instdir.new() + allocate_data = yield rpc.call(network_topic, + {"method": "allocate_fixed_ip", + "args": {"user_id": context.user.id, + "project_id": context.project.id, + "security_group": security_group, + "is_vpn": is_vpn, + "hostname": inst.instance_id}}) + inst['image_id'] = image_id + inst['kernel_id'] = kernel_id + inst['ramdisk_id'] = ramdisk_id + inst['user_data'] = kwargs.get('user_data', '') + inst['instance_type'] = kwargs.get('instance_type', 'm1.small') + inst['reservation_id'] = reservation_id + inst['launch_time'] = launch_time + inst['key_data'] = key_data or '' + inst['key_name'] = kwargs.get('key_name', '') + inst['user_id'] = context.user.id + inst['project_id'] = context.project.id + inst['ami_launch_index'] = num + inst['security_group'] = security_group + inst['hostname'] = inst.instance_id + for (key, value) in allocate_data.iteritems(): + inst[key] = value + + inst.save() + rpc.cast(FLAGS.compute_topic, + {"method": "run_instance", + "args": {"instance_id": inst.instance_id}}) + logging.debug("Casting to node for %s's instance with IP of %s" % + (context.user.name, inst['private_dns_name'])) + # TODO: Make Network figure out the network name from ip. + defer.returnValue(self._format_run_instances(context, reservation_id)) + + @rbac.allow('projectmanager', 'sysadmin') + @defer.inlineCallbacks + def terminate_instances(self, context, instance_id, **kwargs): + logging.debug("Going to start terminating instances") + network_topic = yield self._get_network_topic(context) + for i in instance_id: + logging.debug("Going to try and terminate %s" % i) + try: + instance = self._get_instance(context, i) + except exception.NotFound: + logging.warning("Instance %s was not found during terminate" + % i) + continue + elastic_ip = network_model.get_public_ip_for_instance(i) + if elastic_ip: + logging.debug("Disassociating address %s" % elastic_ip) + # NOTE(vish): Right now we don't really care if the ip is + # disassociated. We may need to worry about + # checking this later. Perhaps in the scheduler? + rpc.cast(network_topic, + {"method": "disassociate_elastic_ip", + "args": {"elastic_ip": elastic_ip}}) + + fixed_ip = instance.get('private_dns_name', None) + if fixed_ip: + logging.debug("Deallocating address %s" % fixed_ip) + # NOTE(vish): Right now we don't really care if the ip is + # actually removed. We may need to worry about + # checking this later. Perhaps in the scheduler? + rpc.cast(network_topic, + {"method": "deallocate_fixed_ip", + "args": {"fixed_ip": fixed_ip}}) + + if instance.get('node_name', 'unassigned') != 'unassigned': + # NOTE(joshua?): It's also internal default + rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), + {"method": "terminate_instance", + "args": {"instance_id": i}}) + else: + instance.destroy() + defer.returnValue(True) + + @rbac.allow('projectmanager', 'sysadmin') + def reboot_instances(self, context, instance_id, **kwargs): + """instance_id is a list of instance ids""" + for i in instance_id: + instance = self._get_instance(context, i) + rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), + {"method": "reboot_instance", + "args": {"instance_id": i}}) + return defer.succeed(True) + + @rbac.allow('projectmanager', 'sysadmin') + def delete_volume(self, context, volume_id, **kwargs): + # TODO: return error if not authorized + volume = self._get_volume(context, volume_id) + volume_node = volume['node_name'] + rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), + {"method": "delete_volume", + "args": {"volume_id": volume_id}}) + return defer.succeed(True) + + @rbac.allow('all') + def describe_images(self, context, image_id=None, **kwargs): + # The objectstore does its own authorization for describe + imageSet = images.list(context, image_id) + return defer.succeed({'imagesSet': imageSet}) + + @rbac.allow('projectmanager', 'sysadmin') + def deregister_image(self, context, image_id, **kwargs): + # FIXME: should the objectstore be doing these authorization checks? + images.deregister(context, image_id) + return defer.succeed({'imageId': image_id}) + + @rbac.allow('projectmanager', 'sysadmin') + def register_image(self, context, image_location=None, **kwargs): + # FIXME: should the objectstore be doing these authorization checks? + if image_location is None and kwargs.has_key('name'): + image_location = kwargs['name'] + image_id = images.register(context, image_location) + logging.debug("Registered %s as %s" % (image_location, image_id)) + + return defer.succeed({'imageId': image_id}) + + @rbac.allow('all') + def describe_image_attribute(self, context, image_id, attribute, **kwargs): + if attribute != 'launchPermission': + raise exception.ApiError('attribute not supported: %s' % attribute) + try: + image = images.list(context, image_id)[0] + except IndexError: + raise exception.ApiError('invalid id: %s' % image_id) + result = {'image_id': image_id, 'launchPermission': []} + if image['isPublic']: + result['launchPermission'].append({'group': 'all'}) + return defer.succeed(result) + + @rbac.allow('projectmanager', 'sysadmin') + def modify_image_attribute(self, context, image_id, attribute, operation_type, **kwargs): + # TODO(devcamcar): Support users and groups other than 'all'. + if attribute != 'launchPermission': + raise exception.ApiError('attribute not supported: %s' % attribute) + if not 'user_group' in kwargs: + raise exception.ApiError('user or group not specified') + if len(kwargs['user_group']) != 1 and kwargs['user_group'][0] != 'all': + raise exception.ApiError('only group "all" is supported') + if not operation_type in ['add', 'remove']: + raise exception.ApiError('operation_type must be add or remove') + result = images.modify(context, image_id, operation_type) + return defer.succeed(result) + + def update_state(self, topic, value): + """ accepts status reports from the queue and consolidates them """ + # TODO(jmc): if an instance has disappeared from + # the node, call instance_death + if topic == "instances": + return defer.succeed(True) + aggregate_state = getattr(self, topic) + node_name = value.keys()[0] + items = value[node_name] + + logging.debug("Updating %s state for %s" % (topic, node_name)) + + for item_id in items.keys(): + if (aggregate_state.has_key('pending') and + aggregate_state['pending'].has_key(item_id)): + del aggregate_state['pending'][item_id] + aggregate_state[node_name] = items + + return defer.succeed(True) diff --git a/nova/endpoint/admin.py b/nova/endpoint/admin.py deleted file mode 100644 index d6f622755..000000000 --- a/nova/endpoint/admin.py +++ /dev/null @@ -1,211 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Admin API controller, exposed through http via the api worker. -""" - -import base64 - -from nova.auth import manager -from nova.compute import model - - -def user_dict(user, base64_file=None): - """Convert the user object to a result dict""" - if user: - return { - 'username': user.id, - 'accesskey': user.access, - 'secretkey': user.secret, - 'file': base64_file} - else: - return {} - - -def project_dict(project): - """Convert the project object to a result dict""" - if project: - return { - 'projectname': project.id, - 'project_manager_id': project.project_manager_id, - 'description': project.description} - else: - return {} - - -def host_dict(host): - """Convert a host model object to a result dict""" - if host: - return host.state - else: - return {} - - -def admin_only(target): - """Decorator for admin-only API calls""" - def wrapper(*args, **kwargs): - """Internal wrapper method for admin-only API calls""" - context = args[1] - if context.user.is_admin(): - return target(*args, **kwargs) - else: - return {} - - return wrapper - - -class AdminController(object): - """ - API Controller for users, hosts, nodes, and workers. - Trivial admin_only wrapper will be replaced with RBAC, - allowing project managers to administer project users. - """ - - def __str__(self): - return 'AdminController' - - @admin_only - def describe_user(self, _context, name, **_kwargs): - """Returns user data, including access and secret keys.""" - return user_dict(manager.AuthManager().get_user(name)) - - @admin_only - def describe_users(self, _context, **_kwargs): - """Returns all users - should be changed to deal with a list.""" - return {'userSet': - [user_dict(u) for u in manager.AuthManager().get_users()] } - - @admin_only - def register_user(self, _context, name, **_kwargs): - """Creates a new user, and returns generated credentials.""" - return user_dict(manager.AuthManager().create_user(name)) - - @admin_only - def deregister_user(self, _context, name, **_kwargs): - """Deletes a single user (NOT undoable.) - Should throw an exception if the user has instances, - volumes, or buckets remaining. - """ - manager.AuthManager().delete_user(name) - - return True - - @admin_only - def describe_roles(self, context, project_roles=True, **kwargs): - """Returns a list of allowed roles.""" - roles = manager.AuthManager().get_roles(project_roles) - return { 'roles': [{'role': r} for r in roles]} - - @admin_only - def describe_user_roles(self, context, user, project=None, **kwargs): - """Returns a list of roles for the given user. - Omitting project will return any global roles that the user has. - Specifying project will return only project specific roles. - """ - roles = manager.AuthManager().get_user_roles(user, project=project) - return { 'roles': [{'role': r} for r in roles]} - - @admin_only - def modify_user_role(self, context, user, role, project=None, - operation='add', **kwargs): - """Add or remove a role for a user and project.""" - if operation == 'add': - manager.AuthManager().add_role(user, role, project) - elif operation == 'remove': - manager.AuthManager().remove_role(user, role, project) - else: - raise exception.ApiError('operation must be add or remove') - - return True - - @admin_only - def generate_x509_for_user(self, _context, name, project=None, **kwargs): - """Generates and returns an x509 certificate for a single user. - Is usually called from a client that will wrap this with - access and secret key info, and return a zip file. - """ - if project is None: - project = name - project = manager.AuthManager().get_project(project) - user = manager.AuthManager().get_user(name) - return user_dict(user, base64.b64encode(project.get_credentials(user))) - - @admin_only - def describe_project(self, context, name, **kwargs): - """Returns project data, including member ids.""" - return project_dict(manager.AuthManager().get_project(name)) - - @admin_only - def describe_projects(self, context, user=None, **kwargs): - """Returns all projects - should be changed to deal with a list.""" - return {'projectSet': - [project_dict(u) for u in - manager.AuthManager().get_projects(user=user)]} - - @admin_only - def register_project(self, context, name, manager_user, description=None, - member_users=None, **kwargs): - """Creates a new project""" - return project_dict( - manager.AuthManager().create_project( - name, - manager_user, - description=None, - member_users=None)) - - @admin_only - def deregister_project(self, context, name): - """Permanently deletes a project.""" - manager.AuthManager().delete_project(name) - return True - - @admin_only - def describe_project_members(self, context, name, **kwargs): - project = manager.AuthManager().get_project(name) - result = { - 'members': [{'member': m} for m in project.member_ids]} - return result - - @admin_only - def modify_project_member(self, context, user, project, operation, **kwargs): - """Add or remove a user from a project.""" - if operation =='add': - manager.AuthManager().add_to_project(user, project) - elif operation == 'remove': - manager.AuthManager().remove_from_project(user, project) - else: - raise exception.ApiError('operation must be add or remove') - return True - - @admin_only - def describe_hosts(self, _context, **_kwargs): - """Returns status info for all nodes. Includes: - * Disk Space - * Instance List - * RAM used - * CPU used - * DHCP servers running - * Iptables / bridges - """ - return {'hostSet': [host_dict(h) for h in model.Host.all()]} - - @admin_only - def describe_host(self, _context, name, **_kwargs): - """Returns status info for single node.""" - return host_dict(model.Host.lookup(name)) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py deleted file mode 100644 index 30634429d..000000000 --- a/nova/endpoint/cloud.py +++ /dev/null @@ -1,739 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Cloud Controller: Implementation of EC2 REST API calls, which are -dispatched to other nodes via AMQP RPC. State is via distributed -datastore. -""" - -import base64 -import logging -import os -import time - -from twisted.internet import defer - -from nova import datastore -from nova import exception -from nova import flags -from nova import rpc -from nova import utils -from nova.auth import rbac -from nova.auth import manager -from nova.compute import model -from nova.compute.instance_types import INSTANCE_TYPES -from nova.endpoint import images -from nova.network import service as network_service -from nova.network import model as network_model -from nova.volume import service - - -FLAGS = flags.FLAGS -flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') - - -def _gen_key(user_id, key_name): - """ Tuck this into AuthManager """ - try: - mgr = manager.AuthManager() - private_key, fingerprint = mgr.generate_key_pair(user_id, key_name) - except Exception as ex: - return {'exception': ex} - return {'private_key': private_key, 'fingerprint': fingerprint} - - -class CloudController(object): - """ CloudController provides the critical dispatch between - inbound API calls through the endpoint and messages - sent to the other nodes. -""" - def __init__(self): - self.instdir = model.InstanceDirectory() - self.setup() - - @property - def instances(self): - """ All instances in the system, as dicts """ - return self.instdir.all - - @property - def volumes(self): - """ returns a list of all volumes """ - for volume_id in datastore.Redis.instance().smembers("volumes"): - volume = service.get_volume(volume_id) - yield volume - - def __str__(self): - return 'CloudController' - - def setup(self): - """ Ensure the keychains and folders exist. """ - # Create keys folder, if it doesn't exist - if not os.path.exists(FLAGS.keys_path): - os.makedirs(FLAGS.keys_path) - # Gen root CA, if we don't have one - root_ca_path = os.path.join(FLAGS.ca_path, FLAGS.ca_file) - if not os.path.exists(root_ca_path): - start = os.getcwd() - os.chdir(FLAGS.ca_path) - utils.runthis("Generating root CA: %s", "sh genrootca.sh") - os.chdir(start) - # TODO: Do this with M2Crypto instead - - def get_instance_by_ip(self, ip): - return self.instdir.by_ip(ip) - - def _get_mpi_data(self, project_id): - result = {} - for instance in self.instdir.all: - if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], - INSTANCE_TYPES[instance['instance_type']]['vcpus']) - if instance['key_name'] in result: - result[instance['key_name']].append(line) - else: - result[instance['key_name']] = [line] - return result - - def get_metadata(self, ipaddress): - i = self.get_instance_by_ip(ipaddress) - if i is None: - return None - mpi = self._get_mpi_data(i['project_id']) - if i['key_name']: - keys = { - '0': { - '_name': i['key_name'], - 'openssh-key': i['key_data'] - } - } - else: - keys = '' - - address_record = network_model.FixedIp(i['private_dns_name']) - if address_record: - hostname = address_record['hostname'] - else: - hostname = 'ip-%s' % i['private_dns_name'].replace('.', '-') - data = { - 'user-data': base64.b64decode(i['user_data']), - 'meta-data': { - 'ami-id': i['image_id'], - 'ami-launch-index': i['ami_launch_index'], - 'ami-manifest-path': 'FIXME', # image property - 'block-device-mapping': { # TODO: replace with real data - 'ami': 'sda1', - 'ephemeral0': 'sda2', - 'root': '/dev/sda1', - 'swap': 'sda3' - }, - 'hostname': hostname, - 'instance-action': 'none', - 'instance-id': i['instance_id'], - 'instance-type': i.get('instance_type', ''), - 'local-hostname': hostname, - 'local-ipv4': i['private_dns_name'], # TODO: switch to IP - 'kernel-id': i.get('kernel_id', ''), - 'placement': { - 'availaibility-zone': i.get('availability_zone', 'nova'), - }, - 'public-hostname': hostname, - 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP - 'public-keys': keys, - 'ramdisk-id': i.get('ramdisk_id', ''), - 'reservation-id': i['reservation_id'], - 'security-groups': i.get('groups', ''), - 'mpi': mpi - } - } - if False: # TODO: store ancestor ids - data['ancestor-ami-ids'] = [] - if i.get('product_codes', None): - data['product-codes'] = i['product_codes'] - return data - - @rbac.allow('all') - def describe_availability_zones(self, context, **kwargs): - return {'availabilityZoneInfo': [{'zoneName': 'nova', - 'zoneState': 'available'}]} - - @rbac.allow('all') - def describe_regions(self, context, region_name=None, **kwargs): - # TODO(vish): region_name is an array. Support filtering - return {'regionInfo': [{'regionName': 'nova', - 'regionUrl': FLAGS.ec2_url}]} - - @rbac.allow('all') - def describe_snapshots(self, - context, - snapshot_id=None, - owner=None, - restorable_by=None, - **kwargs): - return {'snapshotSet': [{'snapshotId': 'fixme', - 'volumeId': 'fixme', - 'status': 'fixme', - 'startTime': 'fixme', - 'progress': 'fixme', - 'ownerId': 'fixme', - 'volumeSize': 0, - 'description': 'fixme'}]} - - @rbac.allow('all') - def describe_key_pairs(self, context, key_name=None, **kwargs): - key_pairs = context.user.get_key_pairs() - if not key_name is None: - key_pairs = [x for x in key_pairs if x.name in key_name] - - result = [] - for key_pair in key_pairs: - # filter out the vpn keys - suffix = FLAGS.vpn_key_suffix - if context.user.is_admin() or not key_pair.name.endswith(suffix): - result.append({ - 'keyName': key_pair.name, - 'keyFingerprint': key_pair.fingerprint, - }) - - return {'keypairsSet': result} - - @rbac.allow('all') - def create_key_pair(self, context, key_name, **kwargs): - dcall = defer.Deferred() - pool = context.handler.application.settings.get('pool') - def _complete(kwargs): - if 'exception' in kwargs: - dcall.errback(kwargs['exception']) - return - dcall.callback({'keyName': key_name, - 'keyFingerprint': kwargs['fingerprint'], - 'keyMaterial': kwargs['private_key']}) - pool.apply_async(_gen_key, [context.user.id, key_name], - callback=_complete) - return dcall - - @rbac.allow('all') - def delete_key_pair(self, context, key_name, **kwargs): - context.user.delete_key_pair(key_name) - # aws returns true even if the key doens't exist - return True - - @rbac.allow('all') - def describe_security_groups(self, context, group_names, **kwargs): - groups = {'securityGroupSet': []} - - # Stubbed for now to unblock other things. - return groups - - @rbac.allow('netadmin') - def create_security_group(self, context, group_name, **kwargs): - return True - - @rbac.allow('netadmin') - def delete_security_group(self, context, group_name, **kwargs): - return True - - @rbac.allow('projectmanager', 'sysadmin') - def get_console_output(self, context, instance_id, **kwargs): - # instance_id is passed in as a list of instances - instance = self._get_instance(context, instance_id[0]) - return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "get_console_output", - "args": {"instance_id": instance_id[0]}}) - - def _get_user_id(self, context): - if context and context.user: - return context.user.id - else: - return None - - @rbac.allow('projectmanager', 'sysadmin') - def describe_volumes(self, context, **kwargs): - volumes = [] - for volume in self.volumes: - if context.user.is_admin() or volume['project_id'] == context.project.id: - v = self.format_volume(context, volume) - volumes.append(v) - return defer.succeed({'volumeSet': volumes}) - - def format_volume(self, context, volume): - v = {} - v['volumeId'] = volume['volume_id'] - v['status'] = volume['status'] - v['size'] = volume['size'] - v['availabilityZone'] = volume['availability_zone'] - v['createTime'] = volume['create_time'] - if context.user.is_admin(): - v['status'] = '%s (%s, %s, %s, %s)' % ( - volume.get('status', None), - volume.get('user_id', None), - volume.get('node_name', None), - volume.get('instance_id', ''), - volume.get('mountpoint', '')) - if volume['attach_status'] == 'attached': - v['attachmentSet'] = [{'attachTime': volume['attach_time'], - 'deleteOnTermination': volume['delete_on_termination'], - 'device': volume['mountpoint'], - 'instanceId': volume['instance_id'], - 'status': 'attached', - 'volume_id': volume['volume_id']}] - else: - v['attachmentSet'] = [{}] - return v - - @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks - def create_volume(self, context, size, **kwargs): - # TODO(vish): refactor this to create the volume object here and tell service to create it - result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", - "args": {"size": size, - "user_id": context.user.id, - "project_id": context.project.id}}) - # NOTE(vish): rpc returned value is in the result key in the dictionary - volume = self._get_volume(context, result) - defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) - - def _get_address(self, context, public_ip): - # FIXME(vish) this should move into network.py - address = network_model.ElasticIp.lookup(public_ip) - if address and (context.user.is_admin() or address['project_id'] == context.project.id): - return address - raise exception.NotFound("Address at ip %s not found" % public_ip) - - def _get_image(self, context, image_id): - """passes in context because - objectstore does its own authorization""" - result = images.list(context, [image_id]) - if not result: - raise exception.NotFound('Image %s could not be found' % image_id) - image = result[0] - return image - - def _get_instance(self, context, instance_id): - for instance in self.instdir.all: - if instance['instance_id'] == instance_id: - if context.user.is_admin() or instance['project_id'] == context.project.id: - return instance - raise exception.NotFound('Instance %s could not be found' % instance_id) - - def _get_volume(self, context, volume_id): - volume = service.get_volume(volume_id) - if context.user.is_admin() or volume['project_id'] == context.project.id: - return volume - raise exception.NotFound('Volume %s could not be found' % volume_id) - - @rbac.allow('projectmanager', 'sysadmin') - def attach_volume(self, context, volume_id, instance_id, device, **kwargs): - volume = self._get_volume(context, volume_id) - if volume['status'] == "attached": - raise exception.ApiError("Volume is already attached") - # TODO(vish): looping through all volumes is slow. We should probably maintain an index - for vol in self.volumes: - if vol['instance_id'] == instance_id and vol['mountpoint'] == device: - raise exception.ApiError("Volume %s is already attached to %s" % (vol['volume_id'], vol['mountpoint'])) - volume.start_attach(instance_id, device) - instance = self._get_instance(context, instance_id) - compute_node = instance['node_name'] - rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), - {"method": "attach_volume", - "args": {"volume_id": volume_id, - "instance_id": instance_id, - "mountpoint": device}}) - return defer.succeed({'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': instance_id, - 'requestId': context.request_id, - 'status': volume['attach_status'], - 'volumeId': volume_id}) - - @rbac.allow('projectmanager', 'sysadmin') - def detach_volume(self, context, volume_id, **kwargs): - volume = self._get_volume(context, volume_id) - instance_id = volume.get('instance_id', None) - if not instance_id: - raise exception.Error("Volume isn't attached to anything!") - if volume['status'] == "available": - raise exception.Error("Volume is already detached") - try: - volume.start_detach() - instance = self._get_instance(context, instance_id) - rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "detach_volume", - "args": {"instance_id": instance_id, - "volume_id": volume_id}}) - except exception.NotFound: - # If the instance doesn't exist anymore, - # then we need to call detach blind - volume.finish_detach() - return defer.succeed({'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': instance_id, - 'requestId': context.request_id, - 'status': volume['attach_status'], - 'volumeId': volume_id}) - - def _convert_to_set(self, lst, label): - if lst == None or lst == []: - return None - if not isinstance(lst, list): - lst = [lst] - return [{label: x} for x in lst] - - @rbac.allow('all') - def describe_instances(self, context, **kwargs): - return defer.succeed(self._format_describe_instances(context)) - - def _format_describe_instances(self, context): - return { 'reservationSet': self._format_instances(context) } - - def _format_run_instances(self, context, reservation_id): - i = self._format_instances(context, reservation_id) - assert len(i) == 1 - return i[0] - - def _format_instances(self, context, reservation_id = None): - reservations = {} - if context.user.is_admin(): - instgenerator = self.instdir.all - else: - instgenerator = self.instdir.by_project(context.project.id) - for instance in instgenerator: - res_id = instance.get('reservation_id', 'Unknown') - if reservation_id != None and reservation_id != res_id: - continue - if not context.user.is_admin(): - if instance['image_id'] == FLAGS.vpn_image_id: - continue - i = {} - i['instance_id'] = instance.get('instance_id', None) - i['image_id'] = instance.get('image_id', None) - i['instance_state'] = { - 'code': instance.get('state', 0), - 'name': instance.get('state_description', 'pending') - } - i['public_dns_name'] = network_model.get_public_ip_for_instance( - i['instance_id']) - i['private_dns_name'] = instance.get('private_dns_name', None) - if not i['public_dns_name']: - i['public_dns_name'] = i['private_dns_name'] - i['dns_name'] = instance.get('dns_name', None) - i['key_name'] = instance.get('key_name', None) - if context.user.is_admin(): - i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), - instance.get('node_name', '')) - i['product_codes_set'] = self._convert_to_set( - instance.get('product_codes', None), 'product_code') - i['instance_type'] = instance.get('instance_type', None) - i['launch_time'] = instance.get('launch_time', None) - i['ami_launch_index'] = instance.get('ami_launch_index', - None) - if not reservations.has_key(res_id): - r = {} - r['reservation_id'] = res_id - r['owner_id'] = instance.get('project_id', None) - r['group_set'] = self._convert_to_set( - instance.get('groups', None), 'group_id') - r['instances_set'] = [] - reservations[res_id] = r - reservations[res_id]['instances_set'].append(i) - - return list(reservations.values()) - - @rbac.allow('all') - def describe_addresses(self, context, **kwargs): - return self.format_addresses(context) - - def format_addresses(self, context): - addresses = [] - for address in network_model.ElasticIp.all(): - # TODO(vish): implement a by_project iterator for addresses - if (context.user.is_admin() or - address['project_id'] == context.project.id): - address_rv = { - 'public_ip': address['address'], - 'instance_id': address.get('instance_id', 'free') - } - if context.user.is_admin(): - address_rv['instance_id'] = "%s (%s, %s)" % ( - address['instance_id'], - address['user_id'], - address['project_id'], - ) - addresses.append(address_rv) - return {'addressesSet': addresses} - - @rbac.allow('netadmin') - @defer.inlineCallbacks - def allocate_address(self, context, **kwargs): - network_topic = yield self._get_network_topic(context) - public_ip = yield rpc.call(network_topic, - {"method": "allocate_elastic_ip", - "args": {"user_id": context.user.id, - "project_id": context.project.id}}) - defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) - - @rbac.allow('netadmin') - @defer.inlineCallbacks - def release_address(self, context, public_ip, **kwargs): - # NOTE(vish): Should we make sure this works? - network_topic = yield self._get_network_topic(context) - rpc.cast(network_topic, - {"method": "deallocate_elastic_ip", - "args": {"elastic_ip": public_ip}}) - defer.returnValue({'releaseResponse': ["Address released."]}) - - @rbac.allow('netadmin') - @defer.inlineCallbacks - def associate_address(self, context, instance_id, public_ip, **kwargs): - instance = self._get_instance(context, instance_id) - address = self._get_address(context, public_ip) - network_topic = yield self._get_network_topic(context) - rpc.cast(network_topic, - {"method": "associate_elastic_ip", - "args": {"elastic_ip": address['address'], - "fixed_ip": instance['private_dns_name'], - "instance_id": instance['instance_id']}}) - defer.returnValue({'associateResponse': ["Address associated."]}) - - @rbac.allow('netadmin') - @defer.inlineCallbacks - def disassociate_address(self, context, public_ip, **kwargs): - address = self._get_address(context, public_ip) - network_topic = yield self._get_network_topic(context) - rpc.cast(network_topic, - {"method": "disassociate_elastic_ip", - "args": {"elastic_ip": address['address']}}) - defer.returnValue({'disassociateResponse': ["Address disassociated."]}) - - @defer.inlineCallbacks - def _get_network_topic(self, context): - """Retrieves the network host for a project""" - host = network_service.get_host_for_project(context.project.id) - if not host: - host = yield rpc.call(FLAGS.network_topic, - {"method": "set_network_host", - "args": {"user_id": context.user.id, - "project_id": context.project.id}}) - defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) - - @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks - def run_instances(self, context, **kwargs): - # make sure user can access the image - # vpn image is private so it doesn't show up on lists - if kwargs['image_id'] != FLAGS.vpn_image_id: - image = self._get_image(context, kwargs['image_id']) - - # FIXME(ja): if image is cloudpipe, this breaks - - # get defaults from imagestore - image_id = image['imageId'] - kernel_id = image.get('kernelId', FLAGS.default_kernel) - ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) - - # API parameters overrides of defaults - kernel_id = kwargs.get('kernel_id', kernel_id) - ramdisk_id = kwargs.get('ramdisk_id', ramdisk_id) - - # make sure we have access to kernel and ramdisk - self._get_image(context, kernel_id) - self._get_image(context, ramdisk_id) - - logging.debug("Going to run instances...") - reservation_id = utils.generate_uid('r') - launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) - key_data = None - if kwargs.has_key('key_name'): - key_pair = context.user.get_key_pair(kwargs['key_name']) - if not key_pair: - raise exception.ApiError('Key Pair %s not found' % - kwargs['key_name']) - key_data = key_pair.public_key - network_topic = yield self._get_network_topic(context) - # TODO: Get the real security group of launch in here - security_group = "default" - for num in range(int(kwargs['max_count'])): - is_vpn = False - if image_id == FLAGS.vpn_image_id: - is_vpn = True - inst = self.instdir.new() - allocate_data = yield rpc.call(network_topic, - {"method": "allocate_fixed_ip", - "args": {"user_id": context.user.id, - "project_id": context.project.id, - "security_group": security_group, - "is_vpn": is_vpn, - "hostname": inst.instance_id}}) - inst['image_id'] = image_id - inst['kernel_id'] = kernel_id - inst['ramdisk_id'] = ramdisk_id - inst['user_data'] = kwargs.get('user_data', '') - inst['instance_type'] = kwargs.get('instance_type', 'm1.small') - inst['reservation_id'] = reservation_id - inst['launch_time'] = launch_time - inst['key_data'] = key_data or '' - inst['key_name'] = kwargs.get('key_name', '') - inst['user_id'] = context.user.id - inst['project_id'] = context.project.id - inst['ami_launch_index'] = num - inst['security_group'] = security_group - inst['hostname'] = inst.instance_id - for (key, value) in allocate_data.iteritems(): - inst[key] = value - - inst.save() - rpc.cast(FLAGS.compute_topic, - {"method": "run_instance", - "args": {"instance_id": inst.instance_id}}) - logging.debug("Casting to node for %s's instance with IP of %s" % - (context.user.name, inst['private_dns_name'])) - # TODO: Make Network figure out the network name from ip. - defer.returnValue(self._format_run_instances(context, reservation_id)) - - @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks - def terminate_instances(self, context, instance_id, **kwargs): - logging.debug("Going to start terminating instances") - network_topic = yield self._get_network_topic(context) - for i in instance_id: - logging.debug("Going to try and terminate %s" % i) - try: - instance = self._get_instance(context, i) - except exception.NotFound: - logging.warning("Instance %s was not found during terminate" - % i) - continue - elastic_ip = network_model.get_public_ip_for_instance(i) - if elastic_ip: - logging.debug("Disassociating address %s" % elastic_ip) - # NOTE(vish): Right now we don't really care if the ip is - # disassociated. We may need to worry about - # checking this later. Perhaps in the scheduler? - rpc.cast(network_topic, - {"method": "disassociate_elastic_ip", - "args": {"elastic_ip": elastic_ip}}) - - fixed_ip = instance.get('private_dns_name', None) - if fixed_ip: - logging.debug("Deallocating address %s" % fixed_ip) - # NOTE(vish): Right now we don't really care if the ip is - # actually removed. We may need to worry about - # checking this later. Perhaps in the scheduler? - rpc.cast(network_topic, - {"method": "deallocate_fixed_ip", - "args": {"fixed_ip": fixed_ip}}) - - if instance.get('node_name', 'unassigned') != 'unassigned': - # NOTE(joshua?): It's also internal default - rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "terminate_instance", - "args": {"instance_id": i}}) - else: - instance.destroy() - defer.returnValue(True) - - @rbac.allow('projectmanager', 'sysadmin') - def reboot_instances(self, context, instance_id, **kwargs): - """instance_id is a list of instance ids""" - for i in instance_id: - instance = self._get_instance(context, i) - rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), - {"method": "reboot_instance", - "args": {"instance_id": i}}) - return defer.succeed(True) - - @rbac.allow('projectmanager', 'sysadmin') - def delete_volume(self, context, volume_id, **kwargs): - # TODO: return error if not authorized - volume = self._get_volume(context, volume_id) - volume_node = volume['node_name'] - rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), - {"method": "delete_volume", - "args": {"volume_id": volume_id}}) - return defer.succeed(True) - - @rbac.allow('all') - def describe_images(self, context, image_id=None, **kwargs): - # The objectstore does its own authorization for describe - imageSet = images.list(context, image_id) - return defer.succeed({'imagesSet': imageSet}) - - @rbac.allow('projectmanager', 'sysadmin') - def deregister_image(self, context, image_id, **kwargs): - # FIXME: should the objectstore be doing these authorization checks? - images.deregister(context, image_id) - return defer.succeed({'imageId': image_id}) - - @rbac.allow('projectmanager', 'sysadmin') - def register_image(self, context, image_location=None, **kwargs): - # FIXME: should the objectstore be doing these authorization checks? - if image_location is None and kwargs.has_key('name'): - image_location = kwargs['name'] - image_id = images.register(context, image_location) - logging.debug("Registered %s as %s" % (image_location, image_id)) - - return defer.succeed({'imageId': image_id}) - - @rbac.allow('all') - def describe_image_attribute(self, context, image_id, attribute, **kwargs): - if attribute != 'launchPermission': - raise exception.ApiError('attribute not supported: %s' % attribute) - try: - image = images.list(context, image_id)[0] - except IndexError: - raise exception.ApiError('invalid id: %s' % image_id) - result = {'image_id': image_id, 'launchPermission': []} - if image['isPublic']: - result['launchPermission'].append({'group': 'all'}) - return defer.succeed(result) - - @rbac.allow('projectmanager', 'sysadmin') - def modify_image_attribute(self, context, image_id, attribute, operation_type, **kwargs): - # TODO(devcamcar): Support users and groups other than 'all'. - if attribute != 'launchPermission': - raise exception.ApiError('attribute not supported: %s' % attribute) - if not 'user_group' in kwargs: - raise exception.ApiError('user or group not specified') - if len(kwargs['user_group']) != 1 and kwargs['user_group'][0] != 'all': - raise exception.ApiError('only group "all" is supported') - if not operation_type in ['add', 'remove']: - raise exception.ApiError('operation_type must be add or remove') - result = images.modify(context, image_id, operation_type) - return defer.succeed(result) - - def update_state(self, topic, value): - """ accepts status reports from the queue and consolidates them """ - # TODO(jmc): if an instance has disappeared from - # the node, call instance_death - if topic == "instances": - return defer.succeed(True) - aggregate_state = getattr(self, topic) - node_name = value.keys()[0] - items = value[node_name] - - logging.debug("Updating %s state for %s" % (topic, node_name)) - - for item_id in items.keys(): - if (aggregate_state.has_key('pending') and - aggregate_state['pending'].has_key(item_id)): - del aggregate_state['pending'][item_id] - aggregate_state[node_name] = items - - return defer.succeed(True) -- cgit From 4f3bb96df8a7e48735c078520e77a47dca7a2bd1 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 31 Aug 2010 12:33:49 -0400 Subject: Remove inlineCallbacks and yield from cloud.py, as eventlet doesn't need it --- nova/api/ec2/__init__.py | 1 + nova/api/ec2/cloud.py | 30 ++++++++++-------------------- nova/endpoint/notes.txt | 16 ++++++++-------- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 7e345d297..b4a1894cc 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -108,6 +108,7 @@ class Router(wsgi.Application): self.controllers = dict(Cloud=cloud.CloudController(), Admin=admin.AdminController()) + @webob.dec.wsgify def __call__(self, req): # Obtain the appropriate controller for this request. match = self.map.match(req.path) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 30634429d..decd2a2c0 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -27,8 +27,6 @@ import logging import os import time -from twisted.internet import defer - from nova import datastore from nova import exception from nova import flags @@ -298,10 +296,9 @@ class CloudController(object): return v @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it - result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", + result = rpc.call(FLAGS.volume_topic, {"method": "create_volume", "args": {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) @@ -480,31 +477,28 @@ class CloudController(object): return {'addressesSet': addresses} @rbac.allow('netadmin') - @defer.inlineCallbacks def allocate_address(self, context, **kwargs): - network_topic = yield self._get_network_topic(context) - public_ip = yield rpc.call(network_topic, + network_topic = self._get_network_topic(context) + public_ip = rpc.call(network_topic, {"method": "allocate_elastic_ip", "args": {"user_id": context.user.id, "project_id": context.project.id}}) defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') - @defer.inlineCallbacks def release_address(self, context, public_ip, **kwargs): # NOTE(vish): Should we make sure this works? - network_topic = yield self._get_network_topic(context) + network_topic = self._get_network_topic(context) rpc.cast(network_topic, {"method": "deallocate_elastic_ip", "args": {"elastic_ip": public_ip}}) defer.returnValue({'releaseResponse': ["Address released."]}) @rbac.allow('netadmin') - @defer.inlineCallbacks def associate_address(self, context, instance_id, public_ip, **kwargs): instance = self._get_instance(context, instance_id) address = self._get_address(context, public_ip) - network_topic = yield self._get_network_topic(context) + network_topic = self._get_network_topic(context) rpc.cast(network_topic, {"method": "associate_elastic_ip", "args": {"elastic_ip": address['address'], @@ -513,28 +507,25 @@ class CloudController(object): defer.returnValue({'associateResponse': ["Address associated."]}) @rbac.allow('netadmin') - @defer.inlineCallbacks def disassociate_address(self, context, public_ip, **kwargs): address = self._get_address(context, public_ip) - network_topic = yield self._get_network_topic(context) + network_topic = self._get_network_topic(context) rpc.cast(network_topic, {"method": "disassociate_elastic_ip", "args": {"elastic_ip": address['address']}}) defer.returnValue({'disassociateResponse': ["Address disassociated."]}) - @defer.inlineCallbacks def _get_network_topic(self, context): """Retrieves the network host for a project""" host = network_service.get_host_for_project(context.project.id) if not host: - host = yield rpc.call(FLAGS.network_topic, + host = rpc.call(FLAGS.network_topic, {"method": "set_network_host", "args": {"user_id": context.user.id, "project_id": context.project.id}}) defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks def run_instances(self, context, **kwargs): # make sure user can access the image # vpn image is private so it doesn't show up on lists @@ -566,7 +557,7 @@ class CloudController(object): raise exception.ApiError('Key Pair %s not found' % kwargs['key_name']) key_data = key_pair.public_key - network_topic = yield self._get_network_topic(context) + network_topic = self._get_network_topic(context) # TODO: Get the real security group of launch in here security_group = "default" for num in range(int(kwargs['max_count'])): @@ -574,7 +565,7 @@ class CloudController(object): if image_id == FLAGS.vpn_image_id: is_vpn = True inst = self.instdir.new() - allocate_data = yield rpc.call(network_topic, + allocate_data = rpc.call(network_topic, {"method": "allocate_fixed_ip", "args": {"user_id": context.user.id, "project_id": context.project.id, @@ -608,10 +599,9 @@ class CloudController(object): defer.returnValue(self._format_run_instances(context, reservation_id)) @rbac.allow('projectmanager', 'sysadmin') - @defer.inlineCallbacks def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") - network_topic = yield self._get_network_topic(context) + network_topic = self._get_network_topic(context) for i in instance_id: logging.debug("Going to try and terminate %s" % i) try: diff --git a/nova/endpoint/notes.txt b/nova/endpoint/notes.txt index 7a85cdc93..cbb7b0cd0 100644 --- a/nova/endpoint/notes.txt +++ b/nova/endpoint/notes.txt @@ -43,20 +43,20 @@ CloudController and AdminController: * Controllers: move the @rbac.allow data into an auth WSGI that is right above the call to the controller - verify @defer.inlineCallbacks is just to allow the yield statements, then - remove the yield statements (untangle from twisted) +x verify @defer.inlineCallbacks is just to allow the yield statements, then +x remove the yield statements (untangle from twisted) * nova-api: verify that cloud_topic is going away which I seem to remember, so we can ignore rpc * apiserverapplication: - replace with a Router to a wsgi.Controller - apirequesthandler stuff is just an entry in api.APIRouter +x replace with a Router to a wsgi.Controller +x apirequesthandler stuff is just an entry in api.APIRouter * apirequesthandler - wsgi.Controller pointed to by api.APIRouter - - basically it's execute() from old APIRequestHandler - change to return data directly instead of _write_callback() and finish() +x wsgi.Controller pointed to by api.APIRouter +x - basically it's execute() from old APIRequestHandler +x change to return data directly instead of _write_callback() and finish() * apirequest - doesn't need to change +x doesn't need to change -- cgit From f22c693e4cf638ef5278d9db444da2c4a99baae4 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 31 Aug 2010 13:25:31 -0400 Subject: Remove all Twisted defer references from cloud.py --- nova/api/ec2/cloud.py | 86 ++++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 49 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index decd2a2c0..05fbf3861 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -48,11 +48,8 @@ flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') def _gen_key(user_id, key_name): """ Tuck this into AuthManager """ - try: - mgr = manager.AuthManager() - private_key, fingerprint = mgr.generate_key_pair(user_id, key_name) - except Exception as ex: - return {'exception': ex} + mgr = manager.AuthManager() + private_key, fingerprint = mgr.generate_key_pair(user_id, key_name) return {'private_key': private_key, 'fingerprint': fingerprint} @@ -213,18 +210,10 @@ class CloudController(object): @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): - dcall = defer.Deferred() - pool = context.handler.application.settings.get('pool') - def _complete(kwargs): - if 'exception' in kwargs: - dcall.errback(kwargs['exception']) - return - dcall.callback({'keyName': key_name, - 'keyFingerprint': kwargs['fingerprint'], - 'keyMaterial': kwargs['private_key']}) - pool.apply_async(_gen_key, [context.user.id, key_name], - callback=_complete) - return dcall + data = _gen_key(context.user.id, key_name) + return {'keyName': key_name, + 'keyFingerprint': data['fingerprint'], + 'keyMaterial': data['private_key']} @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): @@ -268,7 +257,7 @@ class CloudController(object): if context.user.is_admin() or volume['project_id'] == context.project.id: v = self.format_volume(context, volume) volumes.append(v) - return defer.succeed({'volumeSet': volumes}) + return {'volumeSet': volumes} def format_volume(self, context, volume): v = {} @@ -304,7 +293,7 @@ class CloudController(object): "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary volume = self._get_volume(context, result) - defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) + return {'volumeSet': [self.format_volume(context, volume)]} def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py @@ -352,12 +341,12 @@ class CloudController(object): "args": {"volume_id": volume_id, "instance_id": instance_id, "mountpoint": device}}) - return defer.succeed({'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': instance_id, - 'requestId': context.request_id, - 'status': volume['attach_status'], - 'volumeId': volume_id}) + return {'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id} @rbac.allow('projectmanager', 'sysadmin') def detach_volume(self, context, volume_id, **kwargs): @@ -378,12 +367,12 @@ class CloudController(object): # If the instance doesn't exist anymore, # then we need to call detach blind volume.finish_detach() - return defer.succeed({'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': instance_id, - 'requestId': context.request_id, - 'status': volume['attach_status'], - 'volumeId': volume_id}) + return {'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -394,7 +383,7 @@ class CloudController(object): @rbac.allow('all') def describe_instances(self, context, **kwargs): - return defer.succeed(self._format_describe_instances(context)) + return self._format_describe_instances(context) def _format_describe_instances(self, context): return { 'reservationSet': self._format_instances(context) } @@ -483,7 +472,7 @@ class CloudController(object): {"method": "allocate_elastic_ip", "args": {"user_id": context.user.id, "project_id": context.project.id}}) - defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) + return {'addressSet': [{'publicIp': public_ip}]} @rbac.allow('netadmin') def release_address(self, context, public_ip, **kwargs): @@ -492,7 +481,7 @@ class CloudController(object): rpc.cast(network_topic, {"method": "deallocate_elastic_ip", "args": {"elastic_ip": public_ip}}) - defer.returnValue({'releaseResponse': ["Address released."]}) + return {'releaseResponse': ["Address released."]} @rbac.allow('netadmin') def associate_address(self, context, instance_id, public_ip, **kwargs): @@ -504,7 +493,7 @@ class CloudController(object): "args": {"elastic_ip": address['address'], "fixed_ip": instance['private_dns_name'], "instance_id": instance['instance_id']}}) - defer.returnValue({'associateResponse': ["Address associated."]}) + return {'associateResponse': ["Address associated."]} @rbac.allow('netadmin') def disassociate_address(self, context, public_ip, **kwargs): @@ -513,7 +502,7 @@ class CloudController(object): rpc.cast(network_topic, {"method": "disassociate_elastic_ip", "args": {"elastic_ip": address['address']}}) - defer.returnValue({'disassociateResponse': ["Address disassociated."]}) + return {'disassociateResponse': ["Address disassociated."]} def _get_network_topic(self, context): """Retrieves the network host for a project""" @@ -523,7 +512,7 @@ class CloudController(object): {"method": "set_network_host", "args": {"user_id": context.user.id, "project_id": context.project.id}}) - defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) + return '%s.%s' %(FLAGS.network_topic, host) @rbac.allow('projectmanager', 'sysadmin') def run_instances(self, context, **kwargs): @@ -596,7 +585,7 @@ class CloudController(object): logging.debug("Casting to node for %s's instance with IP of %s" % (context.user.name, inst['private_dns_name'])) # TODO: Make Network figure out the network name from ip. - defer.returnValue(self._format_run_instances(context, reservation_id)) + return self._format_run_instances(context, reservation_id) @rbac.allow('projectmanager', 'sysadmin') def terminate_instances(self, context, instance_id, **kwargs): @@ -637,7 +626,7 @@ class CloudController(object): "args": {"instance_id": i}}) else: instance.destroy() - defer.returnValue(True) + return True @rbac.allow('projectmanager', 'sysadmin') def reboot_instances(self, context, instance_id, **kwargs): @@ -647,7 +636,7 @@ class CloudController(object): rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "reboot_instance", "args": {"instance_id": i}}) - return defer.succeed(True) + return True @rbac.allow('projectmanager', 'sysadmin') def delete_volume(self, context, volume_id, **kwargs): @@ -657,19 +646,19 @@ class CloudController(object): rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), {"method": "delete_volume", "args": {"volume_id": volume_id}}) - return defer.succeed(True) + return True @rbac.allow('all') def describe_images(self, context, image_id=None, **kwargs): # The objectstore does its own authorization for describe imageSet = images.list(context, image_id) - return defer.succeed({'imagesSet': imageSet}) + return {'imagesSet': imageSet} @rbac.allow('projectmanager', 'sysadmin') def deregister_image(self, context, image_id, **kwargs): # FIXME: should the objectstore be doing these authorization checks? images.deregister(context, image_id) - return defer.succeed({'imageId': image_id}) + return {'imageId': image_id} @rbac.allow('projectmanager', 'sysadmin') def register_image(self, context, image_location=None, **kwargs): @@ -679,7 +668,7 @@ class CloudController(object): image_id = images.register(context, image_location) logging.debug("Registered %s as %s" % (image_location, image_id)) - return defer.succeed({'imageId': image_id}) + return {'imageId': image_id} @rbac.allow('all') def describe_image_attribute(self, context, image_id, attribute, **kwargs): @@ -692,7 +681,7 @@ class CloudController(object): result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: result['launchPermission'].append({'group': 'all'}) - return defer.succeed(result) + return result @rbac.allow('projectmanager', 'sysadmin') def modify_image_attribute(self, context, image_id, attribute, operation_type, **kwargs): @@ -705,15 +694,14 @@ class CloudController(object): raise exception.ApiError('only group "all" is supported') if not operation_type in ['add', 'remove']: raise exception.ApiError('operation_type must be add or remove') - result = images.modify(context, image_id, operation_type) - return defer.succeed(result) + return images.modify(context, image_id, operation_type) def update_state(self, topic, value): """ accepts status reports from the queue and consolidates them """ # TODO(jmc): if an instance has disappeared from # the node, call instance_death if topic == "instances": - return defer.succeed(True) + return True aggregate_state = getattr(self, topic) node_name = value.keys()[0] items = value[node_name] @@ -726,4 +714,4 @@ class CloudController(object): del aggregate_state['pending'][item_id] aggregate_state[node_name] = items - return defer.succeed(True) + return True -- cgit From 544b73d35895ac79af910a40590095780f224abb Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Sep 2010 10:50:31 -0400 Subject: Return error Responses properly, and don't muck with req.params -- make a copy instead --- nova/api/ec2/__init__.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index b4a1894cc..248a66f55 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -22,6 +22,7 @@ Starting point for routing EC2 requests import logging import routes +import webob import webob.dec import webob.exc @@ -46,8 +47,7 @@ class Authenticate(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): - #TODO(gundlach): where do arguments come from? - args = self.request.arguments + args = dict(req.params) # Read request signature. try: @@ -92,8 +92,9 @@ class Authenticate(wsgi.Middleware): _log.debug('arg: %s\t\tval: %s' % (key, value)) # Authenticated! - req.environ['ec2.action'] = action req.environ['ec2.context'] = APIRequestContext(user, project) + req.environ['ec2.action'] = action + req.environ['ec2.action_args'] = args return self.application @@ -119,24 +120,24 @@ class Router(wsgi.Application): try: controller = self.controllers[controller_name] except KeyError: - self._error('unhandled', 'no controller named %s' % controller_name) - return + return self._error('unhandled', 'no controller named %s' % controller_name) api_request = APIRequest(controller, req.environ['ec2.action']) context = req.environ['ec2.context'] try: - return api_request.send(context, **args) + return api_request.send(context, **req.environ['ec2.action_args']) except exception.ApiError as ex: - self._error(req, type(ex).__name__ + "." + ex.code, ex.message) + return self._error(req, type(ex).__name__ + "." + ex.code, ex.message) # TODO(vish): do something more useful with unknown exceptions except Exception as ex: - self._error(type(ex).__name__, str(ex)) + return self._error(type(ex).__name__, str(ex)) def _error(self, req, code, message): - req.status = 400 - req.headers['Content-Type'] = 'text/xml' - req.response = ('\n' + resp = webob.Response() + resp.status = 400 + resp.headers['Content-Type'] = 'text/xml' + resp.body = ('\n' '%s' '%s' '?') % (code, message)) - + return resp -- cgit From 8de182446993ac24e7b8fba12342f8adb3e179d4 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Sep 2010 12:02:14 -0400 Subject: Break Router() into Router() and Executor(), and put Authorizer() (currently a stub) in between them. --- nova/api/ec2/__init__.py | 101 +++++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 39 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 248a66f55..87a72ca7c 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -40,36 +40,23 @@ class API(wsgi.Middleware): """Routing for all EC2 API requests.""" def __init__(self): - self.application = Authenticate(Router()) + self.application = Authenticate(Router(Authorizer(Executor()))) class Authenticate(wsgi.Middleware): - """Authenticates an EC2 request.""" + """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" @webob.dec.wsgify def __call__(self, req): - args = dict(req.params) - - # Read request signature. + # Read request signature and access id. try: - signature = args.pop('Signature')[0] + signature = req.params['Signature'] + access = req.params['AWSAccessKeyId'] except: raise webob.exc.HTTPBadRequest() # Make a copy of args for authentication and signature verification. - auth_params = {} - for key, value in args.items(): - auth_params[key] = value[0] - - # Get requested action and remove authentication args for final request. - try: - action = args.pop('Action')[0] - access = args.pop('AWSAccessKeyId')[0] - args.pop('SignatureMethod') - args.pop('SignatureVersion') - args.pop('Version') - args.pop('Timestamp') - except: - raise webob.exc.HTTPBadRequest() + auth_params = dict(req.params) + auth_params.pop('Signature') # not part of authentication args # Authenticate the request. try: @@ -86,22 +73,15 @@ class Authenticate(wsgi.Middleware): logging.debug("Authentication Failure: %s" % ex) raise webob.exc.HTTPForbidden() - _log.debug('action: %s' % action) - - for key, value in args.items(): - _log.debug('arg: %s\t\tval: %s' % (key, value)) - # Authenticated! req.environ['ec2.context'] = APIRequestContext(user, project) - req.environ['ec2.action'] = action - req.environ['ec2.action_args'] = args + return self.application class Router(wsgi.Application): """ - Finds controller for a request, executes environ['ec2.action'] upon it, and - returns an XML response. If the action fails, returns a 400. + Add 'ec2.controller', 'ec2.action', and 'ec2.action_args' to WSGI environ. """ def __init__(self): self.map = routes.Mapper() @@ -111,21 +91,63 @@ class Router(wsgi.Application): @webob.dec.wsgify def __call__(self, req): - # Obtain the appropriate controller for this request. - match = self.map.match(req.path) - if not match: - raise webob.exc.HTTPNotFound() - controller_name = match['controller_name'] - + # Obtain the appropriate controller and action for this request. try: + match = self.map.match(req.path) + controller_name = match['controller_name'] controller = self.controllers[controller_name] - except KeyError: - return self._error('unhandled', 'no controller named %s' % controller_name) + except: + raise webob.exc.HTTPNotFound() + non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', + 'SignatureVersion', 'Version', 'Timestamp'] + args = dict(req.params) + try: + action = req.params['Action'] # raise KeyError if omitted + for non_arg in non_args: + args.pop(non_arg) # remove, but raise KeyError if omitted + except: + raise webob.exc.HTTPBadRequest() - api_request = APIRequest(controller, req.environ['ec2.action']) + _log.debug('action: %s' % action) + for key, value in args.items(): + _log.debug('arg: %s\t\tval: %s' % (key, value)) + + # Success! + req.environ['ec2.controller'] = controller + req.environ['ec2.action'] = action + req.environ['ec2.action_args'] = args + + return self.application + + +class Authorization(wsgi.Middleware): + """ + Verify that ec2.controller and ec2.action in WSGI environ may be executed + in ec2.context. + """ + + @webob.dec.wsgify + def __call__(self, req): + #TODO(gundlach): put rbac information here. + return self.application + + +class Executor(wsg.Application): + """ + Executes 'ec2.action' upon 'ec2.controller', passing 'ec2.context' and + 'ec2.action_args' (all variables in WSGI environ.) Returns an XML + response, or a 400 upon failure. + """ + @webob.dec.wsgify + def __call__(self, req): context = req.environ['ec2.context'] + controller = req.environ['ec2.controller'] + action = req.environ['ec2.action'] + args = req.environ['ec2.action_args'] + + api_request = APIRequest(controller, action) try: - return api_request.send(context, **req.environ['ec2.action_args']) + return api_request.send(context, **args) except exception.ApiError as ex: return self._error(req, type(ex).__name__ + "." + ex.code, ex.message) # TODO(vish): do something more useful with unknown exceptions @@ -141,3 +163,4 @@ class Router(wsgi.Application): '%s' '?') % (code, message)) return resp + -- cgit From 83df968cfb050bdb6bac981dfcc2d0b1c3dd80db Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Sep 2010 12:42:06 -0400 Subject: Delete rbac.py, moving @rbac decorator knowledge into api.ec2.Authorizer WSGI middleware. --- nova/api/ec2/__init__.py | 64 +++++++++++++++++++++++++++++++++++++++++--- nova/api/ec2/admin.py | 31 ---------------------- nova/api/ec2/cloud.py | 30 --------------------- nova/auth/rbac.py | 69 ------------------------------------------------ 4 files changed, 60 insertions(+), 134 deletions(-) delete mode 100644 nova/auth/rbac.py diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 87a72ca7c..aee9915d0 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -122,14 +122,70 @@ class Router(wsgi.Application): class Authorization(wsgi.Middleware): """ - Verify that ec2.controller and ec2.action in WSGI environ may be executed - in ec2.context. + Return a 401 if ec2.controller and ec2.action in WSGI environ may not be + executed in ec2.context. """ + def __init__(self, application): + super(Authorization, self).__init__(application) + self.action_roles = { + 'CloudController': { + 'DescribeAvailabilityzones': ['all'], + 'DescribeRegions': ['all'], + 'DescribeSnapshots': ['all'], + 'DescribeKeyPairs': ['all'], + 'CreateKeyPair': ['all'], + 'DeleteKeyPair': ['all'], + 'DescribeSecurityGroups': ['all'], + 'CreateSecurityGroup': ['netadmin'], + 'DeleteSecurityGroup': ['netadmin'], + 'GetConsoleOutput': ['projectmanager', 'sysadmin'], + 'DescribeVolumes': ['projectmanager', 'sysadmin'], + 'CreateVolume': ['projectmanager', 'sysadmin'], + 'AttachVolume': ['projectmanager', 'sysadmin'], + 'DetachVolume': ['projectmanager', 'sysadmin'], + 'DescribeInstances': ['all'], + 'DescribeAddresses': ['all'], + 'AllocateAddress': ['netadmin'], + 'ReleaseAddress': ['netadmin'], + 'AssociateAddress': ['netadmin'], + 'DisassociateAddress': ['netadmin'], + 'RunInstances': ['projectmanager', 'sysadmin'], + 'TerminateInstances': ['projectmanager', 'sysadmin'], + 'RebootInstances': ['projectmanager', 'sysadmin'], + 'DeleteVolume': ['projectmanager', 'sysadmin'], + 'DescribeImages': ['all'], + 'DeregisterImage': ['projectmanager', 'sysadmin'], + 'RegisterImage': ['projectmanager', 'sysadmin'], + 'DescribeImageAttribute': ['all'], + 'ModifyImageAttribute': ['projectmanager', 'sysadmin'], + }, + 'AdminController': { + # All actions have the same permission: [] (the default) + # admins will be allowed to run them + # all others will get HTTPUnauthorized. + }, + } + @webob.dec.wsgify def __call__(self, req): - #TODO(gundlach): put rbac information here. - return self.application + context = req.environ['ec2.context'] + controller_name = req.environ['ec2.controller'].__name__ + action = req.environ['ec2.action'] + allowed_roles = self.action_roles[controller_name].get(action, []) + if self._matches_any_role(context, allowed_roles): + return self.application + else: + raise webob.exc.HTTPUnauthorized() + + def _matches_any_role(self, context, roles): + """Return True if any role in roles is allowed in context.""" + if 'all' in roles: + return True + if 'none' in roles: + return False + return any(context.project.has_role(context.user.id, role) + for role in roles) class Executor(wsg.Application): diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index d6f622755..f0c643bbd 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -57,46 +57,27 @@ def host_dict(host): return {} -def admin_only(target): - """Decorator for admin-only API calls""" - def wrapper(*args, **kwargs): - """Internal wrapper method for admin-only API calls""" - context = args[1] - if context.user.is_admin(): - return target(*args, **kwargs) - else: - return {} - - return wrapper - - class AdminController(object): """ API Controller for users, hosts, nodes, and workers. - Trivial admin_only wrapper will be replaced with RBAC, - allowing project managers to administer project users. """ def __str__(self): return 'AdminController' - @admin_only def describe_user(self, _context, name, **_kwargs): """Returns user data, including access and secret keys.""" return user_dict(manager.AuthManager().get_user(name)) - @admin_only def describe_users(self, _context, **_kwargs): """Returns all users - should be changed to deal with a list.""" return {'userSet': [user_dict(u) for u in manager.AuthManager().get_users()] } - @admin_only def register_user(self, _context, name, **_kwargs): """Creates a new user, and returns generated credentials.""" return user_dict(manager.AuthManager().create_user(name)) - @admin_only def deregister_user(self, _context, name, **_kwargs): """Deletes a single user (NOT undoable.) Should throw an exception if the user has instances, @@ -106,13 +87,11 @@ class AdminController(object): return True - @admin_only def describe_roles(self, context, project_roles=True, **kwargs): """Returns a list of allowed roles.""" roles = manager.AuthManager().get_roles(project_roles) return { 'roles': [{'role': r} for r in roles]} - @admin_only def describe_user_roles(self, context, user, project=None, **kwargs): """Returns a list of roles for the given user. Omitting project will return any global roles that the user has. @@ -121,7 +100,6 @@ class AdminController(object): roles = manager.AuthManager().get_user_roles(user, project=project) return { 'roles': [{'role': r} for r in roles]} - @admin_only def modify_user_role(self, context, user, role, project=None, operation='add', **kwargs): """Add or remove a role for a user and project.""" @@ -134,7 +112,6 @@ class AdminController(object): return True - @admin_only def generate_x509_for_user(self, _context, name, project=None, **kwargs): """Generates and returns an x509 certificate for a single user. Is usually called from a client that will wrap this with @@ -146,19 +123,16 @@ class AdminController(object): user = manager.AuthManager().get_user(name) return user_dict(user, base64.b64encode(project.get_credentials(user))) - @admin_only def describe_project(self, context, name, **kwargs): """Returns project data, including member ids.""" return project_dict(manager.AuthManager().get_project(name)) - @admin_only def describe_projects(self, context, user=None, **kwargs): """Returns all projects - should be changed to deal with a list.""" return {'projectSet': [project_dict(u) for u in manager.AuthManager().get_projects(user=user)]} - @admin_only def register_project(self, context, name, manager_user, description=None, member_users=None, **kwargs): """Creates a new project""" @@ -169,20 +143,17 @@ class AdminController(object): description=None, member_users=None)) - @admin_only def deregister_project(self, context, name): """Permanently deletes a project.""" manager.AuthManager().delete_project(name) return True - @admin_only def describe_project_members(self, context, name, **kwargs): project = manager.AuthManager().get_project(name) result = { 'members': [{'member': m} for m in project.member_ids]} return result - @admin_only def modify_project_member(self, context, user, project, operation, **kwargs): """Add or remove a user from a project.""" if operation =='add': @@ -193,7 +164,6 @@ class AdminController(object): raise exception.ApiError('operation must be add or remove') return True - @admin_only def describe_hosts(self, _context, **_kwargs): """Returns status info for all nodes. Includes: * Disk Space @@ -205,7 +175,6 @@ class AdminController(object): """ return {'hostSet': [host_dict(h) for h in model.Host.all()]} - @admin_only def describe_host(self, _context, name, **_kwargs): """Returns status info for single node.""" return host_dict(model.Host.lookup(name)) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 05fbf3861..566887c1a 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -32,7 +32,6 @@ from nova import exception from nova import flags from nova import rpc from nova import utils -from nova.auth import rbac from nova.auth import manager from nova.compute import model from nova.compute.instance_types import INSTANCE_TYPES @@ -163,18 +162,15 @@ class CloudController(object): data['product-codes'] = i['product_codes'] return data - @rbac.allow('all') def describe_availability_zones(self, context, **kwargs): return {'availabilityZoneInfo': [{'zoneName': 'nova', 'zoneState': 'available'}]} - @rbac.allow('all') def describe_regions(self, context, region_name=None, **kwargs): # TODO(vish): region_name is an array. Support filtering return {'regionInfo': [{'regionName': 'nova', 'regionUrl': FLAGS.ec2_url}]} - @rbac.allow('all') def describe_snapshots(self, context, snapshot_id=None, @@ -190,7 +186,6 @@ class CloudController(object): 'volumeSize': 0, 'description': 'fixme'}]} - @rbac.allow('all') def describe_key_pairs(self, context, key_name=None, **kwargs): key_pairs = context.user.get_key_pairs() if not key_name is None: @@ -208,35 +203,29 @@ class CloudController(object): return {'keypairsSet': result} - @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): data = _gen_key(context.user.id, key_name) return {'keyName': key_name, 'keyFingerprint': data['fingerprint'], 'keyMaterial': data['private_key']} - @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): context.user.delete_key_pair(key_name) # aws returns true even if the key doens't exist return True - @rbac.allow('all') def describe_security_groups(self, context, group_names, **kwargs): groups = {'securityGroupSet': []} # Stubbed for now to unblock other things. return groups - @rbac.allow('netadmin') def create_security_group(self, context, group_name, **kwargs): return True - @rbac.allow('netadmin') def delete_security_group(self, context, group_name, **kwargs): return True - @rbac.allow('projectmanager', 'sysadmin') def get_console_output(self, context, instance_id, **kwargs): # instance_id is passed in as a list of instances instance = self._get_instance(context, instance_id[0]) @@ -250,7 +239,6 @@ class CloudController(object): else: return None - @rbac.allow('projectmanager', 'sysadmin') def describe_volumes(self, context, **kwargs): volumes = [] for volume in self.volumes: @@ -284,7 +272,6 @@ class CloudController(object): v['attachmentSet'] = [{}] return v - @rbac.allow('projectmanager', 'sysadmin') def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it result = rpc.call(FLAGS.volume_topic, {"method": "create_volume", @@ -324,7 +311,6 @@ class CloudController(object): return volume raise exception.NotFound('Volume %s could not be found' % volume_id) - @rbac.allow('projectmanager', 'sysadmin') def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume = self._get_volume(context, volume_id) if volume['status'] == "attached": @@ -348,7 +334,6 @@ class CloudController(object): 'status': volume['attach_status'], 'volumeId': volume_id} - @rbac.allow('projectmanager', 'sysadmin') def detach_volume(self, context, volume_id, **kwargs): volume = self._get_volume(context, volume_id) instance_id = volume.get('instance_id', None) @@ -381,7 +366,6 @@ class CloudController(object): lst = [lst] return [{label: x} for x in lst] - @rbac.allow('all') def describe_instances(self, context, **kwargs): return self._format_describe_instances(context) @@ -442,7 +426,6 @@ class CloudController(object): return list(reservations.values()) - @rbac.allow('all') def describe_addresses(self, context, **kwargs): return self.format_addresses(context) @@ -465,7 +448,6 @@ class CloudController(object): addresses.append(address_rv) return {'addressesSet': addresses} - @rbac.allow('netadmin') def allocate_address(self, context, **kwargs): network_topic = self._get_network_topic(context) public_ip = rpc.call(network_topic, @@ -474,7 +456,6 @@ class CloudController(object): "project_id": context.project.id}}) return {'addressSet': [{'publicIp': public_ip}]} - @rbac.allow('netadmin') def release_address(self, context, public_ip, **kwargs): # NOTE(vish): Should we make sure this works? network_topic = self._get_network_topic(context) @@ -483,7 +464,6 @@ class CloudController(object): "args": {"elastic_ip": public_ip}}) return {'releaseResponse': ["Address released."]} - @rbac.allow('netadmin') def associate_address(self, context, instance_id, public_ip, **kwargs): instance = self._get_instance(context, instance_id) address = self._get_address(context, public_ip) @@ -495,7 +475,6 @@ class CloudController(object): "instance_id": instance['instance_id']}}) return {'associateResponse': ["Address associated."]} - @rbac.allow('netadmin') def disassociate_address(self, context, public_ip, **kwargs): address = self._get_address(context, public_ip) network_topic = self._get_network_topic(context) @@ -514,7 +493,6 @@ class CloudController(object): "project_id": context.project.id}}) return '%s.%s' %(FLAGS.network_topic, host) - @rbac.allow('projectmanager', 'sysadmin') def run_instances(self, context, **kwargs): # make sure user can access the image # vpn image is private so it doesn't show up on lists @@ -587,7 +565,6 @@ class CloudController(object): # TODO: Make Network figure out the network name from ip. return self._format_run_instances(context, reservation_id) - @rbac.allow('projectmanager', 'sysadmin') def terminate_instances(self, context, instance_id, **kwargs): logging.debug("Going to start terminating instances") network_topic = self._get_network_topic(context) @@ -628,7 +605,6 @@ class CloudController(object): instance.destroy() return True - @rbac.allow('projectmanager', 'sysadmin') def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" for i in instance_id: @@ -638,7 +614,6 @@ class CloudController(object): "args": {"instance_id": i}}) return True - @rbac.allow('projectmanager', 'sysadmin') def delete_volume(self, context, volume_id, **kwargs): # TODO: return error if not authorized volume = self._get_volume(context, volume_id) @@ -648,19 +623,16 @@ class CloudController(object): "args": {"volume_id": volume_id}}) return True - @rbac.allow('all') def describe_images(self, context, image_id=None, **kwargs): # The objectstore does its own authorization for describe imageSet = images.list(context, image_id) return {'imagesSet': imageSet} - @rbac.allow('projectmanager', 'sysadmin') def deregister_image(self, context, image_id, **kwargs): # FIXME: should the objectstore be doing these authorization checks? images.deregister(context, image_id) return {'imageId': image_id} - @rbac.allow('projectmanager', 'sysadmin') def register_image(self, context, image_location=None, **kwargs): # FIXME: should the objectstore be doing these authorization checks? if image_location is None and kwargs.has_key('name'): @@ -670,7 +642,6 @@ class CloudController(object): return {'imageId': image_id} - @rbac.allow('all') def describe_image_attribute(self, context, image_id, attribute, **kwargs): if attribute != 'launchPermission': raise exception.ApiError('attribute not supported: %s' % attribute) @@ -683,7 +654,6 @@ class CloudController(object): result['launchPermission'].append({'group': 'all'}) return result - @rbac.allow('projectmanager', 'sysadmin') def modify_image_attribute(self, context, image_id, attribute, operation_type, **kwargs): # TODO(devcamcar): Support users and groups other than 'all'. if attribute != 'launchPermission': diff --git a/nova/auth/rbac.py b/nova/auth/rbac.py deleted file mode 100644 index d157f44b3..000000000 --- a/nova/auth/rbac.py +++ /dev/null @@ -1,69 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Role-based access control decorators to use fpr wrapping other -methods with.""" - -from nova import exception - - -def allow(*roles): - """Allow the given roles access the wrapped function.""" - - def wrap(func): # pylint: disable-msg=C0111 - - def wrapped_func(self, context, *args, - **kwargs): # pylint: disable-msg=C0111 - if context.user.is_superuser(): - return func(self, context, *args, **kwargs) - for role in roles: - if __matches_role(context, role): - return func(self, context, *args, **kwargs) - raise exception.NotAuthorized() - - return wrapped_func - - return wrap - - -def deny(*roles): - """Deny the given roles access the wrapped function.""" - - def wrap(func): # pylint: disable-msg=C0111 - - def wrapped_func(self, context, *args, - **kwargs): # pylint: disable-msg=C0111 - if context.user.is_superuser(): - return func(self, context, *args, **kwargs) - for role in roles: - if __matches_role(context, role): - raise exception.NotAuthorized() - return func(self, context, *args, **kwargs) - - return wrapped_func - - return wrap - - -def __matches_role(context, role): - """Check if a role is allowed.""" - if role == 'all': - return True - if role == 'none': - return False - return context.project.has_role(context.user.id, role) -- cgit From 40778d77936cb63decfc56e6b75fa4c31c13a564 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Sep 2010 13:28:30 -0400 Subject: notes -- conversion 'complete' except now the unit tests won't work and surely i have bugs :) --- nova/endpoint/notes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/notes.txt b/nova/endpoint/notes.txt index cbb7b0cd0..3e48f1749 100644 --- a/nova/endpoint/notes.txt +++ b/nova/endpoint/notes.txt @@ -41,8 +41,8 @@ CloudController and AdminController: ==== STRATEGY TO CONVERT TO EVENTLET+WSGI ==== * Controllers: - move the @rbac.allow data into an auth WSGI that is right above the call - to the controller +x move the @rbac.allow data into an auth WSGI that is right above the call +x to the controller x verify @defer.inlineCallbacks is just to allow the yield statements, then x remove the yield statements (untangle from twisted) -- cgit From b965dde9e95e16a9a207697d5729bd146c2dfd23 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Sep 2010 13:55:38 -0400 Subject: Fix simple errors to the point where we can run the tests [but not pass] --- nova/api/ec2/__init__.py | 10 ++-- nova/api/ec2/apirequestcontext.py | 33 ----------- nova/api/ec2/cloud.py | 2 +- nova/api/ec2/context.py | 33 +++++++++++ nova/endpoint/api.py | 122 -------------------------------------- nova/objectstore/handler.py | 4 +- nova/tests/api_unittest.py | 3 +- nova/tests/auth_unittest.py | 2 +- nova/tests/cloud_unittest.py | 6 +- 9 files changed, 47 insertions(+), 168 deletions(-) delete mode 100644 nova/api/ec2/apirequestcontext.py create mode 100644 nova/api/ec2/context.py delete mode 100755 nova/endpoint/api.py diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index aee9915d0..3335338e0 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -26,9 +26,11 @@ import webob import webob.dec import webob.exc +from nova import exception +from nova import wsgi +from nova.api.ec2 import context from nova.api.ec2 import admin from nova.api.ec2 import cloud -from nova import exception from nova.auth import manager @@ -74,7 +76,7 @@ class Authenticate(wsgi.Middleware): raise webob.exc.HTTPForbidden() # Authenticated! - req.environ['ec2.context'] = APIRequestContext(user, project) + req.environ['ec2.context'] = context.APIRequestContext(user, project) return self.application @@ -188,7 +190,7 @@ class Authorization(wsgi.Middleware): for role in roles) -class Executor(wsg.Application): +class Executor(wsgi.Application): """ Executes 'ec2.action' upon 'ec2.controller', passing 'ec2.context' and 'ec2.action_args' (all variables in WSGI environ.) Returns an XML @@ -217,6 +219,6 @@ class Executor(wsg.Application): resp.body = ('\n' '%s' '%s' - '?') % (code, message)) + '?') % (code, message) return resp diff --git a/nova/api/ec2/apirequestcontext.py b/nova/api/ec2/apirequestcontext.py deleted file mode 100644 index fb3118020..000000000 --- a/nova/api/ec2/apirequestcontext.py +++ /dev/null @@ -1,33 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -APIRequestContext -""" - -import random - -class APIRequestContext(object): - def __init__(self, handler, user, project): - self.handler = handler - self.user = user - self.project = project - self.request_id = ''.join( - [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') - for x in xrange(20)] - ) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 566887c1a..fc0eb2711 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -357,7 +357,7 @@ class CloudController(object): 'instanceId': instance_id, 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': volume_id}) + 'volumeId': volume_id} def _convert_to_set(self, lst, label): if lst == None or lst == []: diff --git a/nova/api/ec2/context.py b/nova/api/ec2/context.py new file mode 100644 index 000000000..fb3118020 --- /dev/null +++ b/nova/api/ec2/context.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +APIRequestContext +""" + +import random + +class APIRequestContext(object): + def __init__(self, handler, user, project): + self.handler = handler + self.user = user + self.project = project + self.request_id = ''.join( + [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') + for x in xrange(20)] + ) diff --git a/nova/endpoint/api.py b/nova/endpoint/api.py deleted file mode 100755 index 311fb1880..000000000 --- a/nova/endpoint/api.py +++ /dev/null @@ -1,122 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Tornado REST API Request Handlers for Nova functions -Most calls are proxied into the responsible controller. -""" - -import multiprocessing -import re -import urllib - -import tornado.web - -from nova import crypto -from nova import flags -import nova.cloudpipe.api -from nova.endpoint import cloud - - -FLAGS = flags.FLAGS -flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') - - -class RootRequestHandler(tornado.web.RequestHandler): - def get(self): - # available api versions - versions = [ - '1.0', - '2007-01-19', - '2007-03-01', - '2007-08-29', - '2007-10-10', - '2007-12-15', - '2008-02-01', - '2008-09-01', - '2009-04-04', - ] - for version in versions: - self.write('%s\n' % version) - self.finish() - - -class MetadataRequestHandler(tornado.web.RequestHandler): - def print_data(self, data): - if isinstance(data, dict): - output = '' - for key in data: - if key == '_name': - continue - output += key - if isinstance(data[key], dict): - if '_name' in data[key]: - output += '=' + str(data[key]['_name']) - else: - output += '/' - output += '\n' - self.write(output[:-1]) # cut off last \n - elif isinstance(data, list): - self.write('\n'.join(data)) - else: - self.write(str(data)) - - def lookup(self, path, data): - items = path.split('/') - for item in items: - if item: - if not isinstance(data, dict): - return data - if not item in data: - return None - data = data[item] - return data - - def get(self, path): - cc = self.application.controllers['Cloud'] - meta_data = cc.get_metadata(self.request.remote_ip) - if meta_data is None: - _log.error('Failed to get metadata for ip: %s' % - self.request.remote_ip) - raise tornado.web.HTTPError(404) - data = self.lookup(path, meta_data) - if data is None: - raise tornado.web.HTTPError(404) - self.print_data(data) - self.finish() - - -class APIServerApplication(tornado.web.Application): - def __init__(self, controllers): - tornado.web.Application.__init__(self, [ - (r'/', RootRequestHandler), - (r'/cloudpipe/(.*)', nova.cloudpipe.api.CloudPipeRequestHandler), - (r'/cloudpipe', nova.cloudpipe.api.CloudPipeRequestHandler), - (r'/services/([A-Za-z0-9]+)/', APIRequestHandler), - (r'/latest/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2009-04-04/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2008-09-01/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2008-02-01/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2007-12-15/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2007-10-10/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2007-08-29/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2007-03-01/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/2007-01-19/([-A-Za-z0-9/]*)', MetadataRequestHandler), - (r'/1.0/([-A-Za-z0-9/]*)', MetadataRequestHandler), - ], pool=multiprocessing.Pool(4)) - self.controllers = controllers diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 035e342ca..09591101b 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -55,7 +55,7 @@ from twisted.web import static from nova import exception from nova import flags from nova.auth import manager -from nova.endpoint import api +from nova.api.ec2 import context from nova.objectstore import bucket from nova.objectstore import image @@ -122,7 +122,7 @@ def get_context(request): request.uri, headers=request.getAllHeaders(), check_type='s3') - return api.APIRequestContext(None, user, project) + return context.APIRequestContext(None, user, project) except exception.Error as ex: logging.debug("Authentication Failure: %s" % ex) raise exception.NotAuthorized diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 9d072866c..d21ded75b 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -27,8 +27,7 @@ from twisted.internet import defer from nova import flags from nova import test from nova.auth import manager -from nova.endpoint import api -from nova.endpoint import cloud +from nova.api.ec2 import cloud FLAGS = flags.FLAGS diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 0b404bfdc..b3b6800a1 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -26,7 +26,7 @@ from nova import crypto from nova import flags from nova import test from nova.auth import manager -from nova.endpoint import cloud +from nova.api.ec2 import cloud FLAGS = flags.FLAGS diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 3501771cc..545cbaede 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -29,8 +29,8 @@ from nova import rpc from nova import test from nova.auth import manager from nova.compute import service -from nova.endpoint import api -from nova.endpoint import cloud +from nova.api.ec2 import context +from nova.api.ec2 import cloud FLAGS = flags.FLAGS @@ -64,7 +64,7 @@ class CloudTestCase(test.BaseTestCase): except: pass admin = manager.AuthManager().get_user('admin') project = manager.AuthManager().create_project('proj', 'admin', 'proj') - self.context = api.APIRequestContext(handler=None,project=project,user=admin) + self.context = context.APIRequestContext(handler=None,project=project,user=admin) def tearDown(self): manager.AuthManager().delete_project('proj') -- cgit From 8169a2a26c5b646a4d6c63c77f15f6aaa6898cb4 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 13:04:05 -0400 Subject: Small typos, plus rework api_unittest to use WSGI instead of Tornado --- nova/api/__init__.py | 2 +- nova/api/ec2/__init__.py | 10 ++-- nova/tests/api_unittest.py | 120 +++++++++------------------------------------ 3 files changed, 29 insertions(+), 103 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 0166b7fc1..786b246ec 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -37,5 +37,5 @@ class API(wsgi.Router): # be dropped; and I'm leaving off CloudPipeRequestHandler until # I hear that we need it. mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API()) - mapper.connect("/ec2/{path_info:.*}", controller=ec2.API()) + mapper.connect("/services/{path_info:.*}", controller=ec2.API()) super(API, self).__init__(mapper) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 3335338e0..a94bcb863 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -44,6 +44,7 @@ class API(wsgi.Middleware): def __init__(self): self.application = Authenticate(Router(Authorizer(Executor()))) + class Authenticate(wsgi.Middleware): """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" @@ -81,11 +82,12 @@ class Authenticate(wsgi.Middleware): return self.application -class Router(wsgi.Application): +class Router(wsgi.Middleware): """ Add 'ec2.controller', 'ec2.action', and 'ec2.action_args' to WSGI environ. """ - def __init__(self): + def __init__(self, application): + super(Router, self).__init__(application) self.map = routes.Mapper() self.map.connect("/{controller_name}/") self.controllers = dict(Cloud=cloud.CloudController(), @@ -122,14 +124,14 @@ class Router(wsgi.Application): return self.application -class Authorization(wsgi.Middleware): +class Authorizer(wsgi.Middleware): """ Return a 401 if ec2.controller and ec2.action in WSGI environ may not be executed in ec2.context. """ def __init__(self, application): - super(Authorization, self).__init__(application) + super(Authorizer, self).__init__(application) self.action_roles = { 'CloudController': { 'DescribeAvailabilityzones': ['all'], diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index d21ded75b..a13bbdeed 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -21,53 +21,14 @@ from boto.ec2 import regioninfo import httplib import random import StringIO -from tornado import httpserver -from twisted.internet import defer +import webob -from nova import flags from nova import test from nova.auth import manager +from nova.api import ec2 from nova.api.ec2 import cloud -FLAGS = flags.FLAGS - - -# NOTE(termie): These are a bunch of helper methods and classes to short -# circuit boto calls and feed them into our tornado handlers, -# it's pretty damn circuitous so apologies if you have to fix -# a bug in it -def boto_to_tornado(method, path, headers, data, host, connection=None): - """ translate boto requests into tornado requests - - connection should be a FakeTornadoHttpConnection instance - """ - try: - headers = httpserver.HTTPHeaders() - except AttributeError: - from tornado import httputil - headers = httputil.HTTPHeaders() - for k, v in headers.iteritems(): - headers[k] = v - - req = httpserver.HTTPRequest(method=method, - uri=path, - headers=headers, - body=data, - host=host, - remote_ip='127.0.0.1', - connection=connection) - return req - - -def raw_to_httpresponse(s): - """ translate a raw tornado http response into an httplib.HTTPResponse """ - sock = FakeHttplibSocket(s) - resp = httplib.HTTPResponse(sock) - resp.begin() - return resp - - class FakeHttplibSocket(object): """ a fake socket implementation for httplib.HTTPResponse, trivial """ def __init__(self, s): @@ -77,73 +38,36 @@ class FakeHttplibSocket(object): return self.fp -class FakeTornadoStream(object): - """ a fake stream to satisfy tornado's assumptions, trivial """ - def set_close_callback(self, f): - pass - - -class FakeTornadoConnection(object): - """ a fake connection object for tornado to pass to its handlers - - web requests are expected to write to this as they get data and call - finish when they are done with the request, we buffer the writes and - kick off a callback when it is done so that we can feed the result back - into boto. - """ - def __init__(self, d): - self.d = d - self._buffer = StringIO.StringIO() - - def write(self, chunk): - self._buffer.write(chunk) - - def finish(self): - s = self._buffer.getvalue() - self.d.callback(s) - - xheaders = None - - @property - def stream(self): - return FakeTornadoStream() - - class FakeHttplibConnection(object): """ a fake httplib.HTTPConnection for boto to use requests made via this connection actually get translated and routed into - our tornado app, we then wait for the response and turn it back into + our WSGI app, we then wait for the response and turn it back into the httplib.HTTPResponse that boto expects. """ def __init__(self, app, host, is_secure=False): self.app = app self.host = host - self.deferred = defer.Deferred() def request(self, method, path, data, headers): - req = boto_to_tornado - conn = FakeTornadoConnection(self.deferred) - request = boto_to_tornado(connection=conn, - method=method, - path=path, - headers=headers, - data=data, - host=self.host) - handler = self.app(request) - self.deferred.addCallback(raw_to_httpresponse) + req = webob.Request.blank(path) + req.method = method + req.body = data + req.headers = headers + req.headers['Accept'] = 'text/html' + req.host = self.host + # Call the WSGI app, get the HTTP response + resp = str(req.get_response(self.app)) + # For some reason, the response doesn't have "HTTP/1.0 " prepended; I + # guess that's a function the web server usually provides. + resp = "HTTP/1.0 %s" % resp + + sock = FakeHttplibSocket(resp) + self.http_response = httplib.HTTPResponse(sock) + self.http_response.begin() def getresponse(self): - @defer.inlineCallbacks - def _waiter(): - result = yield self.deferred - defer.returnValue(result) - d = _waiter() - # NOTE(termie): defer.returnValue above should ensure that - # this deferred has already been called by the time - # we get here, we are going to cheat and return - # the result of the callback - return d.result + return self.http_response def close(self): pass @@ -158,20 +82,20 @@ class ApiEc2TestCase(test.BaseTestCase): self.host = '127.0.0.1' - self.app = api.APIServerApplication({'Cloud': self.cloud}) + self.app = ec2.API() self.ec2 = boto.connect_ec2( aws_access_key_id='fake', aws_secret_access_key='fake', is_secure=False, region=regioninfo.RegionInfo(None, 'test', self.host), - port=FLAGS.cc_port, + port=0, path='/services/Cloud') self.mox.StubOutWithMock(self.ec2, 'new_http_connection') def expect_http(self, host=None, is_secure=False): http = FakeHttplibConnection( - self.app, '%s:%d' % (self.host, FLAGS.cc_port), False) + self.app, '%s:0' % (self.host), False) self.ec2.new_http_connection(host, is_secure).AndReturn(http) return http -- cgit From 7edff9298f7f01e158f90c93432384903d71e033 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 11:32:37 -0700 Subject: scheduler + unittests --- bin/nova-scheduler | 4 +- nova/db/api.py | 13 +++++ nova/db/sqlalchemy/api.py | 31 ++++++++++++ nova/endpoint/cloud.py | 6 +-- nova/flags.py | 2 + nova/scheduler/base.py | 80 ------------------------------ nova/scheduler/bestfit.py | 34 ------------- nova/scheduler/chance.py | 37 +++++++++++--- nova/scheduler/driver.py | 62 +++++++++++++++++++++++ nova/scheduler/manager.py | 60 +++++++++++++++++++++++ nova/scheduler/service.py | 76 ----------------------------- nova/scheduler/simple.py | 81 ++++++++++++++++++++++++++++++ nova/tests/compute_unittest.py | 2 +- nova/tests/scheduler_unittest.py | 103 +++++++++++++++++++++++++++++++++++++++ run_tests.py | 1 + 15 files changed, 390 insertions(+), 202 deletions(-) delete mode 100644 nova/scheduler/base.py delete mode 100644 nova/scheduler/bestfit.py create mode 100644 nova/scheduler/driver.py create mode 100644 nova/scheduler/manager.py delete mode 100644 nova/scheduler/service.py create mode 100644 nova/scheduler/simple.py create mode 100644 nova/tests/scheduler_unittest.py diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 1ad41bbd3..97f98b17f 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -21,12 +21,12 @@ Twistd daemon for the nova scheduler nodes. """ +from nova import service from nova import twistd -from nova.scheduler import service if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = service.SchedulerService.create() + application = service.Service.create() diff --git a/nova/db/api.py b/nova/db/api.py index 6cb49b7e4..07eebd017 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -62,6 +62,19 @@ def daemon_get_by_args(context, host, binary): return IMPL.daemon_get_by_args(context, host, binary) +def daemon_get_all_by_topic(context, topic): + """Get all compute daemons for a given topi """ + return IMPL.daemon_get_all_by_topic(context, topic) + + +def daemon_get_all_compute_sorted(context): + """Get all compute daemons sorted by instance count + + Returns a list of (Daemon, instance_count) tuples + """ + return IMPL.daemon_get_all_compute_sorted(context) + + def daemon_create(context, values): """Create a daemon from the values dictionary.""" return IMPL.daemon_create(context, values) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 5d98ee5bf..aabd74984 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -25,6 +25,7 @@ from nova import flags from nova.db.sqlalchemy import models from nova.db.sqlalchemy.session import managed_session from sqlalchemy import or_ +from sqlalchemy.sql import func FLAGS = flags.FLAGS @@ -43,6 +44,36 @@ def daemon_get_by_args(_context, host, binary): return models.Daemon.find_by_args(host, binary) +def daemon_get_all_by_topic(context, topic): + with managed_session() as session: + return session.query(models.Daemon) \ + .filter_by(deleted=False) \ + .filter_by(topic=topic) \ + .all() + + +def daemon_get_all_compute_sorted(_context): + with managed_session() as session: + # NOTE(vish): The intended query is below + # SELECT daemons.*, inst_count.instance_count + # FROM daemons LEFT OUTER JOIN + # (SELECT host, count(*) AS instance_count + # FROM instances GROUP BY host) AS inst_count + print 'instance', models.Instance.find(1).host + subq = session.query(models.Instance.host, + func.count('*').label('instance_count')) \ + .filter_by(deleted=False) \ + .group_by(models.Instance.host) \ + .subquery() + topic = 'compute' + return session.query(models.Daemon, subq.c.instance_count) \ + .filter_by(topic=topic) \ + .filter_by(deleted=False) \ + .outerjoin((subq, models.Daemon.host == subq.c.host)) \ + .order_by(subq.c.instance_count) \ + .all() + + def daemon_create(_context, values): daemon_ref = models.Daemon() for (key, value) in values.iteritems(): diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 4e86145db..2c88ef406 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -488,9 +488,9 @@ class CloudController(object): host = db.network_get_host(context, network_ref['id']) if not host: host = yield rpc.call(FLAGS.network_topic, - {"method": "set_network_host", - "args": {"context": None, - "project_id": context.project.id}}) + {"method": "set_network_host", + "args": {"context": None, + "project_id": context.project.id}}) defer.returnValue(db.queue_get_for(context, FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') diff --git a/nova/flags.py b/nova/flags.py index aa9648843..40ce9c736 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -220,5 +220,7 @@ DEFINE_string('network_manager', 'nova.network.manager.VlanManager', 'Manager for network') DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager', 'Manager for volume') +DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', + 'Manager for scheduler') diff --git a/nova/scheduler/base.py b/nova/scheduler/base.py deleted file mode 100644 index 2872ae6fe..000000000 --- a/nova/scheduler/base.py +++ /dev/null @@ -1,80 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Openstack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Scheduler base class that all Schedulers should inherit from -""" - -import time - -from nova import flags -from nova.datastore import Redis - -FLAGS = flags.FLAGS -flags.DEFINE_integer('node_down_time', - 60, - 'seconds without heartbeat that determines a ' - 'compute node to be down') - - -class Scheduler(object): - """ - The base class that all Scheduler clases should inherit from - """ - - @staticmethod - def compute_nodes(): - """ - Return a list of compute nodes - """ - - return [identifier.split(':')[0] - for identifier in Redis.instance().smembers("daemons") - if (identifier.split(':')[1] == "nova-compute")] - - @staticmethod - def compute_node_is_up(node): - """ - Given a node name, return whether the node is considered 'up' by - if it's sent a heartbeat recently - """ - - time_str = Redis.instance().hget('%s:%s:%s' % - ('daemon', node, 'nova-compute'), - 'updated_at') - if not time_str: - return False - - # Would be a lot easier if we stored heartbeat time in epoch :) - - # The 'str()' here is to get rid of a pylint error - time_str = str(time_str).replace('Z', 'UTC') - time_split = time.strptime(time_str, '%Y-%m-%dT%H:%M:%S%Z') - epoch_time = int(time.mktime(time_split)) - time.timezone - return (time.time() - epoch_time) < FLAGS.node_down_time - - def compute_nodes_up(self): - """ - Return the list of compute nodes that are considered 'up' - """ - - return [node for node in self.compute_nodes() - if self.compute_node_is_up(node)] - - def pick_node(self, instance_id, **_kwargs): - """You DEFINITELY want to define this in your subclass""" - - raise NotImplementedError("Your subclass should define pick_node") diff --git a/nova/scheduler/bestfit.py b/nova/scheduler/bestfit.py deleted file mode 100644 index bdd4fcbdc..000000000 --- a/nova/scheduler/bestfit.py +++ /dev/null @@ -1,34 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Openstack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Best Fit Scheduler -""" - -from nova.scheduler.base import Scheduler - - -class BestFitScheduler(Scheduler): - """ - Implements Scheduler as a best-fit node selector - """ - - def pick_node(self, instance_id, **_kwargs): - """ - Picks a node that is up and is a best fit for the new instance - """ - - raise NotImplementedError("BestFitScheduler is not done yet") diff --git a/nova/scheduler/chance.py b/nova/scheduler/chance.py index 719c37674..12321cec1 100644 --- a/nova/scheduler/chance.py +++ b/nova/scheduler/chance.py @@ -1,6 +1,9 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright (c) 2010 Openstack, LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,18 +23,40 @@ Chance (Random) Scheduler implementation import random -from nova.scheduler.base import Scheduler +from nova.scheduler import driver -class ChanceScheduler(Scheduler): +class ChanceScheduler(driver.Scheduler): """ Implements Scheduler as a random node selector """ - def pick_node(self, instance_id, **_kwargs): + def pick_compute_host(self, context, instance_id, **_kwargs): """ - Picks a node that is up at random + Picks a host that is up at random """ - nodes = self.compute_nodes_up() - return nodes[int(random.random() * len(nodes))] + hosts = self.hosts_up(context, 'compute') + if not hosts: + raise driver.NoValidHost("No hosts found") + return hosts[int(random.random() * len(hosts))] + + def pick_volume_host(self, context, volume_id, **_kwargs): + """ + Picks a host that is up at random + """ + + hosts = self.hosts_up(context, 'volume') + if not hosts: + raise driver.NoValidHost("No hosts found") + return hosts[int(random.random() * len(hosts))] + + def pick_network_host(self, context, network_id, **_kwargs): + """ + Picks a host that is up at random + """ + + hosts = self.hosts_up(context, 'network') + if not hosts: + raise driver.NoValidHost("No hosts found") + return hosts[int(random.random() * len(hosts))] diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py new file mode 100644 index 000000000..1618342c0 --- /dev/null +++ b/nova/scheduler/driver.py @@ -0,0 +1,62 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Scheduler base class that all Schedulers should inherit from +""" + +import datetime + +from nova import db +from nova import exception +from nova import flags + +FLAGS = flags.FLAGS +flags.DEFINE_integer('daemon_down_time', + 60, + 'seconds without heartbeat that determines a ' + 'compute node to be down') + +class NoValidHost(exception.Error): + """There is no valid host for the command""" + pass + +class Scheduler(object): + """ + The base class that all Scheduler clases should inherit from + """ + + @staticmethod + def daemon_is_up(daemon): + """ + Given a daemon, return whether the deamon is considered 'up' by + if it's sent a heartbeat recently + """ + elapsed = datetime.datetime.now() - daemon['updated_at'] + return elapsed < datetime.timedelta(seconds=FLAGS.daemon_down_time) + + def hosts_up(self, context, topic): + """ + Return the list of hosts that have a running daemon for topic + """ + + daemons = db.daemon_get_all_by_topic(context, topic) + return [daemon.host + for daemon in daemons + if self.daemon_is_up(daemon)] diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py new file mode 100644 index 000000000..a75b4ac41 --- /dev/null +++ b/nova/scheduler/manager.py @@ -0,0 +1,60 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Scheduler Service +""" + +import logging + +from nova import db +from nova import flags +from nova import manager +from nova import rpc +from nova import utils + +FLAGS = flags.FLAGS +flags.DEFINE_string('scheduler_driver', + 'nova.scheduler.chance.ChanceScheduler', + 'Driver to use for the scheduler') + + +class SchedulerManager(manager.Manager): + """ + Chooses a host to run instances on. + """ + def __init__(self, scheduler_driver=None, *args, **kwargs): + if not scheduler_driver: + scheduler_driver = FLAGS.scheduler_driver + self.driver = utils.import_object(scheduler_driver) + super(SchedulerManager, self).__init__(*args, **kwargs) + + def run_instance(self, context, instance_id, **_kwargs): + """ + Picks a node for a running VM and casts the run_instance request + """ + + host = self.driver.pick_host(context, instance_id, **_kwargs) + + rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "run_instance", + "args": {"context": context, + "instance_id": instance_id}}) + logging.debug("Casting to compute %s for running instance %s", + host, instance_id) diff --git a/nova/scheduler/service.py b/nova/scheduler/service.py deleted file mode 100644 index 136f262c2..000000000 --- a/nova/scheduler/service.py +++ /dev/null @@ -1,76 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Openstack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Scheduler Service -""" - -import logging -from twisted.internet import defer - -from nova import exception -from nova import flags -from nova import rpc -from nova import service -from nova.scheduler import chance -from nova.scheduler import bestfit - -FLAGS = flags.FLAGS -flags.DEFINE_string('scheduler_type', - 'chance', - 'the scheduler to use') - -SCHEDULER_CLASSES = {'chance': chance.ChanceScheduler, - 'bestfit': bestfit.BestFitScheduler} - - -class SchedulerService(service.Service): - """ - Manages the running instances. - """ - - def __init__(self): - super(SchedulerService, self).__init__() - if (FLAGS.scheduler_type not in SCHEDULER_CLASSES): - raise exception.Error("Scheduler '%s' does not exist" % - FLAGS.scheduler_type) - self._scheduler_class = SCHEDULER_CLASSES[FLAGS.scheduler_type] - - @staticmethod - def noop(): - """ simple test of an AMQP message call """ - return defer.succeed('PONG') - - def pick_node(self, instance_id, **_kwargs): - """ - Return a node to use based on the selected Scheduler - """ - - return self._scheduler_class().pick_node(instance_id, **_kwargs) - - @exception.wrap_exception - def run_instance(self, instance_id, **_kwargs): - """ - Picks a node for a running VM and casts the run_instance request - """ - - node = self.pick_node(instance_id, **_kwargs) - - rpc.cast('%s.%s' % (FLAGS.compute_topic, node), - {"method": "run_instance", - "args": {"instance_id": instance_id}}) - logging.debug("Casting to node %s for running instance %s", - node, instance_id) diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py new file mode 100644 index 000000000..6c76fd322 --- /dev/null +++ b/nova/scheduler/simple.py @@ -0,0 +1,81 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Openstack, LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Simple Scheduler +""" + +from nova import db +from nova import flags +from nova.scheduler import driver + +FLAGS = flags.FLAGS +flags.DEFINE_integer("max_instances", 16, + "maximum number of instances to allow per host") +flags.DEFINE_integer("max_volumes", 100, + "maximum number of volumes to allow per host") +flags.DEFINE_integer("max_networks", 1000, + "maximum number of networks to allow per host") + +class SimpleScheduler(driver.Scheduler): + """ + Implements Naive Scheduler that tries to find least loaded host + """ + + def pick_compute_host(self, context, instance_id, **_kwargs): + """ + Picks a host that is up and has the fewest running instances + """ + + results = db.daemon_get_all_compute_sorted(context) + for result in results: + (daemon, instance_count) = result + if instance_count >= FLAGS.max_instances: + raise driver.NoValidHost("All hosts have too many instances") + if self.daemon_is_up(daemon): + return daemon['host'] + raise driver.NoValidHost("No hosts found") + + def pick_volume_host(self, context, volume_id, **_kwargs): + """ + Picks a host that is up and has the fewest volumes + """ + + results = db.daemon_get_all_volume_sorted(context) + for result in results: + (daemon, instance_count) = result + if instance_count >= FLAGS.max_volumes: + raise driver.NoValidHost("All hosts have too many volumes") + if self.daemon_is_up(daemon): + return daemon['host'] + raise driver.NoValidHost("No hosts found") + + def pick_network_host(self, context, network_id, **_kwargs): + """ + Picks a host that is up and has the fewest networks + """ + + results = db.daemon_get_all_network_sorted(context) + for result in results: + (daemon, instance_count) = result + if instance_count >= FLAGS.max_networks: + raise driver.NoValidHost("All hosts have too many networks") + if self.daemon_is_up(daemon): + return daemon['host'] + raise driver.NoValidHost("No hosts found") diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 867b572f3..23013e4c7 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -61,7 +61,7 @@ class ComputeTestCase(test.TrialTestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 - return db.instance_create(None, inst) + return db.instance_create(self.context, inst) @defer.inlineCallbacks def test_run_terminate(self): diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py new file mode 100644 index 000000000..d3616dd6f --- /dev/null +++ b/nova/tests/scheduler_unittest.py @@ -0,0 +1,103 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler +""" +import logging + +from twisted.internet import defer + +from nova import db +from nova import flags +from nova import service +from nova import test +from nova import utils +from nova.auth import manager as auth_manager +from nova.scheduler import manager + + +FLAGS = flags.FLAGS + + +class SchedulerTestCase(test.TrialTestCase): + """Test case for scheduler""" + def setUp(self): # pylint: disable-msg=C0103 + super(SchedulerTestCase, self).setUp() + self.flags(connection_type='fake', + scheduler_driver='nova.scheduler.simple.SimpleScheduler') + self.scheduler = manager.SchedulerManager() + self.context = None + self.manager = auth_manager.AuthManager() + self.user = self.manager.create_user('fake', 'fake', 'fake') + self.project = self.manager.create_project('fake', 'fake', 'fake') + self.context = None + + def tearDown(self): # pylint: disable-msg=C0103 + self.manager.delete_user(self.user) + self.manager.delete_project(self.project) + + def _create_instance(self): + """Create a test instance""" + inst = {} + inst['image_id'] = 'ami-test' + inst['reservation_id'] = 'r-fakeres' + inst['launch_time'] = '10' + inst['user_id'] = self.user.id + inst['project_id'] = self.project.id + inst['instance_type'] = 'm1.tiny' + inst['mac_address'] = utils.generate_mac() + inst['ami_launch_index'] = 0 + return db.instance_create(self.context, inst) + + def test_hosts_are_up(self): + # NOTE(vish): constructing service without create method + # because we are going to use it without queue + service1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + service2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + service1.report_state() + service2.report_state() + hosts = self.scheduler.driver.hosts_up(self.context, 'compute') + self.assertEqual(len(hosts), 2) + + def test_least_busy_host_gets_instance(self): + # NOTE(vish): constructing service without create method + # because we are going to use it without queue + service1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + service2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + service1.report_state() + service2.report_state() + instance_id = self._create_instance() + FLAGS.host = 'host1' + service1.run_instance(self.context, + instance_id) + print type(self.scheduler.driver) + host = self.scheduler.driver.pick_compute_host(self.context, + instance_id) + self.assertEqual(host, 'host2') diff --git a/run_tests.py b/run_tests.py index c47cbe2ec..5d76a74ca 100644 --- a/run_tests.py +++ b/run_tests.py @@ -60,6 +60,7 @@ from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * from nova.tests.rpc_unittest import * +from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * from nova.tests.validator_unittest import * from nova.tests.volume_unittest import * -- cgit From 9fc2bb60f1b280e9bf28d68c20f04de2130bd398 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 15:07:24 -0400 Subject: Use port that boto expects --- nova/tests/api_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 534833fba..8087a2e3b 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -93,7 +93,7 @@ class ApiEc2TestCase(test.BaseTestCase): aws_secret_access_key='fake', is_secure=False, region=regioninfo.RegionInfo(None, 'test', self.host), - port=0, + port=8773, path='/services/Cloud') self.mox.StubOutWithMock(self.ec2, 'new_http_connection') @@ -101,7 +101,7 @@ class ApiEc2TestCase(test.BaseTestCase): def expect_http(self, host=None, is_secure=False): """Returns a new EC2 connection""" http = FakeHttplibConnection( - self.app, '%s:0' % (self.host), False) + self.app, '%s:8773' % (self.host), False) # pylint: disable-msg=E1103 self.ec2.new_http_connection(host, is_secure).AndReturn(http) return http -- cgit From 0fa141a231107da931c396f113b00329d63ee430 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 15:10:55 -0400 Subject: Remove unused APIRequestContext.handler --- nova/api/ec2/context.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/api/ec2/context.py b/nova/api/ec2/context.py index fb3118020..f69747622 100644 --- a/nova/api/ec2/context.py +++ b/nova/api/ec2/context.py @@ -23,8 +23,7 @@ APIRequestContext import random class APIRequestContext(object): - def __init__(self, handler, user, project): - self.handler = handler + def __init__(self, user, project): self.user = user self.project = project self.request_id = ''.join( -- cgit From b360aded9cfeebfd7594b1b649bd2a1573203cd3 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 15:43:55 -0400 Subject: send requests to the main API instead of to the EC2 subset -- so that it can parse out the '/services/' prefix. Also, oops, match on path_info instead of path like we're supposed to. --- nova/api/ec2/__init__.py | 2 +- nova/tests/api_unittest.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index a94bcb863..1722617ae 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -97,7 +97,7 @@ class Router(wsgi.Middleware): def __call__(self, req): # Obtain the appropriate controller and action for this request. try: - match = self.map.match(req.path) + match = self.map.match(req.path_info) controller_name = match['controller_name'] controller = self.controllers[controller_name] except: diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 8087a2e3b..9f9d32784 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -27,8 +27,7 @@ import webob from nova import test from nova.auth import manager -from nova.api import ec2 -from nova.api.ec2 import cloud +from nova import api class FakeHttplibSocket(object): @@ -83,11 +82,10 @@ class ApiEc2TestCase(test.BaseTestCase): super(ApiEc2TestCase, self).setUp() self.manager = manager.AuthManager() - self.cloud = cloud.CloudController() self.host = '127.0.0.1' - self.app = ec2.API() + self.app = api.API() self.ec2 = boto.connect_ec2( aws_access_key_id='fake', aws_secret_access_key='fake', -- cgit From 3075cc7440a37118d7784057874887f751e1f6a3 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 15:53:57 -0400 Subject: OMG got api_unittests to pass --- nova/api/ec2/__init__.py | 11 +++++++---- nova/api/ec2/apirequest.py | 6 +++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 1722617ae..e53e7d964 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -28,6 +28,7 @@ import webob.exc from nova import exception from nova import wsgi +from nova.api.ec2 import apirequest from nova.api.ec2 import context from nova.api.ec2 import admin from nova.api.ec2 import cloud @@ -174,7 +175,7 @@ class Authorizer(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): context = req.environ['ec2.context'] - controller_name = req.environ['ec2.controller'].__name__ + controller_name = req.environ['ec2.controller'].__class__.__name__ action = req.environ['ec2.action'] allowed_roles = self.action_roles[controller_name].get(action, []) if self._matches_any_role(context, allowed_roles): @@ -205,14 +206,16 @@ class Executor(wsgi.Application): action = req.environ['ec2.action'] args = req.environ['ec2.action_args'] - api_request = APIRequest(controller, action) + api_request = apirequest.APIRequest(controller, action) try: - return api_request.send(context, **args) + result = api_request.send(context, **args) + req.headers['Content-Type'] = 'text/xml' + return result except exception.ApiError as ex: return self._error(req, type(ex).__name__ + "." + ex.code, ex.message) # TODO(vish): do something more useful with unknown exceptions except Exception as ex: - return self._error(type(ex).__name__, str(ex)) + return self._error(req, type(ex).__name__, str(ex)) def _error(self, req, code, message): resp = webob.Response() diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 261346a09..85ff2fa5e 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -20,9 +20,14 @@ APIRequest class """ +import logging +import re # TODO(termie): replace minidom with etree from xml.dom import minidom +_log = logging.getLogger("api") +_log.setLevel(logging.DEBUG) + _c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') @@ -79,7 +84,6 @@ class APIRequest(object): result = method(context, **args) - req.headers['Content-Type'] = 'text/xml' return self._render_response(result, context.request_id) def _render_response(self, response_data, request_id): -- cgit From 43f1e722b633945a8f5dca005e6fd60515bac4ae Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 15:55:30 -0400 Subject: Cloud tests pass --- nova/tests/cloud_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 689251300..92e726ffa 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -60,7 +60,7 @@ class CloudTestCase(test.BaseTestCase): except: pass admin = manager.AuthManager().get_user('admin') project = manager.AuthManager().create_project('proj', 'admin', 'proj') - self.context = context.APIRequestContext(handler=None,project=project,user=admin) + self.context = context.APIRequestContext(project=project,user=admin) def tearDown(self): manager.AuthManager().delete_project('proj') -- cgit From 59adf260b59dcdcc6bc2df3260a331a4a05f535c Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Sep 2010 15:59:52 -0400 Subject: Move nova.endpoint.images to api.ec2 and delete nova.endpoint --- nova/api/ec2/cloud.py | 2 +- nova/api/ec2/images.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++ nova/cloudpipe/pipelib.py | 4 +-- nova/endpoint/__init__.py | 0 nova/endpoint/images.py | 80 ----------------------------------------------- nova/endpoint/notes.txt | 62 ------------------------------------ 6 files changed, 83 insertions(+), 145 deletions(-) create mode 100644 nova/api/ec2/images.py delete mode 100644 nova/endpoint/__init__.py delete mode 100644 nova/endpoint/images.py delete mode 100644 nova/endpoint/notes.txt diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index e3122bbfc..5c9e1b170 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -35,7 +35,7 @@ from nova import utils from nova.auth import manager from nova.compute import model from nova.compute.instance_types import INSTANCE_TYPES -from nova.endpoint import images +from nova.api.ec2 import images from nova.network import service as network_service from nova.network import model as network_model from nova.volume import service diff --git a/nova/api/ec2/images.py b/nova/api/ec2/images.py new file mode 100644 index 000000000..cfea4c20b --- /dev/null +++ b/nova/api/ec2/images.py @@ -0,0 +1,80 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Proxy AMI-related calls from the cloud controller, to the running +objectstore daemon. +""" + +import json +import urllib + +import boto.s3.connection + +from nova import image +from nova import flags +from nova import utils +from nova.auth import manager + + +FLAGS = flags.FLAGS + + +def modify(context, image_id, operation): + image.S3ImageService(context)._conn().make_request( + method='POST', + bucket='_images', + query_args=qs({'image_id': image_id, 'operation': operation})) + + return True + + +def register(context, image_location): + """ rpc call to register a new image based from a manifest """ + + image_id = utils.generate_uid('ami') + image.S3ImageService(context)._conn().make_request( + method='PUT', + bucket='_images', + query_args=qs({'image_location': image_location, + 'image_id': image_id})) + + return image_id + + +def list(context, filter_list=[]): + """ return a list of all images that a user can see + + optionally filtered by a list of image_id """ + + result = image.S3ImageService(context).index().values() + if not filter_list is None: + return [i for i in result if i['imageId'] in filter_list] + return result + + +def deregister(context, image_id): + """ unregister an image """ + image.S3ImageService(context).delete(image_id) + + +def qs(params): + pairs = [] + for key in params.keys(): + pairs.append(key + '=' + urllib.quote(params[key])) + return '&'.join(pairs) diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 2867bcb21..97272eda6 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -32,7 +32,7 @@ from nova import exception from nova import flags from nova import utils from nova.auth import manager -from nova.endpoint import api +from nova.api.ec2 import context FLAGS = flags.FLAGS @@ -60,7 +60,7 @@ class CloudPipe(object): key_name = self.setup_keypair(project.project_manager_id, project_id) zippy = open(zippath, "r") - context = api.APIRequestContext(handler=None, user=project.project_manager, project=project) + context = context.APIRequestContext(user=project.project_manager, project=project) reservation = self.controller.run_instances(context, # run instances expects encoded userdata, it is decoded in the get_metadata_call diff --git a/nova/endpoint/__init__.py b/nova/endpoint/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py deleted file mode 100644 index cfea4c20b..000000000 --- a/nova/endpoint/images.py +++ /dev/null @@ -1,80 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Proxy AMI-related calls from the cloud controller, to the running -objectstore daemon. -""" - -import json -import urllib - -import boto.s3.connection - -from nova import image -from nova import flags -from nova import utils -from nova.auth import manager - - -FLAGS = flags.FLAGS - - -def modify(context, image_id, operation): - image.S3ImageService(context)._conn().make_request( - method='POST', - bucket='_images', - query_args=qs({'image_id': image_id, 'operation': operation})) - - return True - - -def register(context, image_location): - """ rpc call to register a new image based from a manifest """ - - image_id = utils.generate_uid('ami') - image.S3ImageService(context)._conn().make_request( - method='PUT', - bucket='_images', - query_args=qs({'image_location': image_location, - 'image_id': image_id})) - - return image_id - - -def list(context, filter_list=[]): - """ return a list of all images that a user can see - - optionally filtered by a list of image_id """ - - result = image.S3ImageService(context).index().values() - if not filter_list is None: - return [i for i in result if i['imageId'] in filter_list] - return result - - -def deregister(context, image_id): - """ unregister an image """ - image.S3ImageService(context).delete(image_id) - - -def qs(params): - pairs = [] - for key in params.keys(): - pairs.append(key + '=' + urllib.quote(params[key])) - return '&'.join(pairs) diff --git a/nova/endpoint/notes.txt b/nova/endpoint/notes.txt deleted file mode 100644 index 3e48f1749..000000000 --- a/nova/endpoint/notes.txt +++ /dev/null @@ -1,62 +0,0 @@ -bin/nova-api: - somehow listens for 'cloud_topic' rpc messages and ties them to - the cloud controller (maybe so internal calls can hit the API - via Queuing instead of via HTTP?) - hands CloudController and AdminController to APIServerApplication - and hands that to Tornado. - - -api.py: - -APIServerApplication(tornado.web.Application) - maps routes to APIRequestHandler, CloudPipRequestHandler, MetadataRequestHandler, - RootRequestHandler(just lists versions) - (and to controllers which are passed to __init__) - magical twisted mapping to it - -APIRequestHandler - execute: - authenticates request - picks controller from APIServerApplication's list based on name that was at the - start of the URL (e.g. /services/Cloud has /services mapped here via - APIServerApplication then Cloud is controller_name) - picks action from incoming request arguments - dict = APIRequest(controller, action).send(Context(user, project)) - _write_callback(dict) - self.finish() - -APIRequest - send(context, **kwargs): - dict = controller.action(context, **kwargs) - return _render_response(dict) # turns into XML - - -CloudController and AdminController: - actions return dict (or True which is converted into dict(return=True)) - actions have @rbac.allow('list', 'of', 'roles', 'or', '"all"') - actions can have @defer.inlineCallbacks which is used for yield statements - can use rpc.cast and then defer a returnValue - - -==== STRATEGY TO CONVERT TO EVENTLET+WSGI ==== - -* Controllers: -x move the @rbac.allow data into an auth WSGI that is right above the call -x to the controller -x verify @defer.inlineCallbacks is just to allow the yield statements, then -x remove the yield statements (untangle from twisted) - -* nova-api: - verify that cloud_topic is going away which I seem to remember, so we can ignore rpc - -* apiserverapplication: -x replace with a Router to a wsgi.Controller -x apirequesthandler stuff is just an entry in api.APIRouter - -* apirequesthandler -x wsgi.Controller pointed to by api.APIRouter -x - basically it's execute() from old APIRequestHandler -x change to return data directly instead of _write_callback() and finish() - -* apirequest -x doesn't need to change -- cgit From 68d8f54e00c153eccd426256a25c8a70ccce2dcc Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 15:15:39 -0700 Subject: test for too many instances work --- nova/scheduler/driver.py | 21 +++++++------- nova/scheduler/simple.py | 25 ++++++++-------- nova/tests/scheduler_unittest.py | 63 ++++++++++++++++------------------------ 3 files changed, 48 insertions(+), 61 deletions(-) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 1618342c0..830f05b13 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -28,7 +28,7 @@ from nova import exception from nova import flags FLAGS = flags.FLAGS -flags.DEFINE_integer('daemon_down_time', +flags.DEFINE_integer('service_down_time', 60, 'seconds without heartbeat that determines a ' 'compute node to be down') @@ -43,20 +43,21 @@ class Scheduler(object): """ @staticmethod - def daemon_is_up(daemon): + def service_is_up(service): """ - Given a daemon, return whether the deamon is considered 'up' by + Given a service, return whether the service is considered 'up' by if it's sent a heartbeat recently """ - elapsed = datetime.datetime.now() - daemon['updated_at'] - return elapsed < datetime.timedelta(seconds=FLAGS.daemon_down_time) + last_heartbeat = service['updated_at'] or service['created_at'] + elapsed = datetime.datetime.now() - last_heartbeat + return elapsed < datetime.timedelta(seconds=FLAGS.service_down_time) def hosts_up(self, context, topic): """ - Return the list of hosts that have a running daemon for topic + Return the list of hosts that have a running service for topic """ - daemons = db.daemon_get_all_by_topic(context, topic) - return [daemon.host - for daemon in daemons - if self.daemon_is_up(daemon)] + services = db.service_get_all_by_topic(context, topic) + return [service.host + for service in services + if self.service_is_up(service)] diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 294dc1118..832417208 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -43,14 +43,13 @@ class SimpleScheduler(driver.Scheduler): Picks a host that is up and has the fewest running instances """ - results = db.daemon_get_all_compute_sorted(context) + results = db.service_get_all_compute_sorted(context) for result in results: - (daemon, instance_count) = result - print daemon.host, instance_count + (service, instance_count) = result if instance_count >= FLAGS.max_instances: raise driver.NoValidHost("All hosts have too many instances") - if self.daemon_is_up(daemon): - return daemon['host'] + if self.service_is_up(service): + return service['host'] raise driver.NoValidHost("No hosts found") def pick_volume_host(self, context, volume_id, **_kwargs): @@ -58,13 +57,13 @@ class SimpleScheduler(driver.Scheduler): Picks a host that is up and has the fewest volumes """ - results = db.daemon_get_all_volume_sorted(context) + results = db.service_get_all_volume_sorted(context) for result in results: - (daemon, instance_count) = result + (service, instance_count) = result if instance_count >= FLAGS.max_volumes: raise driver.NoValidHost("All hosts have too many volumes") - if self.daemon_is_up(daemon): - return daemon['host'] + if self.service_is_up(service): + return service['host'] raise driver.NoValidHost("No hosts found") def pick_network_host(self, context, network_id, **_kwargs): @@ -72,11 +71,11 @@ class SimpleScheduler(driver.Scheduler): Picks a host that is up and has the fewest networks """ - results = db.daemon_get_all_network_sorted(context) + results = db.service_get_all_network_sorted(context) for result in results: - (daemon, instance_count) = result + (service, instance_count) = result if instance_count >= FLAGS.max_networks: raise driver.NoValidHost("All hosts have too many networks") - if self.daemon_is_up(daemon): - return daemon['host'] + if self.service_is_up(service): + return service['host'] raise driver.NoValidHost("No hosts found") diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 45ffac438..bdd77713a 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -18,9 +18,6 @@ """ Tests For Scheduler """ -import logging - -from twisted.internet import defer from nova import db from nova import flags @@ -36,10 +33,10 @@ FLAGS = flags.FLAGS flags.DECLARE('max_instances', 'nova.scheduler.simple') -class SchedulerTestCase(test.TrialTestCase): +class SimpleSchedulerTestCase(test.TrialTestCase): """Test case for scheduler""" def setUp(self): # pylint: disable-msg=C0103 - super(SchedulerTestCase, self).setUp() + super(SimpleSchedulerTestCase, self).setUp() self.flags(connection_type='fake', max_instances=4, scheduler_driver='nova.scheduler.simple.SimpleScheduler') @@ -49,10 +46,20 @@ class SchedulerTestCase(test.TrialTestCase): self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = None + self.service1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + self.service2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) def tearDown(self): # pylint: disable-msg=C0103 self.manager.delete_user(self.user) self.manager.delete_project(self.project) + self.service1.kill() + self.service2.kill() def _create_instance(self): """Create a test instance""" @@ -70,53 +77,33 @@ class SchedulerTestCase(test.TrialTestCase): def test_hosts_are_up(self): # NOTE(vish): constructing service without create method # because we are going to use it without queue - service1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - service2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - hosts = self.scheduler.driver.hosts_up(self.context, 'compute') - self.assertEqual(len(hosts), 0) - service1.report_state() - service2.report_state() hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(len(hosts), 2) def test_least_busy_host_gets_instance(self): - service1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - service2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - service1.report_state() - service2.report_state() instance_id = self._create_instance() - service1.run_instance(self.context, instance_id) + self.service1.run_instance(self.context, instance_id) host = self.scheduler.driver.pick_compute_host(self.context, instance_id) self.assertEqual(host, 'host2') - service1.terminate_instance(self.context, instance_id) + self.service1.terminate_instance(self.context, instance_id) def test_too_many_instances(self): - service1 = service.Service('host', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - instance_ids = [] + instance_ids1 = [] + instance_ids2 = [] for index in xrange(FLAGS.max_instances): instance_id = self._create_instance() - service1.run_instance(self.context, instance_id) - instance_ids.append(instance_id) + self.service1.run_instance(self.context, instance_id) + instance_ids1.append(instance_id) + instance_id = self._create_instance() + self.service2.run_instance(self.context, instance_id) + instance_ids2.append(instance_id) instance_id = self._create_instance() self.assertRaises(driver.NoValidHost, self.scheduler.driver.pick_compute_host, self.context, instance_id) - for instance_id in instance_ids: - service1.terminate_instance(self.context, instance_id) + for instance_id in instance_ids1: + self.service1.terminate_instance(self.context, instance_id) + for instance_id in instance_ids2: + self.service2.terminate_instance(self.context, instance_id) -- cgit From e555ce94ee29013901796b570b752f39194ddb12 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 2 Sep 2010 17:06:01 -0700 Subject: send ultimate topic in to scheduler --- nova/endpoint/cloud.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 2c88ef406..1ff22042a 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -567,6 +567,7 @@ class CloudController(object): rpc.cast(FLAGS.scheduler_topic, {"method": "run_instance", "args": {"context": None, + "topic": FLAGS.compute_topic, "instance_id": inst_id}}) logging.debug("Casting to scheduler for %s/%s's instance %s" % (context.project.name, context.user.name, inst_id)) -- cgit From 22aa51638dc221e78de60f7e2ddb10eb0ddf4db3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 3 Sep 2010 00:53:41 -0700 Subject: removed extra file and updated sql note --- bin/nova-listinstances | 38 -------------------------------------- nova/db/sqlalchemy/api.py | 1 + 2 files changed, 1 insertion(+), 38 deletions(-) delete mode 100755 bin/nova-listinstances diff --git a/bin/nova-listinstances b/bin/nova-listinstances deleted file mode 100755 index 386283d2f..000000000 --- a/bin/nova-listinstances +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Openstack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -# Duplicates the functionality of euca-describe-instances, but doesn't require -# going through the API. Does a direct query to the datastore. This is -# mostly a test program written for the scheduler -# - -""" -List instances by doing a direct query to the datastore -""" - -from nova.compute import model - -data_needed = ['image_id', 'memory_kb', 'local_gb', 'node_name', 'vcpus'] - -instances = model.InstanceDirectory().all - -for instance in instances: - print 'Instance: %s' % instance['instance_id'] - for x in data_needed: - print ' %s: %s' % (x, instance[x]) - diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 4ae55eaf4..4fa85b74b 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -69,6 +69,7 @@ def service_get_all_compute_sorted(context): # FROM services LEFT OUTER JOIN # (SELECT host, count(*) AS instance_count # FROM instances GROUP BY host) AS inst_count + # ON services.host == inst_count.host topic = 'compute' label = 'instance_count' subq = session.query(models.Instance.host, -- cgit From a983660008d09276d2749077c1141313381d6eb6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 4 Sep 2010 11:42:15 -0700 Subject: removed extra equals --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 4fa85b74b..cb94023f5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -69,7 +69,7 @@ def service_get_all_compute_sorted(context): # FROM services LEFT OUTER JOIN # (SELECT host, count(*) AS instance_count # FROM instances GROUP BY host) AS inst_count - # ON services.host == inst_count.host + # ON services.host = inst_count.host topic = 'compute' label = 'instance_count' subq = session.query(models.Instance.host, -- cgit From 0cbde24bfc3ad13a67325e6c0e08d4650d05ea05 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 7 Sep 2010 16:00:01 -0400 Subject: Oops, APIRequestContext's signature has changed --- nova/objectstore/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 49c647b4e..aabf6831f 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -131,7 +131,7 @@ def get_context(request): request.uri, headers=request.getAllHeaders(), check_type='s3') - return context.APIRequestContext(None, user, project) + return context.APIRequestContext(user, project) except exception.Error as ex: logging.debug("Authentication Failure: %s", ex) raise exception.NotAuthorized -- cgit From 71566b41619166f61a3fe478524f66908364364b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 13:01:21 -0700 Subject: fix docstrings and formatting --- nova/scheduler/chance.py | 8 ++------ nova/scheduler/driver.py | 25 +++++++------------------ nova/scheduler/manager.py | 4 +--- nova/scheduler/simple.py | 19 ++++++------------- 4 files changed, 16 insertions(+), 40 deletions(-) diff --git a/nova/scheduler/chance.py b/nova/scheduler/chance.py index 1054cdbf5..7fd09b053 100644 --- a/nova/scheduler/chance.py +++ b/nova/scheduler/chance.py @@ -27,14 +27,10 @@ from nova.scheduler import driver class ChanceScheduler(driver.Scheduler): - """ - Implements Scheduler as a random node selector - """ + """Implements Scheduler as a random node selector.""" def schedule(self, context, topic, *_args, **_kwargs): - """ - Picks a host that is up at random - """ + """Picks a host that is up at random.""" hosts = self.hosts_up(context, topic) if not hosts: diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index f5872e9c8..2e6a5a835 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -28,34 +28,25 @@ from nova import exception from nova import flags FLAGS = flags.FLAGS -flags.DEFINE_integer('service_down_time', - 60, - 'seconds without heartbeat that determines a ' - 'compute node to be down') +flags.DEFINE_integer('service_down_time', 60, + 'maximum time since last checkin for up service') class NoValidHost(exception.Error): - """There is no valid host for the command""" + """There is no valid host for the command.""" pass class Scheduler(object): - """ - The base class that all Scheduler clases should inherit from - """ + """The base class that all Scheduler clases should inherit from.""" @staticmethod def service_is_up(service): - """ - Given a service, return whether the service is considered 'up' by - if it's sent a heartbeat recently - """ + """Check whether a service is up based on last heartbeat.""" last_heartbeat = service['updated_at'] or service['created_at'] elapsed = datetime.datetime.now() - last_heartbeat return elapsed < datetime.timedelta(seconds=FLAGS.service_down_time) def hosts_up(self, context, topic): - """ - Return the list of hosts that have a running service for topic - """ + """Return the list of hosts that have a running service for topic.""" services = db.service_get_all_by_topic(context, topic) return [service.host @@ -63,7 +54,5 @@ class Scheduler(object): if self.service_is_up(service)] def schedule(self, context, topic, *_args, **_kwargs): - """ - Must override at least this method for scheduler to work - """ + """Must override at least this method for scheduler to work.""" raise NotImplementedError("Must implement a fallback schedule") diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 1755a6fef..1cabd82c6 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -37,9 +37,7 @@ flags.DEFINE_string('scheduler_driver', class SchedulerManager(manager.Manager): - """ - Chooses a host to run instances on. - """ + """Chooses a host to run instances on.""" def __init__(self, scheduler_driver=None, *args, **kwargs): if not scheduler_driver: scheduler_driver = FLAGS.scheduler_driver diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index d10ddabac..ea4eef98e 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -35,14 +35,10 @@ flags.DEFINE_integer("max_networks", 1000, "maximum number of networks to allow per host") class SimpleScheduler(chance.ChanceScheduler): - """ - Implements Naive Scheduler that tries to find least loaded host - """ + """Implements Naive Scheduler that tries to find least loaded host.""" def schedule_run_instance(self, context, _instance_id, *_args, **_kwargs): - """ - Picks a host that is up and has the fewest running instances - """ + """Picks a host that is up and has the fewest running instances.""" results = db.service_get_all_compute_sorted(context) for result in results: @@ -54,9 +50,7 @@ class SimpleScheduler(chance.ChanceScheduler): raise driver.NoValidHost("No hosts found") def schedule_create_volume(self, context, _volume_id, *_args, **_kwargs): - """ - Picks a host that is up and has the fewest volumes - """ + """Picks a host that is up and has the fewest volumes.""" results = db.service_get_all_volume_sorted(context) for result in results: @@ -67,10 +61,9 @@ class SimpleScheduler(chance.ChanceScheduler): return service['host'] raise driver.NoValidHost("No hosts found") - def schedule_set_network_host(self, context, _network_id, *_args, **_kwargs): - """ - Picks a host that is up and has the fewest networks - """ + def schedule_set_network_host(self, context, _network_id, + *_args, **_kwargs): + """Picks a host that is up and has the fewest networks.""" results = db.service_get_all_network_sorted(context) for result in results: -- cgit From 4ddcc28c944975c6e03cccf7d360b3ec4c26e848 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 7 Sep 2010 16:01:26 -0400 Subject: New version of eventlet handles Twisted & eventlet running at the same time --- tools/pip-requires | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pip-requires b/tools/pip-requires index 13e8e5f45..79897c22b 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -6,7 +6,7 @@ amqplib==0.6.1 anyjson==0.2.4 boto==2.0b1 carrot==0.10.5 -eventlet==0.9.10 +eventlet==0.9.12 lockfile==0.8 python-daemon==1.5.5 python-gflags==1.3 -- cgit From c3531537aef54b2c27a6e1f28308eac98aec08ba Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 18:32:08 -0700 Subject: whitespace fixes --- nova/process.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/nova/process.py b/nova/process.py index 069310802..259e62358 100644 --- a/nova/process.py +++ b/nova/process.py @@ -35,8 +35,8 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to +# This is based on _BackRelay from twister.internal.utils, but modified to +# capture both stdout and stderr, without odd stderr handling, and also to # handle stdin class BackRelayWithInput(protocol.ProcessProtocol): """ @@ -46,21 +46,21 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_ProcessExecutionError} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_ProcessExecutionError} which C{deferred} fires - with earlier in this case so that users can determine when the process + @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are + received over stderr, this attribute will refer to a L{Deferred} which + will be called back when the process ends. This C{Deferred} is also + associated with the L{_ProcessExecutionError} which C{deferred} fires + with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, cmd, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): + def __init__(self, deferred, cmd, started_deferred=None, + terminate_on_stderr=False, check_exit_code=True, + process_input=None): self.deferred = deferred self.cmd = cmd self.stdout = StringIO.StringIO() @@ -70,12 +70,12 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + def _build_execution_error(self, exit_code=None): - return ProcessExecutionError( cmd=self.cmd, - exit_code=exit_code, - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue()) + return ProcessExecutionError(cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) def errReceived(self, text): self.stderr.write(text) @@ -101,7 +101,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.deferred.callback((stdout, stderr)) except: # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be + # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: @@ -115,8 +115,8 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.transport.write(self.process_input) self.transport.closeStdin() -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, +def get_process_output(executable, args=None, env=None, path=None, + process_reactor=None, check_exit_code=True, process_input=None, started_deferred=None, terminate_on_stderr=False): if process_reactor is None: @@ -130,8 +130,8 @@ def get_process_output(executable, args=None, env=None, path=None, process_handler = BackRelayWithInput( deferred, cmd, - started_deferred=started_deferred, - check_exit_code=check_exit_code, + started_deferred=started_deferred, + check_exit_code=check_exit_code, process_input=process_input, terminate_on_stderr=terminate_on_stderr) # NOTE(vish): commands come in as unicode, but self.executes needs @@ -139,7 +139,7 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, + process_reactor.spawnProcess( process_handler, executable, (executable,)+tuple(args), env, path) return deferred -- cgit From 6591ac066f1c6f7ca74c540fe5f39033fb41cd10 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 18:32:31 -0700 Subject: one more whitespace fix --- nova/process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/process.py b/nova/process.py index 259e62358..c3b077dc2 100644 --- a/nova/process.py +++ b/nova/process.py @@ -139,8 +139,8 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, - (executable,)+tuple(args), env, path) + process_reactor.spawnProcess(process_handler, executable, + (executable,)+tuple(args), env, path) return deferred -- cgit From fc5e1c6f0bee14fdb85ad138324062ceaa598eee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 21:53:40 -0700 Subject: a few formatting fixes and moved exception --- nova/exception.py | 12 ++++++++++++ nova/process.py | 4 ++-- nova/utils.py | 17 +++++------------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index 29bcb17f8..b8894758f 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -26,6 +26,18 @@ import sys import traceback +class ProcessExecutionError(IOError): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( + description, cmd, exit_code, stdout, stderr) + IOError.__init__(self, message) + + class Error(Exception): def __init__(self, message=None): super(Error, self).__init__(message) diff --git a/nova/process.py b/nova/process.py index c3b077dc2..5a5d8cbd2 100644 --- a/nova/process.py +++ b/nova/process.py @@ -29,7 +29,7 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags -from nova.utils import ProcessExecutionError +from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, @@ -126,7 +126,7 @@ def get_process_output(executable, args=None, env=None, path=None, deferred = defer.Deferred() cmd = executable if args: - cmd = cmd + " " + ' '.join(args) + cmd = " ".join([cmd] + args) process_handler = BackRelayWithInput( deferred, cmd, diff --git a/nova/utils.py b/nova/utils.py index b8abb5388..d302412ad 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -31,21 +31,12 @@ import sys from nova import exception from nova import flags +from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -class ProcessExecutionError(IOError): - def __init__( self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - if description is None: - description = "Unexpected error while running command." - if exit_code is None: - exit_code = '-' - message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( - description, cmd, exit_code, stdout, stderr) - IOError.__init__(self, message) def import_class(import_str): """Returns a class from a string including module and class""" @@ -118,8 +109,10 @@ def runthis(prompt, cmd, check_exit_code = True): exit_code = subprocess.call(cmd.split(" ")) logging.debug(prompt % (exit_code)) if check_exit_code and exit_code <> 0: - raise Exception( "Unexpected exit code: %s from cmd: %s" - % (exit_code, cmd)) + raise ProcessExecutionError(exit_code=exit_code, + stdout=None, + stderr=None, + cmd=cmd) def generate_uid(topic, size=8): -- cgit From 83402810be11111e3f61f3a9c3771bb96161e551 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 02:30:07 -0700 Subject: put soren's fancy path code in scheduler bin as well --- bin/nova-scheduler | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 97f98b17f..38a8f213f 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -21,6 +21,17 @@ Twistd daemon for the nova scheduler nodes. """ +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from nova import service from nova import twistd -- cgit From 6c4d301eab48b841b4b6ca19a96b3e9748f27b57 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 09:52:24 -0700 Subject: fix logging for scheduler to properly display method name --- nova/scheduler/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 1cabd82c6..0ad7ca86b 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -63,4 +63,4 @@ class SchedulerManager(manager.Manager): rpc.cast(db.queue_get_for(context, topic, host), {"method": method, "args": kwargs}) - logging.debug("Casting to %s %s for %s", topic, host, self.method) + logging.debug("Casting to %s %s for %s", topic, host, method) -- cgit From 0aabb8a6febca8d98a750d1bdc78f3160b9684fe Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 13:40:18 -0700 Subject: mocking out quotas --- nova/db/sqlalchemy/models.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 679a44d21..2fcade7de 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -266,6 +266,20 @@ class Volume(BASE, NovaBase): attach_status = Column(String(255)) # TODO(vish): enum +class Quota(BASE, NovaBase): + """Represents quota overrides for a project""" + __tablename__ = 'quotas' + id = Column(Integer, primary_key=True) + + project_id = Column(String(255)) + + instances = Column(Integer) + cores = Column(Integer) + volumes = Column(Integer) + gigabytes = Column(Integer) + floating_ips = Column(Integer) + + class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on""" __tablename__ = 'export_devices' -- cgit From 345749f514291928913a1ecb280b92daec2c0553 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 9 Sep 2010 19:23:27 -0400 Subject: Correct style issues brought up in termie's review --- nova/api/ec2/__init__.py | 37 +++++++++++++++++++------------------ nova/api/ec2/apirequest.py | 2 -- nova/api/ec2/cloud.py | 15 +++++++-------- nova/api/ec2/context.py | 1 + nova/api/ec2/images.py | 21 +++++++-------------- nova/image/service.py | 7 +++++++ nova/tests/api_unittest.py | 1 - 7 files changed, 41 insertions(+), 43 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index e53e7d964..d500b127c 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -16,9 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Starting point for routing EC2 requests -""" +"""Starting point for routing EC2 requests""" import logging import routes @@ -40,6 +38,7 @@ _log.setLevel(logging.DEBUG) class API(wsgi.Middleware): + """Routing for all EC2 API requests.""" def __init__(self): @@ -47,6 +46,7 @@ class API(wsgi.Middleware): class Authenticate(wsgi.Middleware): + """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" @webob.dec.wsgify @@ -65,28 +65,25 @@ class Authenticate(wsgi.Middleware): # Authenticate the request. try: (user, project) = manager.AuthManager().authenticate( - access, - signature, - auth_params, - req.method, - req.host, - req.path - ) - + access, + signature, + auth_params, + req.method, + req.host, + req.path) except exception.Error, ex: logging.debug("Authentication Failure: %s" % ex) raise webob.exc.HTTPForbidden() # Authenticated! req.environ['ec2.context'] = context.APIRequestContext(user, project) - return self.application class Router(wsgi.Middleware): - """ - Add 'ec2.controller', 'ec2.action', and 'ec2.action_args' to WSGI environ. - """ + + """Add ec2.'controller', .'action', and .'action_args' to WSGI environ.""" + def __init__(self, application): super(Router, self).__init__(application) self.map = routes.Mapper() @@ -121,12 +118,13 @@ class Router(wsgi.Middleware): req.environ['ec2.controller'] = controller req.environ['ec2.action'] = action req.environ['ec2.action_args'] = args - return self.application class Authorizer(wsgi.Middleware): - """ + + """Authorize an EC2 API request. + Return a 401 if ec2.controller and ec2.action in WSGI environ may not be executed in ec2.context. """ @@ -194,11 +192,14 @@ class Authorizer(wsgi.Middleware): class Executor(wsgi.Application): - """ + + """Execute an EC2 API request. + Executes 'ec2.action' upon 'ec2.controller', passing 'ec2.context' and 'ec2.action_args' (all variables in WSGI environ.) Returns an XML response, or a 400 upon failure. """ + @webob.dec.wsgify def __call__(self, req): context = req.environ['ec2.context'] diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 85ff2fa5e..a3b20118f 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -51,7 +51,6 @@ class APIRequest(object): self.action = action def send(self, context, **kwargs): - try: method = getattr(self.controller, _camelcase_to_underscore(self.action)) @@ -83,7 +82,6 @@ class APIRequest(object): args[key] = [v for k, v in s] result = method(context, **args) - return self._render_response(result, context.request_id) def _render_response(self, response_data, request_id): diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 5c9e1b170..e1e04ca90 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -205,8 +205,8 @@ class CloudController(object): def create_key_pair(self, context, key_name, **kwargs): data = _gen_key(context.user.id, key_name) return {'keyName': key_name, - 'keyFingerprint': data['fingerprint'], - 'keyMaterial': data['private_key']} + 'keyFingerprint': data['fingerprint'], + 'keyMaterial': data['private_key']} def delete_key_pair(self, context, key_name, **kwargs): context.user.delete_key_pair(key_name) @@ -273,10 +273,11 @@ class CloudController(object): def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it - result = rpc.call(FLAGS.volume_topic, {"method": "create_volume", - "args": {"size": size, - "user_id": context.user.id, - "project_id": context.project.id}}) + result = rpc.call(FLAGS.volume_topic, + {"method": "create_volume", + "args": {"size": size, + "user_id": context.user.id, + "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary volume = self._get_volume(context, result) return {'volumeSet': [self.format_volume(context, volume)]} @@ -638,7 +639,6 @@ class CloudController(object): image_location = kwargs['name'] image_id = images.register(context, image_location) logging.debug("Registered %s as %s" % (image_location, image_id)) - return {'imageId': image_id} def describe_image_attribute(self, context, image_id, attribute, **kwargs): @@ -682,5 +682,4 @@ class CloudController(object): aggregate_state['pending'].has_key(item_id)): del aggregate_state['pending'][item_id] aggregate_state[node_name] = items - return True diff --git a/nova/api/ec2/context.py b/nova/api/ec2/context.py index f69747622..c53ba98d9 100644 --- a/nova/api/ec2/context.py +++ b/nova/api/ec2/context.py @@ -22,6 +22,7 @@ APIRequestContext import random + class APIRequestContext(object): def __init__(self, user, project): self.user = user diff --git a/nova/api/ec2/images.py b/nova/api/ec2/images.py index cfea4c20b..f0be7b899 100644 --- a/nova/api/ec2/images.py +++ b/nova/api/ec2/images.py @@ -26,20 +26,20 @@ import urllib import boto.s3.connection -from nova import image from nova import flags from nova import utils from nova.auth import manager +from nova.image import service FLAGS = flags.FLAGS def modify(context, image_id, operation): - image.S3ImageService(context)._conn().make_request( + service.S3ImageService(context)._conn().make_request( method='POST', bucket='_images', - query_args=qs({'image_id': image_id, 'operation': operation})) + query_args=service.qs({'image_id': image_id, 'operation': operation})) return True @@ -48,10 +48,10 @@ def register(context, image_location): """ rpc call to register a new image based from a manifest """ image_id = utils.generate_uid('ami') - image.S3ImageService(context)._conn().make_request( + service.S3ImageService(context)._conn().make_request( method='PUT', bucket='_images', - query_args=qs({'image_location': image_location, + query_args=service.qs({'image_location': image_location, 'image_id': image_id})) return image_id @@ -62,7 +62,7 @@ def list(context, filter_list=[]): optionally filtered by a list of image_id """ - result = image.S3ImageService(context).index().values() + result = service.S3ImageService(context).index().values() if not filter_list is None: return [i for i in result if i['imageId'] in filter_list] return result @@ -70,11 +70,4 @@ def list(context, filter_list=[]): def deregister(context, image_id): """ unregister an image """ - image.S3ImageService(context).delete(image_id) - - -def qs(params): - pairs = [] - for key in params.keys(): - pairs.append(key + '=' + urllib.quote(params[key])) - return '&'.join(pairs) + service.S3ImageService(context).delete(image_id) diff --git a/nova/image/service.py b/nova/image/service.py index 25e4bb675..f6719caec 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -133,3 +133,10 @@ class S3ImageService(ImageService): calling_format=calling, port=FLAGS.s3_port, host=FLAGS.s3_host) + + +def qs(params): + pairs = [] + for key in params.keys(): + pairs.append(key + '=' + urllib.quote(params[key])) + return '&'.join(pairs) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 9f9d32784..ffc78e71d 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -63,7 +63,6 @@ class FakeHttplibConnection(object): # For some reason, the response doesn't have "HTTP/1.0 " prepended; I # guess that's a function the web server usually provides. resp = "HTTP/1.0 %s" % resp - sock = FakeHttplibSocket(resp) self.http_response = httplib.HTTPResponse(sock) self.http_response.begin() -- cgit From c577e91ee3a3eb87a393da2449cab95069a785f4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 20:10:31 -0700 Subject: database support for quotas --- nova/db/api.py | 27 +++++++++++++++++++++ nova/db/sqlalchemy/api.py | 43 +++++++++++++++++++++++++++++++++- nova/db/sqlalchemy/models.py | 21 +++++++++++++++++ nova/endpoint/cloud.py | 53 +++++++++++++++++++++++++++++++++++++----- nova/tests/compute_unittest.py | 1 + run_tests.py | 1 + 6 files changed, 139 insertions(+), 7 deletions(-) diff --git a/nova/db/api.py b/nova/db/api.py index d81673fad..c22c84768 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -195,6 +195,10 @@ def instance_create(context, values): return IMPL.instance_create(context, values) +def instance_data_get_for_project(context, project_id): + """Get (instance_count, core_count) for project.""" + return IMPL.instance_data_get_for_project(context, project_id) + def instance_destroy(context, instance_id): """Destroy the instance or raise if it does not exist.""" return IMPL.instance_destroy(context, instance_id) @@ -379,6 +383,29 @@ def export_device_create(context, values): ################### +def quota_create(context, values): + """Create a quota from the values dictionary.""" + return IMPL.quota_create(context, values) + + +def quota_get(context, project_id): + """Retrieve a quota or raise if it does not exist.""" + return IMPL.quota_get(context, project_id) + + +def quota_update(context, project_id, values): + """Update a quota from the values dictionary.""" + return IMPL.quota_update(context, project_id, values) + + +def quota_destroy(context, project_id): + """Destroy the quota or raise if it does not exist.""" + return IMPL.quota_destroy(context, project_id) + + +################### + + def volume_allocate_shelf_and_blade(context, volume_id): """Atomically allocate a free shelf and blade from the pool.""" return IMPL.volume_allocate_shelf_and_blade(context, volume_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 4ea7a9071..4b01725ce 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -26,6 +26,7 @@ from nova.db.sqlalchemy import models from nova.db.sqlalchemy.session import get_session from sqlalchemy import or_ from sqlalchemy.orm import joinedload_all +from sqlalchemy.sql import func FLAGS = flags.FLAGS @@ -264,6 +265,15 @@ def instance_create(_context, values): return instance_ref.id +def instance_data_get_for_project(_context, project_id): + session = get_session() + return session.query(func.count(models.Instance.id), + func.sum(models.Instance.vcpus) + ).filter_by(project_id=project_id + ).filter_by(deleted=False + ).first() + + def instance_destroy(_context, instance_id): session = get_session() with session.begin(): @@ -534,6 +544,37 @@ def export_device_create(_context, values): ################### +def quota_create(_context, values): + quota_ref = models.Quota() + for (key, value) in values.iteritems(): + quota_ref[key] = value + quota_ref.save() + return quota_ref + + +def quota_get(_context, project_id): + return models.Quota.find_by_str(project_id) + + +def quota_update(_context, project_id, values): + session = get_session() + with session.begin(): + quota_ref = models.Quota.find_by_str(project_id, session=session) + for (key, value) in values.iteritems(): + quota_ref[key] = value + quota_ref.save(session=session) + + +def quota_destroy(_context, project_id): + session = get_session() + with session.begin(): + quota_ref = models.Quota.find_by_str(project_id, session=session) + quota_ref.delete(session=session) + + +################### + + def volume_allocate_shelf_and_blade(_context, volume_id): session = get_session() with session.begin(): @@ -621,7 +662,7 @@ def volume_get_instance(_context, volume_id): def volume_get_shelf_and_blade(_context, volume_id): session = get_session() - export_device = session.query(models.ExportDevice + export_device = session.query(models.exportdevice ).filter_by(volume_id=volume_id ).first() if not export_device: diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 2fcade7de..7f510301a 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -222,6 +222,11 @@ class Instance(BASE, NovaBase): state = Column(Integer) state_description = Column(String(255)) + memory_mb = Column(Integer) + vcpus = Column(Integer) + local_gb = Column(Integer) + + hostname = Column(String(255)) host = Column(String(255)) # , ForeignKey('hosts.id')) @@ -279,6 +284,22 @@ class Quota(BASE, NovaBase): gigabytes = Column(Integer) floating_ips = Column(Integer) + @property + def str_id(self): + return self.project_id + + @classmethod + def find_by_str(cls, str_id, session=None, deleted=False): + if not session: + session = get_session() + try: + return session.query(cls + ).filter_by(project_id=str_id + ).filter_by(deleted=deleted + ).one() + except exc.NoResultFound: + new_exc = exception.NotFound("No model for project_id %s" % str_id) + raise new_exc.__class__, new_exc, sys.exc_info()[2] class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on""" diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 2866474e6..b8a00075b 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -32,6 +32,7 @@ from twisted.internet import defer from nova import db from nova import exception from nova import flags +from nova import quota from nova import rpc from nova import utils from nova.auth import rbac @@ -44,6 +45,11 @@ FLAGS = flags.FLAGS flags.DECLARE('storage_availability_zone', 'nova.volume.manager') +class QuotaError(exception.ApiError): + """Quota Exceeeded""" + pass + + def _gen_key(user_id, key_name): """ Tuck this into AuthManager """ try: @@ -276,6 +282,14 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def create_volume(self, context, size, **kwargs): + # check quota + size = int(size) + if quota.allowed_volumes(context, 1, size) < 1: + logging.warn("Quota exceeeded for %s, tried to create %sG volume", + context.project.id, size) + raise QuotaError("Volume quota exceeded. You cannot " + "create a volume of size %s" % + size) vol = {} vol['size'] = size vol['user_id'] = context.user.id @@ -435,6 +449,12 @@ class CloudController(object): @rbac.allow('netadmin') @defer.inlineCallbacks def allocate_address(self, context, **kwargs): + # check quota + if quota.allowed_floating_ips(context, 1) < 1: + logging.warn("Quota exceeeded for %s, tried to allocate address", + context.project.id) + raise QuotaError("Address quota exceeded. You cannot " + "allocate any more addresses") network_topic = yield self._get_network_topic(context) public_ip = yield rpc.call(network_topic, {"method": "allocate_floating_ip", @@ -487,14 +507,30 @@ class CloudController(object): host = network_ref['host'] if not host: host = yield rpc.call(FLAGS.network_topic, - {"method": "set_network_host", - "args": {"context": None, - "project_id": context.project.id}}) + {"method": "set_network_host", + "args": {"context": None, + "project_id": context.project.id}}) defer.returnValue(db.queue_get_for(context, FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks def run_instances(self, context, **kwargs): + instance_type = kwargs.get('instance_type', 'm1.small') + if instance_type not in INSTANCE_TYPES: + raise exception.ApiError("Unknown instance type: %s", + instance_type) + # check quota + max_instances = int(kwargs.get('max_count', 1)) + min_instances = int(kwargs.get('min_count', max_instances)) + num_instances = quota.allowed_instances(context, + max_instances, + instance_type) + if num_instances < min_instances: + logging.warn("Quota exceeeded for %s, tried to run %s instances", + context.project.id, min_instances) + raise QuotaError("Instance quota exceeded. You can only " + "run %s more instances of this type." % + num_instances) # make sure user can access the image # vpn image is private so it doesn't show up on lists vpn = kwargs['image_id'] == FLAGS.vpn_image_id @@ -516,7 +552,7 @@ class CloudController(object): images.get(context, kernel_id) images.get(context, ramdisk_id) - logging.debug("Going to run instances...") + logging.debug("Going to run %s instances...", num_instances) launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) key_data = None if kwargs.has_key('key_name'): @@ -540,10 +576,15 @@ class CloudController(object): base_options['user_id'] = context.user.id base_options['project_id'] = context.project.id base_options['user_data'] = kwargs.get('user_data', '') - base_options['instance_type'] = kwargs.get('instance_type', 'm1.small') base_options['security_group'] = security_group + base_options['instance_type'] = instance_type + + type_data = INSTANCE_TYPES['instance_type'] + base_options['memory_mb'] = type_data['memory_mb'] + base_options['vcpus'] = type_data['vcpus'] + base_options['local_gb'] = type_data['local_gb'] - for num in range(int(kwargs['max_count'])): + for num in range(): inst_id = db.instance_create(context, base_options) inst = {} diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 8a7f7b649..b45367eb2 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -50,6 +50,7 @@ class ComputeTestCase(test.TrialTestCase): def tearDown(self): # pylint: disable-msg=C0103 self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(ComputeTestCase, self).tearDown() def _create_instance(self): """Create a test instance""" diff --git a/run_tests.py b/run_tests.py index d5dc5f934..73bf57f97 100644 --- a/run_tests.py +++ b/run_tests.py @@ -58,6 +58,7 @@ from nova.tests.flags_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * +from nova.tests.quota_unittest import * from nova.tests.rpc_unittest import * from nova.tests.service_unittest import * from nova.tests.validator_unittest import * -- cgit From f40c194977b53b7b99a4234f2c1a3b3bfb39c00e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 21:29:00 -0700 Subject: kwargs don't work if you prepend an underscore --- nova/scheduler/simple.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index ea4eef98e..e53e9fa7e 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -37,7 +37,7 @@ flags.DEFINE_integer("max_networks", 1000, class SimpleScheduler(chance.ChanceScheduler): """Implements Naive Scheduler that tries to find least loaded host.""" - def schedule_run_instance(self, context, _instance_id, *_args, **_kwargs): + def schedule_run_instance(self, context, *_args, **_kwargs): """Picks a host that is up and has the fewest running instances.""" results = db.service_get_all_compute_sorted(context) @@ -49,7 +49,7 @@ class SimpleScheduler(chance.ChanceScheduler): return service['host'] raise driver.NoValidHost("No hosts found") - def schedule_create_volume(self, context, _volume_id, *_args, **_kwargs): + def schedule_create_volume(self, context, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" results = db.service_get_all_volume_sorted(context) @@ -61,8 +61,7 @@ class SimpleScheduler(chance.ChanceScheduler): return service['host'] raise driver.NoValidHost("No hosts found") - def schedule_set_network_host(self, context, _network_id, - *_args, **_kwargs): + def schedule_set_network_host(self, context, *_args, **_kwargs): """Picks a host that is up and has the fewest networks.""" results = db.service_get_all_network_sorted(context) -- cgit From 56779ebfec9cd382f170e307a1dc6403e339807f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 21:42:18 -0700 Subject: add missing files for quota --- nova/quota.py | 91 +++++++++++++++++++++++++++++++ nova/tests/quota_unittest.py | 127 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 nova/quota.py create mode 100644 nova/tests/quota_unittest.py diff --git a/nova/quota.py b/nova/quota.py new file mode 100644 index 000000000..f0e51feeb --- /dev/null +++ b/nova/quota.py @@ -0,0 +1,91 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Quotas for instances, volumes, and floating ips +""" + +from nova import db +from nova import exception +from nova import flags +from nova.compute import instance_types + +FLAGS = flags.FLAGS + +flags.DEFINE_integer('quota_instances', 10, + 'number of instances allowed per project') +flags.DEFINE_integer('quota_cores', 20, + 'number of instance cores allowed per project') +flags.DEFINE_integer('quota_volumes', 10, + 'number of volumes allowed per project') +flags.DEFINE_integer('quota_gigabytes', 1000, + 'number of volume gigabytes allowed per project') +flags.DEFINE_integer('quota_floating_ips', 10, + 'number of floating ips allowed per project') + +def _get_quota(context, project_id): + rval = {'instances': FLAGS.quota_instances, + 'cores': FLAGS.quota_cores, + 'volumes': FLAGS.quota_volumes, + 'gigabytes': FLAGS.quota_gigabytes, + 'floating_ips': FLAGS.quota_floating_ips} + try: + quota = db.quota_get(context, project_id) + for key in rval.keys(): + if quota[key] is not None: + rval[key] = quota[key] + except exception.NotFound: + pass + return rval + +def allowed_instances(context, num_instances, instance_type): + """Check quota and return min(num_instances, allowed_instances)""" + project_id = context.project.id + used_instances, used_cores = db.instance_data_get_for_project(context, + project_id) + quota = _get_quota(context, project_id) + allowed_instances = quota['instances'] - used_instances + allowed_cores = quota['cores'] - used_cores + type_cores = instance_types.INSTANCE_TYPES[instance_type]['vcpus'] + num_cores = num_instances * type_cores + allowed_instances = min(allowed_instances, + int(allowed_cores // type_cores)) + return min(num_instances, allowed_instances) + + +def allowed_volumes(context, num_volumes, size): + """Check quota and return min(num_volumes, allowed_volumes)""" + project_id = context.project.id + used_volumes, used_gigabytes = db.volume_data_get_for_project(context, + project_id) + quota = _get_quota(context, project_id) + allowed_volumes = quota['volumes'] - used_volumes + allowed_gigabytes = quota['gigabytes'] - used_gigabytes + num_gigabytes = num_volumes * size + allowed_volumes = min(allowed_volumes, + int(allowed_gigabytes // size)) + return min(num_volumes, allowed_volumes) + + +def allowed_floating_ips(context, num_floating_ips): + """Check quota and return min(num_floating_ips, allowed_floating_ips)""" + project_id = context.project.id + used_floating_ips = db.floating_ip_count_by_project(context, project_id) + quota = _get_quota(context, project_id) + allowed_floating_ips = quota['floating_ips'] - used_floating_ips + return min(num_floating_ips, allowed_floating_ips) + diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py new file mode 100644 index 000000000..bf3506c78 --- /dev/null +++ b/nova/tests/quota_unittest.py @@ -0,0 +1,127 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from nova import db +from nova import flags +from nova import quota +from nova import test +from nova import utils +from nova.auth import manager +from nova.endpoint import cloud +from nova.endpoint import api + + +FLAGS = flags.FLAGS + + +class QuotaTestCase(test.TrialTestCase): + def setUp(self): # pylint: disable-msg=C0103 + logging.getLogger().setLevel(logging.DEBUG) + super(QuotaTestCase, self).setUp() + self.flags(connection_type='fake', + quota_instances=2, + quota_cores=4, + quota_volumes=2, + quota_gigabytes=20, + quota_floating_ips=2) + + self.cloud = cloud.CloudController() + self.manager = manager.AuthManager() + self.user = self.manager.create_user('admin', 'admin', 'admin', True) + self.project = self.manager.create_project('admin', 'admin', 'admin') + self.context = api.APIRequestContext(handler=None, + project=self.project, + user=self.user) + + def tearDown(self): # pylint: disable-msg=C0103 + manager.AuthManager().delete_project(self.project) + manager.AuthManager().delete_user(self.user) + super(QuotaTestCase, self).tearDown() + + def _create_instance(self, cores=2): + """Create a test instance""" + inst = {} + inst['image_id'] = 'ami-test' + inst['reservation_id'] = 'r-fakeres' + inst['user_id'] = self.user.id + inst['project_id'] = self.project.id + inst['instance_type'] = 'm1.large' + inst['vcpus'] = cores + inst['mac_address'] = utils.generate_mac() + return db.instance_create(self.context, inst) + + def _create_volume(self, size=10): + """Create a test volume""" + vol = {} + vol['user_id'] = self.user.id + vol['project_id'] = self.project.id + vol['size'] = size + return db.volume_create(self.context, vol)['id'] + + def test_quota_overrides(self): + """Make sure overriding a projects quotas works""" + num_instances = quota.allowed_instances(self.context, 100, 'm1.small') + self.assertEqual(num_instances, 2) + db.quota_create(self.context, {'project_id': self.project.id, + 'instances': 10}) + num_instances = quota.allowed_instances(self.context, 100, 'm1.small') + self.assertEqual(num_instances, 4) + db.quota_update(self.context, self.project.id, {'cores': 100}) + num_instances = quota.allowed_instances(self.context, 100, 'm1.small') + self.assertEqual(num_instances, 10) + db.quota_destroy(self.context, self.project.id) + + def test_too_many_instances(self): + instance_ids = [] + for i in range(FLAGS.quota_instances): + instance_id = self._create_instance() + instance_ids.append(instance_id) + self.assertFailure(self.cloud.run_instances(self.context, + min_count=1, + max_count=1, + instance_type='m1.small'), + cloud.QuotaError) + for instance_id in instance_ids: + db.instance_destroy(self.context, instance_id) + + def test_too_many_cores(self): + instance_ids = [] + instance_id = self._create_instance(cores=4) + instance_ids.append(instance_id) + self.assertFailure(self.cloud.run_instances(self.context, + min_count=1, + max_count=1, + instance_type='m1.small'), + cloud.QuotaError) + for instance_id in instance_ids: + db.instance_destroy(self.context, instance_id) + + def test_too_many_volumes(self): + volume_ids = [] + for i in range(FLAGS.quota_volumes): + volume_id = self._create_volume() + volume_ids.append(volume_id) + self.assertRaises(cloud.QuotaError, + self.cloud.create_volume, + self.context, + size=10) + for volume_id in volume_ids: + db.volume_destroy(self.context, volume_id) + -- cgit From c5bfa37c92bd066fa2bc3565b251edced3255438 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 21:59:09 -0700 Subject: fix unittest --- nova/tests/scheduler_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 51b9aeaad..09e45ea68 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -109,7 +109,7 @@ class SimpleDriverTestCase(test.TrialTestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 - return db.instance_create(self.context, inst) + return db.instance_create(self.context, inst)['id'] def test_hosts_are_up(self): # NOTE(vish): constructing service without create method -- cgit From 5cb90074df70daa60241930da9940e093a3812ba Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 22:13:38 -0700 Subject: quota tests --- nova/endpoint/cloud.py | 1 + nova/tests/quota_unittest.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 5209ec906..b5ac5be4d 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -450,6 +450,7 @@ class CloudController(object): @defer.inlineCallbacks def allocate_address(self, context, **kwargs): # check quota + print quota.allowed_floating_ips(context, 1) if quota.allowed_floating_ips(context, 1) < 1: logging.warn("Quota exceeeded for %s, tried to allocate address", context.project.id) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index e3f23b84e..d7c07bfab 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -19,6 +19,7 @@ import logging from nova import db +from nova import exception from nova import flags from nova import quota from nova import test @@ -46,6 +47,7 @@ class QuotaTestCase(test.TrialTestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('admin', 'admin', 'admin') + self.network = utils.import_object(FLAGS.network_manager) self.context = api.APIRequestContext(handler=None, project=self.project, user=self.user) @@ -125,3 +127,26 @@ class QuotaTestCase(test.TrialTestCase): for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) + def test_too_many_gigabytes(self): + volume_ids = [] + volume_id = self._create_volume(size=20) + volume_ids.append(volume_id) + self.assertRaises(cloud.QuotaError, + self.cloud.create_volume, + self.context, + size=10) + for volume_id in volume_ids: + db.volume_destroy(self.context, volume_id) + + def test_too_many_addresses(self): + address = '192.168.0.100' + try: + db.floating_ip_get_by_address(None, address) + except exception.NotFound: + db.floating_ip_create(None, {'address': address, + 'host': FLAGS.host}) + #float_addr = self.network.allocate_floating_ip(self.context, + # self.project.id) + self.assertFailure(self.cloud.allocate_address(self.context), + cloud.QuotaError) + -- cgit From ece1c84203890e87834bb53acaf98420fdeee6dc Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 22:53:31 -0700 Subject: address test almost works --- nova/tests/quota_unittest.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index d7c07bfab..9d697ccd3 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -41,7 +41,7 @@ class QuotaTestCase(test.TrialTestCase): quota_cores=4, quota_volumes=2, quota_gigabytes=20, - quota_floating_ips=2) + quota_floating_ips=1) self.cloud = cloud.CloudController() self.manager = manager.AuthManager() @@ -145,8 +145,18 @@ class QuotaTestCase(test.TrialTestCase): except exception.NotFound: db.floating_ip_create(None, {'address': address, 'host': FLAGS.host}) - #float_addr = self.network.allocate_floating_ip(self.context, - # self.project.id) + float_addr = self.network.allocate_floating_ip(self.context, + self.project.id) + # NOTE(vish): This assert doesn't work. When cloud attempts to + # make an rpc.call, the test just finishes with OK. It + # appears to be something in the magic inline callbacks + # that is breaking. self.assertFailure(self.cloud.allocate_address(self.context), cloud.QuotaError) + try: + yield self.cloud.allocate_address(self.context) + self.fail('Should have raised QuotaError') + except cloud.QuotaError: + pass + -- cgit From a7a46ea93186ca68ca90efdcd86b4d2a7d3bd8e8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 23:04:30 -0700 Subject: quotas working and tests passing --- nova/endpoint/cloud.py | 1 - nova/tests/quota_unittest.py | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index b5ac5be4d..5209ec906 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -450,7 +450,6 @@ class CloudController(object): @defer.inlineCallbacks def allocate_address(self, context, **kwargs): # check quota - print quota.allowed_floating_ips(context, 1) if quota.allowed_floating_ips(context, 1) < 1: logging.warn("Quota exceeeded for %s, tried to allocate address", context.project.id) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index 9d697ccd3..cab9f663d 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -147,16 +147,9 @@ class QuotaTestCase(test.TrialTestCase): 'host': FLAGS.host}) float_addr = self.network.allocate_floating_ip(self.context, self.project.id) - # NOTE(vish): This assert doesn't work. When cloud attempts to + # NOTE(vish): This assert never fails. When cloud attempts to # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. self.assertFailure(self.cloud.allocate_address(self.context), cloud.QuotaError) - try: - yield self.cloud.allocate_address(self.context) - self.fail('Should have raised QuotaError') - except cloud.QuotaError: - pass - - -- cgit From d534655b636563fa71ca78758340b2dd49bc2527 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 23:32:43 -0700 Subject: don't pass topic into schedule_run_instance --- nova/scheduler/manager.py | 2 ++ nova/tests/scheduler_unittest.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 0ad7ca86b..af76334a8 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -54,6 +54,8 @@ class SchedulerManager(manager.Manager): Falls back to schedule(context, topic) if method doesn't exist. """ driver_method = 'schedule_%s' % method + print topic + print args try: host = getattr(self.driver, driver_method)(context, *args, **kwargs) except AttributeError: diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 09e45ea68..27e100fa0 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -121,7 +121,6 @@ class SimpleDriverTestCase(test.TrialTestCase): instance_id = self._create_instance() self.service1.run_instance(self.context, instance_id) host = self.scheduler.driver.schedule_run_instance(self.context, - 'compute', instance_id) self.assertEqual(host, 'host2') self.service1.terminate_instance(self.context, instance_id) -- cgit From ffb2d740a1d8fba997c043cc3066282afedebae8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 23:37:08 -0700 Subject: removed extra quotes around instance_type --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 5209ec906..ad5db6668 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -578,7 +578,7 @@ class CloudController(object): base_options['security_group'] = security_group base_options['instance_type'] = instance_type - type_data = INSTANCE_TYPES['instance_type'] + type_data = INSTANCE_TYPES[instance_type] base_options['memory_mb'] = type_data['memory_mb'] base_options['vcpus'] = type_data['vcpus'] base_options['local_gb'] = type_data['local_gb'] -- cgit From 1867c2aae81e4a73374bde0169b4e16cd8e18846 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Sep 2010 23:43:51 -0700 Subject: remove print statements --- nova/scheduler/manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index af76334a8..0ad7ca86b 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -54,8 +54,6 @@ class SchedulerManager(manager.Manager): Falls back to schedule(context, topic) if method doesn't exist. """ driver_method = 'schedule_%s' % method - print topic - print args try: host = getattr(self.driver, driver_method)(context, *args, **kwargs) except AttributeError: -- cgit From 1c01b37a5f2372f4e61fdff8a16a9efe6f6b7e7b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 01:13:11 -0700 Subject: set host when item is scheduled --- nova/scheduler/simple.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index e53e9fa7e..48be4c1a6 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -37,7 +37,7 @@ flags.DEFINE_integer("max_networks", 1000, class SimpleScheduler(chance.ChanceScheduler): """Implements Naive Scheduler that tries to find least loaded host.""" - def schedule_run_instance(self, context, *_args, **_kwargs): + def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): """Picks a host that is up and has the fewest running instances.""" results = db.service_get_all_compute_sorted(context) @@ -46,10 +46,13 @@ class SimpleScheduler(chance.ChanceScheduler): if instance_count >= FLAGS.max_instances: raise driver.NoValidHost("All hosts have too many instances") if self.service_is_up(service): + db.instance_update(context, + instance_id, + {'host': service['host']}) return service['host'] raise driver.NoValidHost("No hosts found") - def schedule_create_volume(self, context, *_args, **_kwargs): + def schedule_create_volume(self, context, volume_id, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" results = db.service_get_all_volume_sorted(context) @@ -58,6 +61,9 @@ class SimpleScheduler(chance.ChanceScheduler): if instance_count >= FLAGS.max_volumes: raise driver.NoValidHost("All hosts have too many volumes") if self.service_is_up(service): + db.instance_update(context, + volume_id, + {'host': service['host']}) return service['host'] raise driver.NoValidHost("No hosts found") -- cgit From 9330ebc110aeb7591567c66939b39f4345b5778d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 04:52:48 -0700 Subject: added modify project command to allow project manager and description to be updated --- nova/auth/fakeldap.py | 5 ++++- nova/auth/ldapdriver.py | 18 ++++++++++++++++++ nova/auth/manager.py | 20 ++++++++++++++++++++ nova/tests/auth_unittest.py | 6 ++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index bfc3433c5..2791dfde6 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -33,6 +33,7 @@ SCOPE_ONELEVEL = 1 # not implemented SCOPE_SUBTREE = 2 MOD_ADD = 0 MOD_DELETE = 1 +MOD_REPLACE = 2 class NO_SUCH_OBJECT(Exception): # pylint: disable-msg=C0103 @@ -175,7 +176,7 @@ class FakeLDAP(object): Args: dn -- a dn attrs -- a list of tuples in the following form: - ([MOD_ADD | MOD_DELETE], attribute, value) + ([MOD_ADD | MOD_DELETE | MOD_REPACE], attribute, value) """ redis = datastore.Redis.instance() @@ -185,6 +186,8 @@ class FakeLDAP(object): values = _from_json(redis.hget(key, k)) if cmd == MOD_ADD: values.append(v) + elif cmd == MOD_REPLACE: + values = [v] else: values.remove(v) values = redis.hset(key, k, _to_json(values)) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 74ba011b5..cc8e2caa3 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -202,6 +202,24 @@ class LdapDriver(object): self.conn.add_s('cn=%s,%s' % (name, FLAGS.ldap_project_subtree), attr) return self.__to_project(dict(attr)) + def modify_project(self, project_id, manager_uid=None, description=None): + """Modify an existing project""" + if not manager_uid and not description: + return + attr = [] + if manager_uid: + if not self.__user_exists(manager_uid): + raise exception.NotFound("Project can't be modified because " + "manager %s doesn't exist" % + manager_uid) + manager_dn = self.__uid_to_dn(manager_uid) + attr.append((self.ldap.MOD_REPLACE, 'projectManager', manager_dn)) + if description: + attr.append((self.ldap.MOD_REPLACE, 'description', description)) + self.conn.modify_s('cn=%s,%s' % (project_id, + FLAGS.ldap_project_subtree), + attr) + def add_to_project(self, uid, project_id): """Add user to project""" dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 284b29502..d094bb7e1 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -525,6 +525,26 @@ class AuthManager(object): if project_dict: return Project(**project_dict) + def modify_project(self, project, manager_user=None, description=None): + """Modify a project + + @type name: Project or project_id + @param project: The project to modify. + + @type manager_user: User or uid + @param manager_user: This user will be the new project manager. + + @type description: str + @param project: This will be the new description of the project. + + """ + if manager_user: + manager_user = User.safe_id(manager_user) + with self.driver() as drv: + drv.modify_project(Project.safe_id(project), + manager_user, + description) + def add_to_project(self, user, project): """Add user to project""" with self.driver() as drv: diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 0b404bfdc..2fc780640 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -206,6 +206,12 @@ class AuthTestCase(test.BaseTestCase): self.assert_(len(self.manager.get_projects()) > 1) self.assertEqual(len(self.manager.get_projects('test2')), 1) + def test_220_can_modify_project(self): + self.manager.modify_project('testproj', 'test2', 'new description') + project = self.manager.get_project('testproj') + self.assertEqual(project.project_manager_id, 'test2') + self.assertEqual(project.description, 'new description') + def test_299_can_delete_project(self): self.manager.delete_project('testproj') self.assertFalse(filter(lambda p: p.name == 'testproj', self.manager.get_projects())) -- cgit From ac27df3f4bea1a1a05a84de99c098dc91741a7ee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 16:40:49 -0700 Subject: make api error messages more readable --- nova/endpoint/api.py | 5 ++++- nova/endpoint/cloud.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/api.py b/nova/endpoint/api.py index 40be00bb7..12eedfe67 100755 --- a/nova/endpoint/api.py +++ b/nova/endpoint/api.py @@ -304,7 +304,10 @@ class APIRequestHandler(tornado.web.RequestHandler): try: failure.raiseException() except exception.ApiError as ex: - self._error(type(ex).__name__ + "." + ex.code, ex.message) + if ex.code: + self._error(ex.code, ex.message) + else: + self._error(type(ex).__name__, ex.message) # TODO(vish): do something more useful with unknown exceptions except Exception as ex: self._error(type(ex).__name__, str(ex)) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ad5db6668..adb63351f 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -529,7 +529,7 @@ class CloudController(object): context.project.id, min_instances) raise QuotaError("Instance quota exceeded. You can only " "run %s more instances of this type." % - num_instances) + num_instances, "InstanceLimitExceeded") # make sure user can access the image # vpn image is private so it doesn't show up on lists vpn = kwargs['image_id'] == FLAGS.vpn_image_id -- cgit From bc265bbc9b3b42e46e044c18252218a375192123 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 17:12:49 -0700 Subject: multi-region flag for describe regions --- nova/endpoint/cloud.py | 15 ++++++++++++--- nova/flags.py | 15 +++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 8e2beb1e3..180af0540 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -174,9 +174,18 @@ class CloudController(object): @rbac.allow('all') def describe_regions(self, context, region_name=None, **kwargs): - # TODO(vish): region_name is an array. Support filtering - return {'regionInfo': [{'regionName': 'nova', - 'regionUrl': FLAGS.ec2_url}]} + if FLAGS.region_list: + regions = [] + for region in FLAGS.region_list: + name, _sep, url = region.partition(',') + regions.append({'regionName': name, + 'regionUrl': url}) + else: + regions = [{'regionName': 'nova', + 'regionUrl': FLAGS.ec2_url}] + if region_name: + regions = [r for r in regions if r['regionName'] in region_name] + return {'regionInfo': regions } @rbac.allow('all') def describe_snapshots(self, diff --git a/nova/flags.py b/nova/flags.py index 2bca36f7e..19dcb96ba 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -34,7 +34,7 @@ class FlagValues(gflags.FlagValues): Unknown flags will be ignored when parsing the command line, but the command line will be kept so that it can be replayed if new flags are defined after the initial parsing. - + """ def __init__(self): @@ -50,7 +50,7 @@ class FlagValues(gflags.FlagValues): # leftover args at the end sneaky_unparsed_args = {"value": None} original_argv = list(argv) - + if self.IsGnuGetOpt(): orig_getopt = getattr(getopt, 'gnu_getopt') orig_name = 'gnu_getopt' @@ -81,7 +81,7 @@ class FlagValues(gflags.FlagValues): args = argv[:1] finally: setattr(getopt, orig_name, orig_getopt) - + # Store the arguments for later, we'll need them for new flags # added at runtime self.__dict__['__stored_argv'] = original_argv @@ -92,7 +92,7 @@ class FlagValues(gflags.FlagValues): def SetDirty(self, name): """Mark a flag as dirty so that accessing it will case a reparse.""" self.__dict__['__dirty'].append(name) - + def IsDirty(self, name): return name in self.__dict__['__dirty'] @@ -113,12 +113,12 @@ class FlagValues(gflags.FlagValues): for k in self.__dict__['__dirty']: setattr(self, k, getattr(new_flags, k)) self.ClearDirty() - + def __setitem__(self, name, flag): gflags.FlagValues.__setitem__(self, name, flag) if self.WasAlreadyParsed(): self.SetDirty(name) - + def __getitem__(self, name): if self.IsDirty(name): self.ParseNewFlags() @@ -166,6 +166,9 @@ def DECLARE(name, module_string, flag_values=FLAGS): # Define any app-specific flags in their own files, docs at: # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 +DEFINE_list('region_list', + [], + 'list of region,url pairs') DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') -- cgit From ee206cd08bd2d82bb5d64b84b6804ba51ab56b37 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 18:51:22 -0700 Subject: moved keypairs to db using the same interface --- nova/auth/manager.py | 36 +++++++++++++++--------------------- nova/db/api.py | 23 +++++++++++++++++++++++ nova/db/sqlalchemy/api.py | 32 ++++++++++++++++++++++++++++++++ nova/db/sqlalchemy/models.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 21 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index d5fbec7c5..4cb23bea6 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -668,42 +668,36 @@ class AuthManager(object): with self.driver() as drv: if not drv.get_user(uid): raise exception.NotFound("User %s doesn't exist" % user) - if drv.get_key_pair(uid, key_name): - raise exception.Duplicate("The keypair %s already exists" - % key_name) + try: + db.keypair_get(None, uid, key_name) + raise exception.Duplicate("The keypair %s already exists" + % key_name) + except exception.NotFound: + pass private_key, public_key, fingerprint = crypto.generate_key_pair() self.create_key_pair(uid, key_name, public_key, fingerprint) return private_key, fingerprint def create_key_pair(self, user, key_name, public_key, fingerprint): """Creates a key pair for user""" - with self.driver() as drv: - kp_dict = drv.create_key_pair(User.safe_id(user), - key_name, - public_key, - fingerprint) - if kp_dict: - return KeyPair(**kp_dict) + key = {} + key['user_id'] = User.safe_id(user) + key['name'] = key_name + key['public_key'] = public_key + key['fingerprint'] = fingerprint + return db.keypair_create(None, key) def get_key_pair(self, user, key_name): """Retrieves a key pair for user""" - with self.driver() as drv: - kp_dict = drv.get_key_pair(User.safe_id(user), key_name) - if kp_dict: - return KeyPair(**kp_dict) + return db.keypair_get(None, User.safe_id(user), key_name) def get_key_pairs(self, user): """Retrieves all key pairs for user""" - with self.driver() as drv: - kp_list = drv.get_key_pairs(User.safe_id(user)) - if not kp_list: - return [] - return [KeyPair(**kp_dict) for kp_dict in kp_list] + return db.keypair_get_all_by_user(None, User.safe_id(user)) def delete_key_pair(self, user, key_name): """Deletes a key pair for user""" - with self.driver() as drv: - drv.delete_key_pair(User.safe_id(user), key_name) + return db.keypair_destroy(None, User.safe_id(user), key_name) def get_credentials(self, user, project=None): """Get credential zip for user in project""" diff --git a/nova/db/api.py b/nova/db/api.py index d81673fad..1db978c52 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -254,6 +254,29 @@ def instance_update(context, instance_id, values): return IMPL.instance_update(context, instance_id, values) +################### + + +def keypair_create(context, values): + """Create a keypair from the values dictionary.""" + return IMPL.keypair_create(context, values) + + +def keypair_destroy(context, user_id, name): + """Destroy the keypair or raise if it does not exist.""" + return IMPL.keypair_destroy(context, user_id, name) + + +def keypair_get(context, user_id, name): + """Get a keypair or raise if it does not exist.""" + return IMPL.keypair_get(context, user_id, name) + + +def keypair_get_all_by_user(context, user_id): + """Get all keypairs by user.""" + return IMPL.keypair_get_all_by_user(context, user_id) + + #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 02ebdd222..b3a307043 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -355,6 +355,38 @@ def instance_update(_context, instance_id, values): ################### +def keypair_create(_context, values): + keypair_ref = models.Keypair() + for (key, value) in values.iteritems(): + keypair_ref[key] = value + keypair_ref.save() + return keypair_ref + + +def keypair_destroy(_context, user_id, name): + session = get_session() + with session.begin(): + keypair_ref = models.Keypair.find_by_args(user_id, + name, + session=session) + keypair_ref.delete(session=session) + + +def keypair_get(_context, user_id, name): + return models.Keypair.find_by_args(user_id, name) + + +def keypair_get_all_by_user(_context, user_id): + session = get_session() + return session.query(models.Keypair + ).filter_by(user_id=user_id + ).filter_by(deleted=False + ).all() + + +################### + + def network_count(_context): return models.Network.count() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 6818f838c..81c0a77a8 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -284,6 +284,42 @@ class ExportDevice(BASE, NovaBase): uselist=False)) +class Keypair(BASE, NovaBase): + """Represents a keypair""" + __tablename__ = 'keypairs' + id = Column(Integer, primary_key=True) + name = Column(String(255)) + + user_id = Column(String(255)) + + fingerprint = Column(String(255)) + public_key = Column(Text) + + @property + def str_id(self): + return '%s.%s' % (self.user_id, self.name) + + @classmethod + def find_by_str(cls, str_id, session=None, deleted=False): + user_id, _sep, name = str_id.partition('.') + return cls.find_by_str(user_id, name, session, deleted) + + @classmethod + def find_by_args(cls, user_id, name, session=None, deleted=False): + if not session: + session = get_session() + try: + return session.query(cls + ).filter_by(user_id=user_id + ).filter_by(name=name + ).filter_by(deleted=deleted + ).one() + except exc.NoResultFound: + new_exc = exception.NotFound("No model for user %s, name %s" % + (user_id, name)) + raise new_exc.__class__, new_exc, sys.exc_info()[2] + + class Network(BASE, NovaBase): """Represents a network""" __tablename__ = 'networks' -- cgit From d3273e594daf5f94f09c7904bac53fbb895ffeb6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 18:55:11 -0700 Subject: remove keypair from driver --- nova/auth/ldapdriver.py | 60 ------------------------------------------------- nova/auth/manager.py | 23 ------------------- 2 files changed, 83 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 74ba011b5..4e9afc858 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -99,13 +99,6 @@ class LdapDriver(object): dn = FLAGS.ldap_user_subtree return self.__to_user(self.__find_object(dn, query)) - def get_key_pair(self, uid, key_name): - """Retrieve key pair by uid and key name""" - dn = 'cn=%s,%s' % (key_name, - self.__uid_to_dn(uid)) - attr = self.__find_object(dn, '(objectclass=novaKeyPair)') - return self.__to_key_pair(uid, attr) - def get_project(self, pid): """Retrieve project by id""" dn = 'cn=%s,%s' % (pid, @@ -119,12 +112,6 @@ class LdapDriver(object): '(objectclass=novaUser)') return [self.__to_user(attr) for attr in attrs] - def get_key_pairs(self, uid): - """Retrieve list of key pairs""" - attrs = self.__find_objects(self.__uid_to_dn(uid), - '(objectclass=novaKeyPair)') - return [self.__to_key_pair(uid, attr) for attr in attrs] - def get_projects(self, uid=None): """Retrieve list of projects""" pattern = '(objectclass=novaProject)' @@ -154,21 +141,6 @@ class LdapDriver(object): self.conn.add_s(self.__uid_to_dn(name), attr) return self.__to_user(dict(attr)) - def create_key_pair(self, uid, key_name, public_key, fingerprint): - """Create a key pair""" - # TODO(vish): possibly refactor this to store keys in their own ou - # and put dn reference in the user object - attr = [ - ('objectclass', ['novaKeyPair']), - ('cn', [key_name]), - ('sshPublicKey', [public_key]), - ('keyFingerprint', [fingerprint]), - ] - self.conn.add_s('cn=%s,%s' % (key_name, - self.__uid_to_dn(uid)), - attr) - return self.__to_key_pair(uid, dict(attr)) - def create_project(self, name, manager_uid, description=None, member_uids=None): """Create a project""" @@ -265,19 +237,10 @@ class LdapDriver(object): """Delete a user""" if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) - self.__delete_key_pairs(uid) self.__remove_from_all(uid) self.conn.delete_s('uid=%s,%s' % (uid, FLAGS.ldap_user_subtree)) - def delete_key_pair(self, uid, key_name): - """Delete a key pair""" - if not self.__key_pair_exists(uid, key_name): - raise exception.NotFound("Key Pair %s doesn't exist for user %s" % - (key_name, uid)) - self.conn.delete_s('cn=%s,uid=%s,%s' % (key_name, uid, - FLAGS.ldap_user_subtree)) - def delete_project(self, project_id): """Delete a project""" project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) @@ -288,10 +251,6 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None - def __key_pair_exists(self, uid, key_name): - """Check if key pair exists""" - return self.get_key_pair(uid, key_name) != None - def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None @@ -341,13 +300,6 @@ class LdapDriver(object): """Check if group exists""" return self.__find_object(dn, '(objectclass=groupOfNames)') != None - def __delete_key_pairs(self, uid): - """Delete all key pairs for user""" - keys = self.get_key_pairs(uid) - if keys != None: - for key in keys: - self.delete_key_pair(uid, key['name']) - @staticmethod def __role_to_dn(role, project_id=None): """Convert role to corresponding dn""" @@ -472,18 +424,6 @@ class LdapDriver(object): 'secret': attr['secretKey'][0], 'admin': (attr['isAdmin'][0] == 'TRUE')} - @staticmethod - def __to_key_pair(owner, attr): - """Convert ldap attributes to KeyPair object""" - if attr == None: - return None - return { - 'id': attr['cn'][0], - 'name': attr['cn'][0], - 'owner_id': owner, - 'public_key': attr['sshPublicKey'][0], - 'fingerprint': attr['keyFingerprint'][0]} - def __to_project(self, attr): """Convert ldap attributes to Project object""" if attr == None: diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 4cb23bea6..ef6a5a486 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -154,29 +154,6 @@ class User(AuthBase): self.admin) -class KeyPair(AuthBase): - """Represents an ssh key returned from the datastore - - Even though this object is named KeyPair, only the public key and - fingerprint is stored. The user's private key is not saved. - """ - - def __init__(self, id, name, owner_id, public_key, fingerprint): - AuthBase.__init__(self) - self.id = id - self.name = name - self.owner_id = owner_id - self.public_key = public_key - self.fingerprint = fingerprint - - def __repr__(self): - return "KeyPair('%s', '%s', '%s', '%s', '%s')" % (self.id, - self.name, - self.owner_id, - self.public_key, - self.fingerprint) - - class Project(AuthBase): """Represents a Project returned from the datastore""" -- cgit From adb9cf9e71908844fd720e6f9bab9588610878e1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 19:03:35 -0700 Subject: delete keypairs when a user is deleted --- nova/auth/manager.py | 8 ++++++-- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 8 ++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index ef6a5a486..e2bb748b0 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -620,9 +620,13 @@ class AuthManager(object): return User(**user_dict) def delete_user(self, user): - """Deletes a user""" + """Deletes a user + + Additionally deletes all users keypairs""" + uid = User.safe_id(user) + db.keypair_destroy_all_by_user(None, uid) with self.driver() as drv: - drv.delete_user(User.safe_id(user)) + drv.delete_user(uid) def generate_key_pair(self, user, key_name): """Generates a key pair for a user diff --git a/nova/db/api.py b/nova/db/api.py index 1db978c52..e96d803db 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -267,6 +267,11 @@ def keypair_destroy(context, user_id, name): return IMPL.keypair_destroy(context, user_id, name) +def keypair_destroy_all_by_user(context, user_id): + """Destroy all keypairs by user.""" + return IMPL.keypair_destroy_all_by_user(context, user_id) + + def keypair_get(context, user_id, name): """Get a keypair or raise if it does not exist.""" return IMPL.keypair_get(context, user_id, name) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index b3a307043..4fd1bf216 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -372,6 +372,14 @@ def keypair_destroy(_context, user_id, name): keypair_ref.delete(session=session) +def keypair_destroy_all_by_user(_context, user_id): + session = get_session() + with session.begin(): + # TODO(vish): do we have to use sql here? + session.execute('update keypairs set deleted=1 where user_id=:id', + {'id': user_id}) + + def keypair_get(_context, user_id, name): return models.Keypair.find_by_args(user_id, name) -- cgit From 8e834931087c54585a7aa2716c7a0708fd658f30 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 22:13:36 -0700 Subject: move keypair generation out of auth and fix tests --- nova/auth/manager.py | 70 -------------------------------------------- nova/endpoint/cloud.py | 48 +++++++++++++++++++++--------- nova/tests/api_unittest.py | 7 +++-- nova/tests/auth_unittest.py | 31 -------------------- nova/tests/cloud_unittest.py | 53 ++++++++++++++++++++++++++++----- 5 files changed, 83 insertions(+), 126 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index e2bb748b0..fb87847d5 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -128,24 +128,6 @@ class User(AuthBase): def is_project_manager(self, project): return AuthManager().is_project_manager(self, project) - def generate_key_pair(self, name): - return AuthManager().generate_key_pair(self.id, name) - - def create_key_pair(self, name, public_key, fingerprint): - return AuthManager().create_key_pair(self.id, - name, - public_key, - fingerprint) - - def get_key_pair(self, name): - return AuthManager().get_key_pair(self.id, name) - - def delete_key_pair(self, name): - return AuthManager().delete_key_pair(self.id, name) - - def get_key_pairs(self): - return AuthManager().get_key_pairs(self.id) - def __repr__(self): return "User('%s', '%s', '%s', '%s', %s)" % (self.id, self.name, @@ -628,58 +610,6 @@ class AuthManager(object): with self.driver() as drv: drv.delete_user(uid) - def generate_key_pair(self, user, key_name): - """Generates a key pair for a user - - Generates a public and private key, stores the public key using the - key_name, and returns the private key and fingerprint. - - @type user: User or uid - @param user: User for which to create key pair. - - @type key_name: str - @param key_name: Name to use for the generated KeyPair. - - @rtype: tuple (private_key, fingerprint) - @return: A tuple containing the private_key and fingerprint. - """ - # NOTE(vish): generating key pair is slow so check for legal - # creation before creating keypair - uid = User.safe_id(user) - with self.driver() as drv: - if not drv.get_user(uid): - raise exception.NotFound("User %s doesn't exist" % user) - try: - db.keypair_get(None, uid, key_name) - raise exception.Duplicate("The keypair %s already exists" - % key_name) - except exception.NotFound: - pass - private_key, public_key, fingerprint = crypto.generate_key_pair() - self.create_key_pair(uid, key_name, public_key, fingerprint) - return private_key, fingerprint - - def create_key_pair(self, user, key_name, public_key, fingerprint): - """Creates a key pair for user""" - key = {} - key['user_id'] = User.safe_id(user) - key['name'] = key_name - key['public_key'] = public_key - key['fingerprint'] = fingerprint - return db.keypair_create(None, key) - - def get_key_pair(self, user, key_name): - """Retrieves a key pair for user""" - return db.keypair_get(None, User.safe_id(user), key_name) - - def get_key_pairs(self, user): - """Retrieves all key pairs for user""" - return db.keypair_get_all_by_user(None, User.safe_id(user)) - - def delete_key_pair(self, user, key_name): - """Deletes a key pair for user""" - return db.keypair_destroy(None, User.safe_id(user), key_name) - def get_credentials(self, user, project=None): """Get credential zip for user in project""" if not isinstance(user, User): diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 6ca6855ca..172c65d79 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -29,13 +29,13 @@ import time from twisted.internet import defer +from nova import crypto from nova import db from nova import exception from nova import flags from nova import rpc from nova import utils from nova.auth import rbac -from nova.auth import manager from nova.compute.instance_types import INSTANCE_TYPES from nova.endpoint import images @@ -44,14 +44,30 @@ FLAGS = flags.FLAGS flags.DECLARE('storage_availability_zone', 'nova.volume.manager') -def _gen_key(user_id, key_name): - """ Tuck this into AuthManager """ +def _gen_key(context, user_id, key_name): + """Generate a key + + This is a module level method because it is slow and we need to defer + it into a process pool.""" try: - mgr = manager.AuthManager() - private_key, fingerprint = mgr.generate_key_pair(user_id, key_name) + # NOTE(vish): generating key pair is slow so check for legal + # creation before creating keypair + try: + db.keypair_get(context, user_id, key_name) + raise exception.Duplicate("The keypair %s already exists" + % key_name) + except exception.NotFound: + pass + private_key, public_key, fingerprint = crypto.generate_key_pair() + key = {} + key['user_id'] = user_id + key['name'] = key_name + key['public_key'] = public_key + key['fingerprint'] = fingerprint + db.keypair_create(context, key) + return {'private_key': private_key, 'fingerprint': fingerprint} except Exception as ex: return {'exception': ex} - return {'private_key': private_key, 'fingerprint': fingerprint} class CloudController(object): @@ -177,18 +193,18 @@ class CloudController(object): @rbac.allow('all') def describe_key_pairs(self, context, key_name=None, **kwargs): - key_pairs = context.user.get_key_pairs() + key_pairs = db.keypair_get_all_by_user(context, context.user.id) if not key_name is None: - key_pairs = [x for x in key_pairs if x.name in key_name] + key_pairs = [x for x in key_pairs if x['name'] in key_name] result = [] for key_pair in key_pairs: # filter out the vpn keys suffix = FLAGS.vpn_key_suffix - if context.user.is_admin() or not key_pair.name.endswith(suffix): + if context.user.is_admin() or not key_pair['name'].endswith(suffix): result.append({ - 'keyName': key_pair.name, - 'keyFingerprint': key_pair.fingerprint, + 'keyName': key_pair['name'], + 'keyFingerprint': key_pair['fingerprint'], }) return {'keypairsSet': result} @@ -204,14 +220,18 @@ class CloudController(object): dcall.callback({'keyName': key_name, 'keyFingerprint': kwargs['fingerprint'], 'keyMaterial': kwargs['private_key']}) - pool.apply_async(_gen_key, [context.user.id, key_name], + # TODO(vish): when context is no longer an object, pass it here + pool.apply_async(_gen_key, [None, context.user.id, key_name], callback=_complete) return dcall @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): - context.user.delete_key_pair(key_name) - # aws returns true even if the key doens't exist + try: + db.keypair_destroy(context, context.user.id, key_name) + except exception.NotFound: + # aws returns true even if the key doesn't exist + pass return True @rbac.allow('all') diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 462d1b295..fdb9e21d8 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -41,8 +41,8 @@ FLAGS = flags.FLAGS # it's pretty damn circuitous so apologies if you have to fix # a bug in it # NOTE(jaypipes) The pylint disables here are for R0913 (too many args) which -# isn't controllable since boto's HTTPRequest needs that many -# args, and for the version-differentiated import of tornado's +# isn't controllable since boto's HTTPRequest needs that many +# args, and for the version-differentiated import of tornado's # httputil. # NOTE(jaypipes): The disable-msg=E1101 and E1103 below is because pylint is # unable to introspect the deferred's return value properly @@ -224,7 +224,8 @@ class ApiEc2TestCase(test.BaseTestCase): for x in range(random.randint(4, 8))) user = self.manager.create_user('fake', 'fake', 'fake') project = self.manager.create_project('fake', 'fake', 'fake') - self.manager.generate_key_pair(user.id, keyname) + # NOTE(vish): create depends on pool, so call helper directly + cloud._gen_key(None, user.id, keyname) rv = self.ec2.get_all_key_pairs() results = [k for k in rv if k.name == keyname] diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index b54e68274..1b4e12677 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -17,8 +17,6 @@ # under the License. import logging -from M2Crypto import BIO -from M2Crypto import RSA from M2Crypto import X509 import unittest @@ -65,35 +63,6 @@ class AuthTestCase(test.BaseTestCase): 'export S3_URL="http://127.0.0.1:3333/"\n' + 'export EC2_USER_ID="test1"\n') - def test_006_test_key_storage(self): - user = self.manager.get_user('test1') - user.create_key_pair('public', 'key', 'fingerprint') - key = user.get_key_pair('public') - self.assertEqual('key', key.public_key) - self.assertEqual('fingerprint', key.fingerprint) - - def test_007_test_key_generation(self): - user = self.manager.get_user('test1') - private_key, fingerprint = user.generate_key_pair('public2') - key = RSA.load_key_string(private_key, callback=lambda: None) - bio = BIO.MemoryBuffer() - public_key = user.get_key_pair('public2').public_key - key.save_pub_key_bio(bio) - converted = crypto.ssl_pub_to_ssh_pub(bio.read()) - # assert key fields are equal - self.assertEqual(public_key.split(" ")[1].strip(), - converted.split(" ")[1].strip()) - - def test_008_can_list_key_pairs(self): - keys = self.manager.get_user('test1').get_key_pairs() - self.assertTrue(filter(lambda k: k.name == 'public', keys)) - self.assertTrue(filter(lambda k: k.name == 'public2', keys)) - - def test_009_can_delete_key_pair(self): - self.manager.get_user('test1').delete_key_pair('public') - keys = self.manager.get_user('test1').get_key_pairs() - self.assertFalse(filter(lambda k: k.name == 'public', keys)) - def test_010_can_list_users(self): users = self.manager.get_users() logging.warn(users) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 29947e03c..4bad25c2b 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -17,13 +17,18 @@ # under the License. import logging +from M2Crypto import BIO +from M2Crypto import RSA import StringIO import time + from tornado import ioloop from twisted.internet import defer import unittest from xml.etree import ElementTree +from nova import crypto +from nova import db from nova import flags from nova import rpc from nova import test @@ -54,16 +59,21 @@ class CloudTestCase(test.BaseTestCase): proxy=self.compute) self.injected.append(self.compute_consumer.attach_to_tornado(self.ioloop)) - try: - manager.AuthManager().create_user('admin', 'admin', 'admin') - except: pass - admin = manager.AuthManager().get_user('admin') - project = manager.AuthManager().create_project('proj', 'admin', 'proj') - self.context = api.APIRequestContext(handler=None,project=project,user=admin) + self.manager = manager.AuthManager() + self.user = self.manager.create_user('admin', 'admin', 'admin', True) + self.project = self.manager.create_project('proj', 'admin', 'proj') + self.context = api.APIRequestContext(handler=None, + user=self.user, + project=self.project) def tearDown(self): - manager.AuthManager().delete_project('proj') - manager.AuthManager().delete_user('admin') + self.manager.delete_project(self.project) + self.manager.delete_user(self.user) + super(CloudTestCase, self).setUp() + + def _create_key(self, name): + # NOTE(vish): create depends on pool, so just call helper directly + return cloud._gen_key(self.context, self.context.user.id, name) def test_console_output(self): if FLAGS.connection_type == 'fake': @@ -76,6 +86,33 @@ class CloudTestCase(test.BaseTestCase): self.assert_(output) rv = yield self.compute.terminate_instance(instance_id) + + def test_key_generation(self): + result = self._create_key('test') + private_key = result['private_key'] + key = RSA.load_key_string(private_key, callback=lambda: None) + bio = BIO.MemoryBuffer() + public_key = db.keypair_get(self.context, + self.context.user.id, + 'test')['public_key'] + key.save_pub_key_bio(bio) + converted = crypto.ssl_pub_to_ssh_pub(bio.read()) + # assert key fields are equal + self.assertEqual(public_key.split(" ")[1].strip(), + converted.split(" ")[1].strip()) + + def test_describe_key_pairs(self): + self._create_key('test1') + self._create_key('test2') + result = self.cloud.describe_key_pairs(self.context) + keys = result["keypairsSet"] + self.assertTrue(filter(lambda k: k['keyName'] == 'test1', keys)) + self.assertTrue(filter(lambda k: k['keyName'] == 'test2', keys)) + + def test_delete_key_pair(self): + self._create_key('test') + self.cloud.delete_key_pair(self.context, 'test') + def test_run_instances(self): if FLAGS.connection_type == 'fake': logging.debug("Can't test instances without a real virtual env.") -- cgit From 38070f19036dcf24367429bcc79ffb55fad4b3cd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Sep 2010 22:42:51 -0700 Subject: it is called regionEndpoint, and use pipe as a separator --- nova/endpoint/cloud.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 180af0540..02eb50b19 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -177,12 +177,12 @@ class CloudController(object): if FLAGS.region_list: regions = [] for region in FLAGS.region_list: - name, _sep, url = region.partition(',') + name, _sep, url = region.partition('|') regions.append({'regionName': name, - 'regionUrl': url}) + 'regionEndpoint': url}) else: regions = [{'regionName': 'nova', - 'regionUrl': FLAGS.ec2_url}] + 'regionEndpoint': FLAGS.ec2_url}] if region_name: regions = [r for r in regions if r['regionName'] in region_name] return {'regionInfo': regions } -- cgit From 66c583b1883af6e3452271df4b302fd32d1ee25d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 04:18:30 -0700 Subject: fixed old key reference and made keypair name constistent -> key_pair --- nova/auth/manager.py | 4 ++-- nova/cloudpipe/pipelib.py | 4 ++-- nova/crypto.py | 2 +- nova/db/api.py | 30 +++++++++++++++--------------- nova/db/sqlalchemy/api.py | 28 ++++++++++++++-------------- nova/db/sqlalchemy/models.py | 6 +++--- nova/endpoint/cloud.py | 21 ++++++++++----------- nova/tests/cloud_unittest.py | 2 +- 8 files changed, 48 insertions(+), 49 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index fb87847d5..4e321c1bd 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -604,9 +604,9 @@ class AuthManager(object): def delete_user(self, user): """Deletes a user - Additionally deletes all users keypairs""" + Additionally deletes all users key_pairs""" uid = User.safe_id(user) - db.keypair_destroy_all_by_user(None, uid) + db.key_pair_destroy_all_by_user(None, uid) with self.driver() as drv: drv.delete_user(uid) diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 2867bcb21..de6a97fb6 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -58,7 +58,7 @@ class CloudPipe(object): z.write(FLAGS.boot_script_template,'autorun.sh') z.close() - key_name = self.setup_keypair(project.project_manager_id, project_id) + key_name = self.setup_key_pair(project.project_manager_id, project_id) zippy = open(zippath, "r") context = api.APIRequestContext(handler=None, user=project.project_manager, project=project) @@ -74,7 +74,7 @@ class CloudPipe(object): security_groups=["vpn-secgroup"]) zippy.close() - def setup_keypair(self, user_id, project_id): + def setup_key_pair(self, user_id, project_id): key_name = '%s%s' % (project_id, FLAGS.vpn_key_suffix) try: private_key, fingerprint = self.manager.generate_key_pair(user_id, key_name) diff --git a/nova/crypto.py b/nova/crypto.py index b05548ea1..1c6fe57ad 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -18,7 +18,7 @@ """ Wrappers around standard crypto, including root and intermediate CAs, -SSH keypairs and x509 certificates. +SSH key_pairs and x509 certificates. """ import base64 diff --git a/nova/db/api.py b/nova/db/api.py index e96d803db..507f70dd5 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -257,29 +257,29 @@ def instance_update(context, instance_id, values): ################### -def keypair_create(context, values): - """Create a keypair from the values dictionary.""" - return IMPL.keypair_create(context, values) +def key_pair_create(context, values): + """Create a key_pair from the values dictionary.""" + return IMPL.key_pair_create(context, values) -def keypair_destroy(context, user_id, name): - """Destroy the keypair or raise if it does not exist.""" - return IMPL.keypair_destroy(context, user_id, name) +def key_pair_destroy(context, user_id, name): + """Destroy the key_pair or raise if it does not exist.""" + return IMPL.key_pair_destroy(context, user_id, name) -def keypair_destroy_all_by_user(context, user_id): - """Destroy all keypairs by user.""" - return IMPL.keypair_destroy_all_by_user(context, user_id) +def key_pair_destroy_all_by_user(context, user_id): + """Destroy all key_pairs by user.""" + return IMPL.key_pair_destroy_all_by_user(context, user_id) -def keypair_get(context, user_id, name): - """Get a keypair or raise if it does not exist.""" - return IMPL.keypair_get(context, user_id, name) +def key_pair_get(context, user_id, name): + """Get a key_pair or raise if it does not exist.""" + return IMPL.key_pair_get(context, user_id, name) -def keypair_get_all_by_user(context, user_id): - """Get all keypairs by user.""" - return IMPL.keypair_get_all_by_user(context, user_id) +def key_pair_get_all_by_user(context, user_id): + """Get all key_pairs by user.""" + return IMPL.key_pair_get_all_by_user(context, user_id) #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 4fd1bf216..ce97f6710 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -355,38 +355,38 @@ def instance_update(_context, instance_id, values): ################### -def keypair_create(_context, values): - keypair_ref = models.Keypair() +def key_pair_create(_context, values): + key_pair_ref = models.KeyPair() for (key, value) in values.iteritems(): - keypair_ref[key] = value - keypair_ref.save() - return keypair_ref + key_pair_ref[key] = value + key_pair_ref.save() + return key_pair_ref -def keypair_destroy(_context, user_id, name): +def key_pair_destroy(_context, user_id, name): session = get_session() with session.begin(): - keypair_ref = models.Keypair.find_by_args(user_id, + key_pair_ref = models.KeyPair.find_by_args(user_id, name, session=session) - keypair_ref.delete(session=session) + key_pair_ref.delete(session=session) -def keypair_destroy_all_by_user(_context, user_id): +def key_pair_destroy_all_by_user(_context, user_id): session = get_session() with session.begin(): # TODO(vish): do we have to use sql here? - session.execute('update keypairs set deleted=1 where user_id=:id', + session.execute('update key_pairs set deleted=1 where user_id=:id', {'id': user_id}) -def keypair_get(_context, user_id, name): - return models.Keypair.find_by_args(user_id, name) +def key_pair_get(_context, user_id, name): + return models.KeyPair.find_by_args(user_id, name) -def keypair_get_all_by_user(_context, user_id): +def key_pair_get_all_by_user(_context, user_id): session = get_session() - return session.query(models.Keypair + return session.query(models.KeyPair ).filter_by(user_id=user_id ).filter_by(deleted=False ).all() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 81c0a77a8..0ecc48bae 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -284,9 +284,9 @@ class ExportDevice(BASE, NovaBase): uselist=False)) -class Keypair(BASE, NovaBase): - """Represents a keypair""" - __tablename__ = 'keypairs' +class KeyPair(BASE, NovaBase): + """Represents a public key pair for ssh""" + __tablename__ = 'key_pairs' id = Column(Integer, primary_key=True) name = Column(String(255)) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 172c65d79..f30565aca 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -51,10 +51,10 @@ def _gen_key(context, user_id, key_name): it into a process pool.""" try: # NOTE(vish): generating key pair is slow so check for legal - # creation before creating keypair + # creation before creating key_pair try: - db.keypair_get(context, user_id, key_name) - raise exception.Duplicate("The keypair %s already exists" + db.key_pair_get(context, user_id, key_name) + raise exception.Duplicate("The key_pair %s already exists" % key_name) except exception.NotFound: pass @@ -64,7 +64,7 @@ def _gen_key(context, user_id, key_name): key['name'] = key_name key['public_key'] = public_key key['fingerprint'] = fingerprint - db.keypair_create(context, key) + db.key_pair_create(context, key) return {'private_key': private_key, 'fingerprint': fingerprint} except Exception as ex: return {'exception': ex} @@ -193,7 +193,7 @@ class CloudController(object): @rbac.allow('all') def describe_key_pairs(self, context, key_name=None, **kwargs): - key_pairs = db.keypair_get_all_by_user(context, context.user.id) + key_pairs = db.key_pair_get_all_by_user(context, context.user.id) if not key_name is None: key_pairs = [x for x in key_pairs if x['name'] in key_name] @@ -228,7 +228,7 @@ class CloudController(object): @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): try: - db.keypair_destroy(context, context.user.id, key_name) + db.key_pair_destroy(context, context.user.id, key_name) except exception.NotFound: # aws returns true even if the key doesn't exist pass @@ -545,11 +545,10 @@ class CloudController(object): launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) key_data = None if kwargs.has_key('key_name'): - key_pair = context.user.get_key_pair(kwargs['key_name']) - if not key_pair: - raise exception.ApiError('Key Pair %s not found' % - kwargs['key_name']) - key_data = key_pair.public_key + key_pair_ref = db.key_pair_get(context, + context.user.id, + kwargs['key_name']) + key_data = key_pair_ref['public_key'] # TODO: Get the real security group of launch in here security_group = "default" diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 4bad25c2b..e56ea6ac2 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -92,7 +92,7 @@ class CloudTestCase(test.BaseTestCase): private_key = result['private_key'] key = RSA.load_key_string(private_key, callback=lambda: None) bio = BIO.MemoryBuffer() - public_key = db.keypair_get(self.context, + public_key = db.key_pair_get(self.context, self.context.user.id, 'test')['public_key'] key.save_pub_key_bio(bio) -- cgit From 619e9fd636854b55e7f3334f93ed759ff82759f0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 04:48:37 -0700 Subject: fixed typo network => network_manager in cloud.py --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 622b4e2a4..0eedd9fec 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -613,7 +613,7 @@ class CloudController(object): # NOTE(vish): Currently, nothing needs to be done on the # network node until release. If this changes, # we will need to cast here. - self.network.deallocate_fixed_ip(context, address) + self.network_manager.deallocate_fixed_ip(context, address) host = instance_ref['host'] if host: -- cgit From 5b9908ff2601adfac3565ff900ef254df27102b9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 06:29:13 -0700 Subject: fixed reversed admin logic on describe instances --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 0eedd9fec..a25598dc8 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -366,7 +366,7 @@ class CloudController(object): instances = db.instance_get_by_reservation(context, reservation_id) else: - if not context.user.is_admin(): + if context.user.is_admin(): instances = db.instance_get_all(context) else: instances = db.instance_get_by_project(context, -- cgit From c000a1f88141c7887943a96a8a7ced3b79d70f7e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 08:43:48 -0700 Subject: added terminated_at to volume and moved setting of terminated_at into cloud --- nova/compute/manager.py | 8 ++++---- nova/db/sqlalchemy/models.py | 2 ++ nova/endpoint/cloud.py | 9 +++++++++ nova/volume/manager.py | 3 +++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ae7099812..954227b42 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -85,7 +85,9 @@ class ComputeManager(manager.Manager): try: yield self.driver.spawn(instance_ref) now = datetime.datetime.utcnow() - self.db.instance_update(None, instance_id, {'launched_at': now}) + self.db.instance_update(context, + instance_id, + {'launched_at': now}) except Exception: # pylint: disable-msg=W0702 logging.exception("instance %s: Failed to spawn", instance_ref['name']) @@ -100,8 +102,8 @@ class ComputeManager(manager.Manager): def terminate_instance(self, context, instance_id): """Terminate an instance on this machine.""" logging.debug("instance %s: terminating", instance_id) - instance_ref = self.db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) if instance_ref['state'] == power_state.SHUTOFF: self.db.instance_destroy(context, instance_id) raise exception.Error('trying to destroy already destroyed' @@ -112,8 +114,6 @@ class ComputeManager(manager.Manager): power_state.NOSTATE, 'shutting_down') yield self.driver.destroy(instance_ref) - now = datetime.datetime.utcnow() - self.db.instance_update(None, instance_id, {'terminated_at': now}) # TODO(ja): should we keep it in a terminated state for a bit? self.db.instance_destroy(context, instance_id) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 09bd4b4dc..fde153dc4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -274,6 +274,8 @@ class Volume(BASE, NovaBase): attach_status = Column(String(255)) # TODO(vish): enum scheduled_at = Column(DateTime) + launched_at = Column(DateTime) + terminated_at = Column(DateTime) class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on""" diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 4b82e6d4d..faa646b53 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -23,6 +23,7 @@ datastore. """ import base64 +import datetime import logging import os import time @@ -594,6 +595,10 @@ class CloudController(object): % id_str) continue + now = datetime.datetime.utcnow() + self.db.instance_update(context, + instance_ref['id'], + {'terminated_at': now}) # FIXME(ja): where should network deallocate occur? address = db.instance_get_floating_address(context, instance_ref['id']) @@ -643,6 +648,10 @@ class CloudController(object): def delete_volume(self, context, volume_id, **kwargs): # TODO: return error if not authorized volume_ref = db.volume_get_by_str(context, volume_id) + now = datetime.datetime.utcnow() + self.db.volume_update(context, + volume_ref['id'], + {'terminated_at': now}) host = volume_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.volume_topic, host), {"method": "delete_volume", diff --git a/nova/volume/manager.py b/nova/volume/manager.py index a6f4a6baf..7ca03b319 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -22,6 +22,7 @@ destroying persistent storage volumes, ala EBS. """ import logging +import datetime from twisted.internet import defer @@ -97,6 +98,8 @@ class AOEManager(manager.Manager): logging.debug("volume %s: re-exporting all values", volume_id) yield self.driver.ensure_exports() + now = datetime.datetime.utcnow() + self.db.volume_update(context, volume_id, {'launched_at': now}) logging.debug("volume %s: created successfully", volume_id) defer.returnValue(volume_id) -- cgit From 023c7c018cfad28d0f53a73fa7d211427ad8339b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 17:12:43 -0700 Subject: db not self.db --- nova/endpoint/cloud.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index faa646b53..9d8e45f30 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -596,9 +596,9 @@ class CloudController(object): continue now = datetime.datetime.utcnow() - self.db.instance_update(context, - instance_ref['id'], - {'terminated_at': now}) + db.instance_update(context, + instance_ref['id'], + {'terminated_at': now}) # FIXME(ja): where should network deallocate occur? address = db.instance_get_floating_address(context, instance_ref['id']) @@ -649,9 +649,7 @@ class CloudController(object): # TODO: return error if not authorized volume_ref = db.volume_get_by_str(context, volume_id) now = datetime.datetime.utcnow() - self.db.volume_update(context, - volume_ref['id'], - {'terminated_at': now}) + db.volume_update(context, volume_ref['id'], {'terminated_at': now}) host = volume_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.volume_topic, host), {"method": "delete_volume", -- cgit From 06a799d2668723bbaead7ca2afbfb4b0cbf28abb Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 18:16:10 -0700 Subject: use a string version of key name when constructing mpi dict because None doesn't work well in lookup --- nova/endpoint/cloud.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 622b4e2a4..45291ca34 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -88,10 +88,11 @@ class CloudController(object): if instance['fixed_ip']: line = '%s slots=%d' % (instance['fixed_ip']['str_id'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) - if instance['key_name'] in result: - result[instance['key_name']].append(line) + key = str(instance['key_name']) + if key in result: + result[key].append(line) else: - result[instance['key_name']] = [line] + result[key] = [line] return result def get_metadata(self, address): -- cgit From 8e4f102819a1424a25f89ed34040b1298ed9563a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 18:45:15 -0700 Subject: use gigabytes and cores --- nova/db/sqlalchemy/api.py | 8 ++++---- nova/scheduler/simple.py | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 735e88145..8ca0f790b 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -85,9 +85,9 @@ def service_get_all_compute_sorted(context): # FROM instances GROUP BY host) AS inst_count # ON services.host = inst_count.host topic = 'compute' - label = 'instance_count' + label = 'instance_cores' subq = session.query(models.Instance.host, - func.count('*').label(label) + func.sum('cores').label(label) ).filter_by(deleted=False ).group_by(models.Instance.host ).subquery() @@ -119,9 +119,9 @@ def service_get_all_volume_sorted(context): session = get_session() with session.begin(): topic = 'volume' - label = 'volume_count' + label = 'volume_gigabytes' subq = session.query(models.Volume.host, - func.count('*').label(label) + func.count('size').label(label) ).filter_by(deleted=False ).group_by(models.Volume.host ).subquery() diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 48be4c1a6..6e77debf3 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -27,10 +27,10 @@ from nova.scheduler import driver from nova.scheduler import chance FLAGS = flags.FLAGS -flags.DEFINE_integer("max_instances", 16, - "maximum number of instances to allow per host") -flags.DEFINE_integer("max_volumes", 100, - "maximum number of volumes to allow per host") +flags.DEFINE_integer("max_cores", 16, + "maximum number of instance cores to allow per host") +flags.DEFINE_integer("max_gigabytes", 10000, + "maximum number of volume gigabytes to allow per host") flags.DEFINE_integer("max_networks", 1000, "maximum number of networks to allow per host") @@ -42,9 +42,9 @@ class SimpleScheduler(chance.ChanceScheduler): results = db.service_get_all_compute_sorted(context) for result in results: - (service, instance_count) = result - if instance_count >= FLAGS.max_instances: - raise driver.NoValidHost("All hosts have too many instances") + (service, instance_cores) = result + if instance_cores >= FLAGS.max_cores: + raise driver.NoValidHost("All hosts have too many cores") if self.service_is_up(service): db.instance_update(context, instance_id, @@ -57,13 +57,13 @@ class SimpleScheduler(chance.ChanceScheduler): results = db.service_get_all_volume_sorted(context) for result in results: - (service, instance_count) = result - if instance_count >= FLAGS.max_volumes: - raise driver.NoValidHost("All hosts have too many volumes") + (service, volume_gigabytes) = result + if volume_gigabytes >= FLAGS.max_gigabytes: + raise driver.NoValidHost("All hosts have too many gigabytes") if self.service_is_up(service): - db.instance_update(context, - volume_id, - {'host': service['host']}) + db.volume_update(context, + volume_id, + {'host': service['host']}) return service['host'] raise driver.NoValidHost("No hosts found") -- cgit From 68ff059c7a6287825871f96cde8039f04aec1f37 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 18:57:15 -0700 Subject: update query and test --- nova/db/sqlalchemy/api.py | 12 ++++++------ nova/scheduler/simple.py | 1 + nova/tests/scheduler_unittest.py | 10 +++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 574a6f460..75131e093 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -79,15 +79,15 @@ def service_get_all_compute_sorted(context): session = get_session() with session.begin(): # NOTE(vish): The intended query is below - # SELECT services.*, inst_count.instance_count + # SELECT services.*, inst_cores.instance_cores # FROM services LEFT OUTER JOIN - # (SELECT host, count(*) AS instance_count - # FROM instances GROUP BY host) AS inst_count - # ON services.host = inst_count.host + # (SELECT host, sum(instances.vcpus) AS instance_cores + # FROM instances GROUP BY host) AS inst_cores + # ON services.host = inst_cores.host topic = 'compute' label = 'instance_cores' subq = session.query(models.Instance.host, - func.sum('cores').label(label) + func.sum(models.Instance.vcpus).label(label) ).filter_by(deleted=False ).group_by(models.Instance.host ).subquery() @@ -121,7 +121,7 @@ def service_get_all_volume_sorted(context): topic = 'volume' label = 'volume_gigabytes' subq = session.query(models.Volume.host, - func.count('size').label(label) + func.sum(models.Volume.size).label(label) ).filter_by(deleted=False ).group_by(models.Volume.host ).subquery() diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 6e77debf3..3feeca846 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -43,6 +43,7 @@ class SimpleScheduler(chance.ChanceScheduler): results = db.service_get_all_compute_sorted(context) for result in results: (service, instance_cores) = result + print service, instance_cores if instance_cores >= FLAGS.max_cores: raise driver.NoValidHost("All hosts have too many cores") if self.service_is_up(service): diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 27e100fa0..b9371e86d 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -33,7 +33,7 @@ from nova.scheduler import driver FLAGS = flags.FLAGS -flags.DECLARE('max_instances', 'nova.scheduler.simple') +flags.DECLARE('max_cores', 'nova.scheduler.simple') class TestDriver(driver.Scheduler): """Scheduler Driver for Tests""" @@ -75,7 +75,7 @@ class SimpleDriverTestCase(test.TrialTestCase): def setUp(self): # pylint: disable-msg=C0103 super(SimpleDriverTestCase, self).setUp() self.flags(connection_type='fake', - max_instances=4, + max_cores=4, scheduler_driver='nova.scheduler.simple.SimpleScheduler') self.scheduler = manager.SchedulerManager() self.context = None @@ -109,6 +109,7 @@ class SimpleDriverTestCase(test.TrialTestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 + inst['vcpus'] = 1 return db.instance_create(self.context, inst)['id'] def test_hosts_are_up(self): @@ -125,10 +126,10 @@ class SimpleDriverTestCase(test.TrialTestCase): self.assertEqual(host, 'host2') self.service1.terminate_instance(self.context, instance_id) - def test_too_many_instances(self): + def test_too_many_cores(self): instance_ids1 = [] instance_ids2 = [] - for index in xrange(FLAGS.max_instances): + for index in xrange(FLAGS.max_cores): instance_id = self._create_instance() self.service1.run_instance(self.context, instance_id) instance_ids1.append(instance_id) @@ -139,7 +140,6 @@ class SimpleDriverTestCase(test.TrialTestCase): self.assertRaises(driver.NoValidHost, self.scheduler.driver.schedule_run_instance, self.context, - 'compute', instance_id) for instance_id in instance_ids1: self.service1.terminate_instance(self.context, instance_id) -- cgit From d05fe5d18ba3a62a1792634e7ba3c2f11d7b89bd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 19:40:38 -0700 Subject: tests for volumes work --- nova/db/sqlalchemy/api.py | 9 +-- nova/scheduler/simple.py | 9 ++- nova/tests/scheduler_unittest.py | 126 ++++++++++++++++++++++++++++++++------- 3 files changed, 114 insertions(+), 30 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 75131e093..d612fe669 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -67,7 +67,7 @@ def service_get_all_by_topic(context, topic): def _service_get_all_topic_subquery(_context, session, topic, subq, label): sort_value = getattr(subq.c, label) - return session.query(models.Service, sort_value + return session.query(models.Service, func.coalesce(sort_value, 0) ).filter_by(topic=topic ).filter_by(deleted=False ).outerjoin((subq, models.Service.host == subq.c.host) @@ -79,9 +79,10 @@ def service_get_all_compute_sorted(context): session = get_session() with session.begin(): # NOTE(vish): The intended query is below - # SELECT services.*, inst_cores.instance_cores + # SELECT services.*, COALESCE(inst_cores.instance_cores, + # 0) # FROM services LEFT OUTER JOIN - # (SELECT host, sum(instances.vcpus) AS instance_cores + # (SELECT host, SUM(instances.vcpus) AS instance_cores # FROM instances GROUP BY host) AS inst_cores # ON services.host = inst_cores.host topic = 'compute' @@ -104,7 +105,7 @@ def service_get_all_network_sorted(context): topic = 'network' label = 'network_count' subq = session.query(models.Network.host, - func.count('*').label(label) + func.count(models.Network.id).label(label) ).filter_by(deleted=False ).group_by(models.Network.host ).subquery() diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 3feeca846..c4ba17caf 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -39,12 +39,11 @@ class SimpleScheduler(chance.ChanceScheduler): def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): """Picks a host that is up and has the fewest running instances.""" - + instance_ref = db.instance_get(context, instance_id) results = db.service_get_all_compute_sorted(context) for result in results: (service, instance_cores) = result - print service, instance_cores - if instance_cores >= FLAGS.max_cores: + if instance_cores + instance_ref['vcpus'] > FLAGS.max_cores: raise driver.NoValidHost("All hosts have too many cores") if self.service_is_up(service): db.instance_update(context, @@ -55,11 +54,11 @@ class SimpleScheduler(chance.ChanceScheduler): def schedule_create_volume(self, context, volume_id, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" - + volume_ref = db.volume_get(context, volume_id) results = db.service_get_all_volume_sorted(context) for result in results: (service, volume_gigabytes) = result - if volume_gigabytes >= FLAGS.max_gigabytes: + if volume_gigabytes + volume_ref['size'] > FLAGS.max_gigabytes: raise driver.NoValidHost("All hosts have too many gigabytes") if self.service_is_up(service): db.volume_update(context, diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index b9371e86d..fde30f81e 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -19,8 +19,6 @@ Tests For Scheduler """ -import mox - from nova import db from nova import flags from nova import service @@ -76,6 +74,8 @@ class SimpleDriverTestCase(test.TrialTestCase): super(SimpleDriverTestCase, self).setUp() self.flags(connection_type='fake', max_cores=4, + max_gigabytes=4, + volume_driver='nova.volume.driver.FakeAOEDriver', scheduler_driver='nova.scheduler.simple.SimpleScheduler') self.scheduler = manager.SchedulerManager() self.context = None @@ -83,27 +83,16 @@ class SimpleDriverTestCase(test.TrialTestCase): self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = None - self.service1 = service.Service('host1', - 'nova-compute', - 'compute', - FLAGS.compute_manager) - self.service2 = service.Service('host2', - 'nova-compute', - 'compute', - FLAGS.compute_manager) def tearDown(self): # pylint: disable-msg=C0103 self.manager.delete_user(self.user) self.manager.delete_project(self.project) - self.service1.kill() - self.service2.kill() def _create_instance(self): """Create a test instance""" inst = {} inst['image_id'] = 'ami-test' inst['reservation_id'] = 'r-fakeres' - inst['launch_time'] = '10' inst['user_id'] = self.user.id inst['project_id'] = self.project.id inst['instance_type'] = 'm1.tiny' @@ -112,29 +101,70 @@ class SimpleDriverTestCase(test.TrialTestCase): inst['vcpus'] = 1 return db.instance_create(self.context, inst)['id'] + def _create_volume(self): + """Create a test volume""" + vol = {} + vol['image_id'] = 'ami-test' + vol['reservation_id'] = 'r-fakeres' + vol['size'] = 1 + return db.volume_create(self.context, vol)['id'] + def test_hosts_are_up(self): + """Ensures driver can find the hosts that are up""" # NOTE(vish): constructing service without create method # because we are going to use it without queue + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(len(hosts), 2) + compute1.kill() + compute2.kill() def test_least_busy_host_gets_instance(self): - instance_id = self._create_instance() - self.service1.run_instance(self.context, instance_id) + """Ensures the host with less cores gets the next one""" + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + instance_id1 = self._create_instance() + compute1.run_instance(self.context, instance_id1) + instance_id2 = self._create_instance() host = self.scheduler.driver.schedule_run_instance(self.context, - instance_id) + instance_id2) self.assertEqual(host, 'host2') - self.service1.terminate_instance(self.context, instance_id) + compute1.terminate_instance(self.context, instance_id1) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + compute2.kill() def test_too_many_cores(self): + """Ensures we don't go over max cores""" + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): instance_id = self._create_instance() - self.service1.run_instance(self.context, instance_id) + compute1.run_instance(self.context, instance_id) instance_ids1.append(instance_id) instance_id = self._create_instance() - self.service2.run_instance(self.context, instance_id) + compute2.run_instance(self.context, instance_id) instance_ids2.append(instance_id) instance_id = self._create_instance() self.assertRaises(driver.NoValidHost, @@ -142,6 +172,60 @@ class SimpleDriverTestCase(test.TrialTestCase): self.context, instance_id) for instance_id in instance_ids1: - self.service1.terminate_instance(self.context, instance_id) + compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: - self.service2.terminate_instance(self.context, instance_id) + compute2.terminate_instance(self.context, instance_id) + compute1.kill() + compute2.kill() + + def test_least_busy_host_gets_volume(self): + """Ensures the host with less gigabytes gets the next one""" + volume1 = service.Service('host1', + 'nova-volume', + 'volume', + FLAGS.volume_manager) + volume2 = service.Service('host2', + 'nova-volume', + 'volume', + FLAGS.volume_manager) + volume_id1 = self._create_volume() + volume1.create_volume(self.context, volume_id1) + volume_id2 = self._create_volume() + host = self.scheduler.driver.schedule_create_volume(self.context, + volume_id2) + self.assertEqual(host, 'host2') + volume1.delete_volume(self.context, volume_id1) + db.volume_destroy(self.context, volume_id2) + volume1.kill() + volume2.kill() + + def test_too_many_gigabytes(self): + """Ensures we don't go over max gigabytes""" + volume1 = service.Service('host1', + 'nova-volume', + 'volume', + FLAGS.volume_manager) + volume2 = service.Service('host2', + 'nova-volume', + 'volume', + FLAGS.volume_manager) + volume_ids1 = [] + volume_ids2 = [] + for index in xrange(FLAGS.max_gigabytes): + volume_id = self._create_volume() + volume1.create_volume(self.context, volume_id) + volume_ids1.append(volume_id) + volume_id = self._create_volume() + volume2.create_volume(self.context, volume_id) + volume_ids2.append(volume_id) + volume_id = self._create_volume() + self.assertRaises(driver.NoValidHost, + self.scheduler.driver.schedule_create_volume, + self.context, + volume_id) + for volume_id in volume_ids1: + volume1.delete_volume(self.context, volume_id) + for volume_id in volume_ids2: + volume2.delete_volume(self.context, volume_id) + volume1.kill() + volume2.kill() -- cgit From 15ca1fe1670cfd95880f2e1c2a5270be787c6035 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 19:43:02 -0700 Subject: move volume to the scheduler --- nova/endpoint/cloud.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 4fda484e3..584c9c643 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -300,9 +300,11 @@ class CloudController(object): vol['attach_status'] = "detached" volume_ref = db.volume_create(context, vol) - rpc.cast(FLAGS.volume_topic, {"method": "create_volume", - "args": {"context": None, - "volume_id": volume_ref['id']}}) + rpc.cast(FLAGS.scheduler_topic, + {"method": "create_volume", + "args": {"context": None, + "topic": FLAGS.volume_topic, + "volume_id": volume_ref['id']}}) return {'volumeSet': [self._format_volume(context, volume_ref)]} -- cgit From ef1913292dd8a88041f603d79c09c738a7ecbb04 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 20:00:02 -0700 Subject: fix instance time --- nova/db/sqlalchemy/models.py | 2 +- nova/tests/compute_unittest.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index fde153dc4..c16f684fe 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -103,7 +103,7 @@ class NovaBase(object): def delete(self, session=None): """Delete this object""" self.deleted = True - self.deleted_at = datetime.datetime.now() + self.deleted_at = datetime.datetime.utcnow() self.save(session=session) def __setitem__(self, key, value): diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index de2bf3d3b..c983e05c9 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -83,21 +83,21 @@ class ComputeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_terminate_timestamps(self): - """Make sure it is possible to run and terminate instance""" + """Make sure timestamps are set for launched and destroyed""" instance_id = self._create_instance() instance_ref = db.instance_get(self.context, instance_id) self.assertEqual(instance_ref['launched_at'], None) - self.assertEqual(instance_ref['terminated_at'], None) + self.assertEqual(instance_ref['deleted_at'], None) launch = datetime.datetime.utcnow() yield self.compute.run_instance(self.context, instance_id) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] > launch) - self.assertEqual(instance_ref['terminated_at'], None) + self.assertEqual(instance_ref['deleted_at'], None) terminate = datetime.datetime.utcnow() yield self.compute.terminate_instance(self.context, instance_id) instance_ref = db.instance_get({'deleted': True}, instance_id) self.assert_(instance_ref['launched_at'] < terminate) - self.assert_(instance_ref['terminated_at'] > terminate) + self.assert_(instance_ref['deleted_at'] > terminate) @defer.inlineCallbacks def test_reboot(self): -- cgit From dff482e992b25580728955ae83ea2e38a18e7736 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 20:54:12 -0700 Subject: manage command for project quotas --- bin/nova-manage | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 325245ac4..c7ef33a48 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -50,7 +50,6 @@ """ CLI interface for nova management. - Connects to the running ADMIN api in the api daemon. """ import os @@ -68,7 +67,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) from nova import db +from nova import exception from nova import flags +from nova import quota from nova import utils from nova.auth import manager from nova.cloudpipe import pipelib @@ -186,6 +187,13 @@ class RoleCommands(object): class UserCommands(object): """Class for managing users.""" + @staticmethod + def _print_export(user): + """Print export variables to use with API.""" + print 'export EC2_ACCESS_KEY=%s' % user.access + print 'export EC2_SECRET_KEY=%s' % user.secret + + def __init__(self): self.manager = manager.AuthManager() @@ -193,13 +201,13 @@ class UserCommands(object): """creates a new admin and prints exports arguments: name [access] [secret]""" user = self.manager.create_user(name, access, secret, True) - print_export(user) + self._print_export(user) def create(self, name, access=None, secret=None): """creates a new user and prints exports arguments: name [access] [secret]""" user = self.manager.create_user(name, access, secret, False) - print_export(user) + self._print_export(user) def delete(self, name): """deletes an existing user @@ -211,7 +219,7 @@ class UserCommands(object): arguments: name""" user = self.manager.get_user(name) if user: - print_export(user) + self._print_export(user) else: print "User %s doesn't exist" % name @@ -222,12 +230,6 @@ class UserCommands(object): print user.name -def print_export(user): - """Print export variables to use with API.""" - print 'export EC2_ACCESS_KEY=%s' % user.access - print 'export EC2_SECRET_KEY=%s' % user.secret - - class ProjectCommands(object): """Class for managing projects.""" @@ -262,6 +264,19 @@ class ProjectCommands(object): for project in self.manager.get_projects(): print project.name + def quota(self, project_id, key=None, value=None): + """Set or display quotas for project + arguments: project_id [key] [value]""" + if key: + quo = {'project_id': project_id, key: value} + try: + db.quota_update(None, project_id, quo) + except exception.NotFound: + db.quota_create(None, quo) + project_quota = quota._get_quota(None, project_id) + for key, value in project_quota.iteritems(): + print '%s: %s' % (key, value) + def remove(self, project, user): """Removes user from project arguments: project user""" @@ -274,6 +289,7 @@ class ProjectCommands(object): with open(filename, 'w') as f: f.write(zip_file) + class FloatingIpCommands(object): """Class for managing floating ip.""" @@ -306,6 +322,7 @@ class FloatingIpCommands(object): floating_ip['address'], instance) + CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), -- cgit From c7921fd14e680288c5626294105761005684b343 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 11 Sep 2010 22:48:59 -0700 Subject: don't allow deletion or attachment of volume unless it is available --- nova/endpoint/cloud.py | 8 ++++++-- nova/volume/manager.py | 7 ++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 94a04fb1b..6e2fedd69 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -314,6 +314,8 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_str(context, volume_id) # TODO(vish): abstract status checking? + if volume_ref['status'] != "available": + raise exception.ApiError("Volume status must be available") if volume_ref['attach_status'] == "attached": raise exception.ApiError("Volume is already attached") instance_ref = db.instance_get_by_str(context, instance_id) @@ -336,10 +338,10 @@ class CloudController(object): volume_ref = db.volume_get_by_str(context, volume_id) instance_ref = db.volume_get_instance(context, volume_ref['id']) if not instance_ref: - raise exception.Error("Volume isn't attached to anything!") + raise exception.ApiError("Volume isn't attached to anything!") # TODO(vish): abstract status checking? if volume_ref['status'] == "available": - raise exception.Error("Volume is already detached") + raise exception.ApiError("Volume is already detached") try: host = instance_ref['host'] rpc.cast(db.queue_get_for(context, FLAGS.compute_topic, host), @@ -691,6 +693,8 @@ class CloudController(object): def delete_volume(self, context, volume_id, **kwargs): # TODO: return error if not authorized volume_ref = db.volume_get_by_str(context, volume_id) + if volume_ref['status'] != "available": + raise exception.ApiError("Volume status must be available") now = datetime.datetime.utcnow() db.volume_update(context, volume_ref['id'], {'terminated_at': now}) host = volume_ref['host'] diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 7ca03b319..034763512 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -90,16 +90,13 @@ class AOEManager(manager.Manager): yield self.driver.create_export(volume_ref['str_id'], shelf_id, blade_id) - # TODO(joshua): We need to trigger a fanout message - # for aoe-discover on all the nodes - - self.db.volume_update(context, volume_id, {'status': 'available'}) logging.debug("volume %s: re-exporting all values", volume_id) yield self.driver.ensure_exports() now = datetime.datetime.utcnow() - self.db.volume_update(context, volume_id, {'launched_at': now}) + self.db.volume_update(context, volume_id, {'status': 'available', + 'launched_at': now}) logging.debug("volume %s: created successfully", volume_id) defer.returnValue(volume_id) -- cgit From 6cbf8b736cc2c9929c2ad69ddc8e8b4fc2d0f4ae Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 12 Sep 2010 23:09:15 -0700 Subject: removed second copy of ProcessExecutionError --- nova/utils.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 8939043e6..d18dd9843 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -39,17 +39,6 @@ from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -class ProcessExecutionError(IOError): - def __init__( self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - if description is None: - description = "Unexpected error while running command." - if exit_code is None: - exit_code = '-' - message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( - description, cmd, exit_code, stdout, stderr) - IOError.__init__(self, message) - def import_class(import_str): """Returns a class from a string including module and class""" mod_str, _sep, class_str = import_str.rpartition('.') -- cgit From 86cd30b749e6da78d4ceb6c77f2116975429a81a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 13 Sep 2010 01:15:35 -0700 Subject: renamed _get_quota to get_quota and moved int(size) into quota.py --- bin/nova-manage | 2 +- nova/endpoint/cloud.py | 1 - nova/quota.py | 9 +++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index c7ef33a48..824e00ac5 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -273,7 +273,7 @@ class ProjectCommands(object): db.quota_update(None, project_id, quo) except exception.NotFound: db.quota_create(None, quo) - project_quota = quota._get_quota(None, project_id) + project_quota = quota.get_quota(None, project_id) for key, value in project_quota.iteritems(): print '%s: %s' % (key, value) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 749bf5f9c..1618b784b 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -284,7 +284,6 @@ class CloudController(object): @rbac.allow('projectmanager', 'sysadmin') def create_volume(self, context, size, **kwargs): # check quota - size = int(size) if quota.allowed_volumes(context, 1, size) < 1: logging.warn("Quota exceeeded for %s, tried to create %sG volume", context.project.id, size) diff --git a/nova/quota.py b/nova/quota.py index f0e51feeb..edbb83111 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,7 +37,7 @@ flags.DEFINE_integer('quota_gigabytes', 1000, flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') -def _get_quota(context, project_id): +def get_quota(context, project_id): rval = {'instances': FLAGS.quota_instances, 'cores': FLAGS.quota_cores, 'volumes': FLAGS.quota_volumes, @@ -57,7 +57,7 @@ def allowed_instances(context, num_instances, instance_type): project_id = context.project.id used_instances, used_cores = db.instance_data_get_for_project(context, project_id) - quota = _get_quota(context, project_id) + quota = get_quota(context, project_id) allowed_instances = quota['instances'] - used_instances allowed_cores = quota['cores'] - used_cores type_cores = instance_types.INSTANCE_TYPES[instance_type]['vcpus'] @@ -72,9 +72,10 @@ def allowed_volumes(context, num_volumes, size): project_id = context.project.id used_volumes, used_gigabytes = db.volume_data_get_for_project(context, project_id) - quota = _get_quota(context, project_id) + quota = get_quota(context, project_id) allowed_volumes = quota['volumes'] - used_volumes allowed_gigabytes = quota['gigabytes'] - used_gigabytes + size = int(size) num_gigabytes = num_volumes * size allowed_volumes = min(allowed_volumes, int(allowed_gigabytes // size)) @@ -85,7 +86,7 @@ def allowed_floating_ips(context, num_floating_ips): """Check quota and return min(num_floating_ips, allowed_floating_ips)""" project_id = context.project.id used_floating_ips = db.floating_ip_count_by_project(context, project_id) - quota = _get_quota(context, project_id) + quota = get_quota(context, project_id) allowed_floating_ips = quota['floating_ips'] - used_floating_ips return min(num_floating_ips, allowed_floating_ips) -- cgit From 2774466197a0dda3763569fe7aa1a578baf5e059 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 13 Sep 2010 02:15:02 -0700 Subject: added missing yield in detach_volume --- nova/compute/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 954227b42..24538e4f1 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -189,7 +189,7 @@ class ComputeManager(manager.Manager): volume_id) instance_ref = self.db.instance_get(context, instance_id) volume_ref = self.db.volume_get(context, volume_id) - self.driver.detach_volume(instance_ref['str_id'], - volume_ref['mountpoint']) + yield self.driver.detach_volume(instance_ref['str_id'], + volume_ref['mountpoint']) self.db.volume_detached(context, volume_id) defer.returnValue(True) -- cgit From 5e02ee47c0e86986bb21f67a4d6556895de5d0ef Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 13 Sep 2010 11:53:53 -0400 Subject: Pull S3ImageService out of this mergeprop --- nova/api/ec2/images.py | 34 ++++++++++++++++++++++++++------- nova/image/service.py | 52 -------------------------------------------------- 2 files changed, 27 insertions(+), 59 deletions(-) diff --git a/nova/api/ec2/images.py b/nova/api/ec2/images.py index f0be7b899..b5ce2b2cc 100644 --- a/nova/api/ec2/images.py +++ b/nova/api/ec2/images.py @@ -29,17 +29,16 @@ import boto.s3.connection from nova import flags from nova import utils from nova.auth import manager -from nova.image import service FLAGS = flags.FLAGS def modify(context, image_id, operation): - service.S3ImageService(context)._conn().make_request( + conn(context).make_request( method='POST', bucket='_images', - query_args=service.qs({'image_id': image_id, 'operation': operation})) + query_args=qs({'image_id': image_id, 'operation': operation})) return True @@ -48,10 +47,10 @@ def register(context, image_location): """ rpc call to register a new image based from a manifest """ image_id = utils.generate_uid('ami') - service.S3ImageService(context)._conn().make_request( + conn(context).make_request( method='PUT', bucket='_images', - query_args=service.qs({'image_location': image_location, + query_args=qs({'image_location': image_location, 'image_id': image_id})) return image_id @@ -62,7 +61,12 @@ def list(context, filter_list=[]): optionally filtered by a list of image_id """ - result = service.S3ImageService(context).index().values() + # FIXME: send along the list of only_images to check for + response = conn(context).make_request( + method='GET', + bucket='_images') + + result = json.loads(response.read()) if not filter_list is None: return [i for i in result if i['imageId'] in filter_list] return result @@ -70,4 +74,20 @@ def list(context, filter_list=[]): def deregister(context, image_id): """ unregister an image """ - service.S3ImageService(context).delete(image_id) + conn(context).make_request( + method='DELETE', + bucket='_images', + query_args=qs({'image_id': image_id})) + + +def conn(context): + access = manager.AuthManager().get_access_key(context.user, + context.project) + secret = str(context.user.secret) + calling = boto.s3.connection.OrdinaryCallingFormat() + return boto.s3.connection.S3Connection(aws_access_key_id=access, + aws_secret_access_key=secret, + is_secure=False, + calling_format=calling, + port=FLAGS.s3_port, + host=FLAGS.s3_host) diff --git a/nova/image/service.py b/nova/image/service.py index f6719caec..1a7a258b7 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -38,8 +38,6 @@ class ImageService(object): def show(self, id): """ Returns a dict containing image data for the given opaque image id. - - Returns None if the id does not exist. """ @@ -90,53 +88,3 @@ class LocalImageService(ImageService): Delete the given image. Raises OSError if the image does not exist. """ os.unlink(self._path_to(image_id)) - - -# TODO(gundlach): before this can be loaded dynamically in ImageService.load(), -# we'll have to make __init__() not require a context. Right now it -# is only used by the AWS API, which hard-codes it, so that's OK. -class S3ImageService(ImageService): - """Service that stores images in an S3 provider.""" - - def __init__(self, context): - self._context = context - - def index(self): - response = self._conn().make_request( - method='GET', - bucket='_images') - items = json.loads(response.read()) - return dict((item['imageId'], item) for item in items) - - def show(self, id): - response = self._conn().make_request( - method='GET', - bucket='_images', - query_args=qs({'image_id': image_id})) - return json.loads(response.read()) - - def delete(self, image_id): - self._conn().make_request( - method='DELETE', - bucket='_images', - query_args=qs({'image_id': image_id})) - - def _conn(self): - """Return a boto S3Connection to the S3 store.""" - access = manager.AuthManager().get_access_key(self._context.user, - self._context.project) - secret = str(self._context.user.secret) - calling = boto.s3.connection.OrdinaryCallingFormat() - return boto.s3.connection.S3Connection(aws_access_key_id=access, - aws_secret_access_key=secret, - is_secure=False, - calling_format=calling, - port=FLAGS.s3_port, - host=FLAGS.s3_host) - - -def qs(params): - pairs = [] - for key in params.keys(): - pairs.append(key + '=' + urllib.quote(params[key])) - return '&'.join(pairs) -- cgit From 2b87ea1ab445a5a9fb089acb0220189f736d420a Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 13 Sep 2010 12:02:50 -0400 Subject: Finish pulling S3ImageService out of this mergeprop --- nova/api/ec2/images.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/api/ec2/images.py b/nova/api/ec2/images.py index b5ce2b2cc..2a88d66af 100644 --- a/nova/api/ec2/images.py +++ b/nova/api/ec2/images.py @@ -91,3 +91,10 @@ def conn(context): calling_format=calling, port=FLAGS.s3_port, host=FLAGS.s3_host) + + +def qs(params): + pairs = [] + for key in params.keys(): + pairs.append(key + '=' + urllib.quote(params[key])) + return '&'.join(pairs) -- cgit From 3d68f1f74cd7fe6ddb9eec003a9e31f8ad036b27 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 14 Sep 2010 16:26:19 -0400 Subject: Add ratelimiting package into Nova. After Austin it'll be pulled out into PyPI. --- nova/api/rackspace/ratelimiting/__init__.py | 103 ++++++++++++++++++++++++++++ nova/api/rackspace/ratelimiting/tests.py | 60 ++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 nova/api/rackspace/ratelimiting/__init__.py create mode 100644 nova/api/rackspace/ratelimiting/tests.py diff --git a/nova/api/rackspace/ratelimiting/__init__.py b/nova/api/rackspace/ratelimiting/__init__.py new file mode 100644 index 000000000..176e7d66e --- /dev/null +++ b/nova/api/rackspace/ratelimiting/__init__.py @@ -0,0 +1,103 @@ +"""Rate limiting of arbitrary actions.""" + +import time +import urllib +import webob.dec +import webob.exc + + +# Convenience constants for the limits dictionary passed to Limiter(). +PER_SECOND = 1 +PER_MINUTE = 60 +PER_HOUR = 60 * 60 +PER_DAY = 60 * 60 * 24 + +class Limiter(object): + + """Class providing rate limiting of arbitrary actions.""" + + def __init__(self, limits): + """Create a rate limiter. + + limits: a dict mapping from action name to a tuple. The tuple contains + the number of times the action may be performed, and the time period + (in seconds) during which the number must not be exceeded for this + action. Example: dict(reboot=(10, ratelimiting.PER_MINUTE)) would + allow 10 'reboot' actions per minute. + """ + self.limits = limits + self._levels = {} + + def perform(self, action_name, username='nobody'): + """Attempt to perform an action by the given username. + + action_name: the string name of the action to perform. This must + be a key in the limits dict passed to the ctor. + + username: an optional string name of the user performing the action. + Each user has her own set of rate limiting counters. Defaults to + 'nobody' (so that if you never specify a username when calling + perform(), a single set of counters will be used.) + + Return None if the action may proceed. If the action may not proceed + because it has been rate limited, return the float number of seconds + until the action would succeed. + """ + # Think of rate limiting as a bucket leaking water at 1cc/second. The + # bucket can hold as many ccs as there are seconds in the rate + # limiting period (e.g. 3600 for per-hour ratelimits), and if you can + # perform N actions in that time, each action fills the bucket by + # 1/Nth of its volume. You may only perform an action if the bucket + # would not overflow. + now = time.time() + key = '%s:%s' % (username, action_name) + last_time_performed, water_level = self._levels.get(key, (now, 0)) + # The bucket leaks 1cc/second. + water_level -= (now - last_time_performed) + if water_level < 0: + water_level = 0 + num_allowed_per_period, period_in_secs = self.limits[action_name] + # Fill the bucket by 1/Nth its capacity, and hope it doesn't overflow. + capacity = period_in_secs + new_level = water_level + (capacity * 1.0 / num_allowed_per_period) + if new_level > capacity: + # Delay this many seconds. + return new_level - capacity + self._levels[key] = (now, new_level) + return None + + +# If one instance of this WSGIApps is unable to handle your load, put a +# sharding app in front that shards by username to one of many backends. + +class WSGIApp(object): + + """Application that tracks rate limits in memory. Send requests to it of + this form: + + POST /limiter// + + and receive a 200 OK, or a 403 Forbidden with an X-Wait-Seconds header + containing the number of seconds to wait before the action would succeed. + """ + + def __init__(self, limiter): + """Create the WSGI application using the given Limiter instance.""" + self.limiter = limiter + + @webob.dec.wsgify + def __call__(req): + parts = req.path_info.split('/') + # format: /limiter// + if req.method != 'POST': + raise webob.exc.HTTPMethodNotAllowed() + if len(parts) != 4 or parts[1] != 'limiter': + raise webob.exc.HTTPNotFound() + username = parts[2] + action_name = urllib.unquote(parts[3]) + delay = self.limiter.perform(action_name, username) + if delay: + return webob.exc.HTTPForbidden( + headers={'X-Wait-Seconds': delay}) + else: + return '' # 200 OK diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py new file mode 100644 index 000000000..1983cdea8 --- /dev/null +++ b/nova/api/rackspace/ratelimiting/tests.py @@ -0,0 +1,60 @@ +import ratelimiting +import time +import unittest + +class Test(unittest.TestCase): + + def setUp(self): + self.limits = { + 'a': (5, ratelimiting.PER_SECOND), + 'b': (5, ratelimiting.PER_MINUTE), + 'c': (5, ratelimiting.PER_HOUR), + 'd': (1, ratelimiting.PER_SECOND), + 'e': (100, ratelimiting.PER_SECOND)} + self.rl = ratelimiting.Limiter(self.limits) + + def exhaust(self, action, times_until_exhausted, **kwargs): + for i in range(times_until_exhausted): + when = self.rl.perform(action, **kwargs) + self.assertEqual(when, None) + num, period = self.limits[action] + delay = period * 1.0 / num + # Verify that we are now thoroughly delayed + for i in range(10): + when = self.rl.perform(action, **kwargs) + self.assertAlmostEqual(when, delay, 2) + + def test_second(self): + self.exhaust('a', 5) + time.sleep(0.2) + self.exhaust('a', 1) + time.sleep(1) + self.exhaust('a', 5) + + def test_minute(self): + self.exhaust('b', 5) + + def test_one_per_period(self): + def allow_once_and_deny_once(): + when = self.rl.perform('d') + self.assertEqual(when, None) + when = self.rl.perform('d') + self.assertAlmostEqual(when, 1, 2) + return when + time.sleep(allow_once_and_deny_once()) + time.sleep(allow_once_and_deny_once()) + allow_once_and_deny_once() + + def test_we_can_go_indefinitely_if_we_spread_out_requests(self): + for i in range(200): + when = self.rl.perform('e') + self.assertEqual(when, None) + time.sleep(0.01) + + def test_users_get_separate_buckets(self): + self.exhaust('c', 5, username='alice') + self.exhaust('c', 5, username='bob') + self.exhaust('c', 5, username='chuck') + self.exhaust('c', 0, username='chuck') + self.exhaust('c', 0, username='bob') + self.exhaust('c', 0, username='alice') -- cgit From 8138a35d3672e08640762b7533c1c527568d0b4f Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 14 Sep 2010 18:59:02 -0400 Subject: RateLimitingMiddleware --- nova/api/rackspace/__init__.py | 52 ++++++++++++++++++++++++++++++++- nova/tests/api/rackspace/__init__.py | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py index b4d666d63..e35109b43 100644 --- a/nova/api/rackspace/__init__.py +++ b/nova/api/rackspace/__init__.py @@ -31,6 +31,7 @@ from nova import flags from nova import wsgi from nova.api.rackspace import flavors from nova.api.rackspace import images +from nova.api.rackspace import ratelimiting from nova.api.rackspace import servers from nova.api.rackspace import sharedipgroups from nova.auth import manager @@ -40,7 +41,7 @@ class API(wsgi.Middleware): """WSGI entry point for all Rackspace API requests.""" def __init__(self): - app = AuthMiddleware(APIRouter()) + app = AuthMiddleware(RateLimitingMiddleware(APIRouter())) super(API, self).__init__(app) @@ -65,6 +66,55 @@ class AuthMiddleware(wsgi.Middleware): return self.application +class RateLimitingMiddleware(wsgi.Middleware): + """Rate limit incoming requests according to the OpenStack rate limits.""" + + def __init__(self, application): + super(RateLimitingMiddleware, self).__init__(application) + #TODO(gundlach): These limits were based on limitations of Cloud + #Servers. We should revisit them in Nova. + self.limiter = ratelimiting.Limiter(limits={ + 'DELETE': (100, ratelimiting.PER_MINUTE), + 'PUT': (10, ratelimiting.PER_MINUTE), + 'POST': (10, ratelimiting.PER_MINUTE), + 'POST servers': (50, ratelimiting.PER_DAY), + 'GET changes-since': (3, ratelimiting.PER_MINUTE), + }) + + @webob.dec.wsgify + def __call__(self, req): + """Rate limit the request. + + If the request should be rate limited, return a 413 status with a + Retry-After header giving the time when the request would succeed. + """ + username = req.headers['X-Auth-User'] + action_name = self.get_action_name(req) + if not action_name: # not rate limited + return self.application + delay = self.limiter.perform(action_name, username=username) + if action_name == 'POST servers': + # "POST servers" is a POST, so it counts against "POST" too. + delay2 = self.limiter.perform('POST', username=username) + delay = max(delay or 0, delay2 or 0) + if delay: + # TODO(gundlach): Get the retry-after format correct. + raise webob.exc.HTTPRequestEntityTooLarge(headers={ + 'Retry-After': time.time() + delay}) + else: + return self.application + + def get_action_name(self, req): + """Return the action name for this request.""" + if req.method == 'GET' and 'changes-since' in req.GET: + return 'GET changes-since' + if req.method == 'POST' and req.path_info.starts_with('/servers'): + return 'POST servers' + if req.method in ['PUT', 'POST', 'DELETE']: + return req.method + return None + + class APIRouter(wsgi.Router): """ Routes requests on the Rackspace API to the appropriate controller diff --git a/nova/tests/api/rackspace/__init__.py b/nova/tests/api/rackspace/__init__.py index e69de29bb..f7537a4e7 100644 --- a/nova/tests/api/rackspace/__init__.py +++ b/nova/tests/api/rackspace/__init__.py @@ -0,0 +1,56 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from nova.api.rackspace.ratelimiting import RateLimitingMiddleware +from nova.tests.api.test_helper import * +from webob import Request + +class RateLimitingMiddlewareTest(unittest.TestCase): + def setUp(self): + self.middleware = RateLimitingMiddleware(APIStub()) + self.stubs = stubout.StubOutForTesting() + + def tearDown(self): + self.stubs.UnsetAll() + + def test_get_action_name(self): + middleware = RateLimitingMiddleware(APIStub()) + def verify(method, url, action_name): + req = Request(url) + req.method = method + action = middleware.get_action_name(req) + self.assertEqual(action, action_name) + verify('PUT', '/servers/4', 'PUT') + verify('DELETE', '/servers/4', 'DELETE') + verify('POST', '/images/4', 'POST') + verify('POST', '/servers/4', 'POST servers') + verify('GET', '/foo?a=4&changes-since=never&b=5', 'GET changes-since') + verify('GET', '/foo?a=4&monkeys-since=never&b=5', None) + verify('GET', '/servers/4', None) + verify('HEAD', '/servers/4', None) + + def TODO_test_call(self): + pass + #mw = make_middleware() + #req = build_request('DELETE', '/servers/4') + #for i in range(5): + # resp = req.get_response(mw) + # assert resp is OK + #resp = req.get_response(mw) + #assert resp is rate limited -- cgit From 63ad073efd0b20f59f02bc37182c0180cac3f405 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 15 Sep 2010 09:25:53 -0400 Subject: RateLimitingMiddleware tests --- nova/api/rackspace/__init__.py | 24 ++++++++++----- nova/api/rackspace/ratelimiting/tests.py | 3 ++ nova/tests/api/rackspace/__init__.py | 51 +++++++++++++++++++++----------- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py index e35109b43..66d80a5b7 100644 --- a/nova/api/rackspace/__init__.py +++ b/nova/api/rackspace/__init__.py @@ -92,23 +92,31 @@ class RateLimitingMiddleware(wsgi.Middleware): action_name = self.get_action_name(req) if not action_name: # not rate limited return self.application - delay = self.limiter.perform(action_name, username=username) - if action_name == 'POST servers': - # "POST servers" is a POST, so it counts against "POST" too. - delay2 = self.limiter.perform('POST', username=username) - delay = max(delay or 0, delay2 or 0) + delay = self.get_delay(action_name, username) if delay: # TODO(gundlach): Get the retry-after format correct. raise webob.exc.HTTPRequestEntityTooLarge(headers={ 'Retry-After': time.time() + delay}) - else: - return self.application + return self.application + + def get_delay(self, action_name, username): + """Return the delay for the given action and username, or None if + the action would not be rate limited. + """ + if action_name == 'POST servers': + # "POST servers" is a POST, so it counts against "POST" too. + # Attempt the "POST" first, lest we are rate limited by "POST" but + # use up a precious "POST servers" call. + delay = self.limiter.perform("POST", username=username) + if delay: + return delay + return self.limiter.perform(action_name, username=username) def get_action_name(self, req): """Return the action name for this request.""" if req.method == 'GET' and 'changes-since' in req.GET: return 'GET changes-since' - if req.method == 'POST' and req.path_info.starts_with('/servers'): + if req.method == 'POST' and req.path_info.startswith('/servers'): return 'POST servers' if req.method in ['PUT', 'POST', 'DELETE']: return req.method diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py index 1983cdea8..545e1d1b6 100644 --- a/nova/api/rackspace/ratelimiting/tests.py +++ b/nova/api/rackspace/ratelimiting/tests.py @@ -58,3 +58,6 @@ class Test(unittest.TestCase): self.exhaust('c', 0, username='chuck') self.exhaust('c', 0, username='bob') self.exhaust('c', 0, username='alice') + +if __name__ == '__main__': + unittest.main() diff --git a/nova/tests/api/rackspace/__init__.py b/nova/tests/api/rackspace/__init__.py index f7537a4e7..2fab1a4da 100644 --- a/nova/tests/api/rackspace/__init__.py +++ b/nova/tests/api/rackspace/__init__.py @@ -17,22 +17,15 @@ import unittest -from nova.api.rackspace.ratelimiting import RateLimitingMiddleware +from nova.api.rackspace import RateLimitingMiddleware from nova.tests.api.test_helper import * from webob import Request class RateLimitingMiddlewareTest(unittest.TestCase): - def setUp(self): - self.middleware = RateLimitingMiddleware(APIStub()) - self.stubs = stubout.StubOutForTesting() - - def tearDown(self): - self.stubs.UnsetAll() - def test_get_action_name(self): middleware = RateLimitingMiddleware(APIStub()) def verify(method, url, action_name): - req = Request(url) + req = Request.blank(url) req.method = method action = middleware.get_action_name(req) self.assertEqual(action, action_name) @@ -45,12 +38,34 @@ class RateLimitingMiddlewareTest(unittest.TestCase): verify('GET', '/servers/4', None) verify('HEAD', '/servers/4', None) - def TODO_test_call(self): - pass - #mw = make_middleware() - #req = build_request('DELETE', '/servers/4') - #for i in range(5): - # resp = req.get_response(mw) - # assert resp is OK - #resp = req.get_response(mw) - #assert resp is rate limited + def exhaust(self, middleware, method, url, username, times): + req = Request.blank(url, dict(REQUEST_METHOD=method), + headers={'X-Auth-User': username}) + for i in range(times): + resp = req.get_response(middleware) + self.assertEqual(resp.status_int, 200) + resp = req.get_response(middleware) + self.assertEqual(resp.status_int, 413) + self.assertTrue('Retry-After' in resp.headers) + + def test_single_action(self): + middleware = RateLimitingMiddleware(APIStub()) + self.exhaust(middleware, 'DELETE', '/servers/4', 'usr1', 100) + self.exhaust(middleware, 'DELETE', '/servers/4', 'usr2', 100) + + def test_POST_servers_action_implies_POST_action(self): + middleware = RateLimitingMiddleware(APIStub()) + self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) + self.exhaust(middleware, 'POST', '/images/4', 'usr2', 10) + self.assertTrue(set(middleware.limiter._levels) == + set(['usr1:POST', 'usr1:POST servers', 'usr2:POST'])) + + def test_POST_servers_action_correctly_ratelimited(self): + middleware = RateLimitingMiddleware(APIStub()) + # Use up all of our "POST" allowance for the minute, 5 times + for i in range(5): + self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) + # Reset the 'POST' action counter. + del middleware.limiter._levels['usr1:POST'] + # All 50 daily "POST servers" actions should be all used up + self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 0) -- cgit From fd4d5787d5b6f6e550d33c13eb76f4562a87a118 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 15 Sep 2010 11:23:08 -0400 Subject: Test the WSGIApp --- nova/api/rackspace/ratelimiting/__init__.py | 2 +- nova/api/rackspace/ratelimiting/tests.py | 69 ++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/nova/api/rackspace/ratelimiting/__init__.py b/nova/api/rackspace/ratelimiting/__init__.py index 176e7d66e..64d5fff2c 100644 --- a/nova/api/rackspace/ratelimiting/__init__.py +++ b/nova/api/rackspace/ratelimiting/__init__.py @@ -86,7 +86,7 @@ class WSGIApp(object): self.limiter = limiter @webob.dec.wsgify - def __call__(req): + def __call__(self, req): parts = req.path_info.split('/') # format: /limiter// if req.method != 'POST': diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py index 545e1d1b6..f924e7805 100644 --- a/nova/api/rackspace/ratelimiting/tests.py +++ b/nova/api/rackspace/ratelimiting/tests.py @@ -1,8 +1,10 @@ -import ratelimiting import time import unittest +import webob -class Test(unittest.TestCase): +import nova.api.rackspace.ratelimiting as ratelimiting + +class LimiterTest(unittest.TestCase): def setUp(self): self.limits = { @@ -59,5 +61,68 @@ class Test(unittest.TestCase): self.exhaust('c', 0, username='bob') self.exhaust('c', 0, username='alice') + +class WSGIAppTest(unittest.TestCase): + + def setUp(self): + test = self + class FakeLimiter(object): + def __init__(self): + self._action = self._username = self._delay = None + def mock(self, action, username, delay): + self._action = action + self._username = username + self._delay = delay + def perform(self, action, username): + test.assertEqual(action, self._action) + test.assertEqual(username, self._username) + return self._delay + self.limiter = FakeLimiter() + self.app = ratelimiting.WSGIApp(self.limiter) + + def test_invalid_methods(self): + requests = [] + for method in ['GET', 'PUT', 'DELETE']: + req = webob.Request.blank('/limits/michael/breakdance', + dict(REQUEST_METHOD=method)) + requests.append(req) + for req in requests: + self.assertEqual(req.get_response(self.app).status_int, 405) + + def test_invalid_urls(self): + requests = [] + for prefix in ['limit', '', 'limiter2', 'limiter/limits', 'limiter/1']: + req = webob.Request.blank('/%s/michael/breakdance' % prefix, + dict(REQUEST_METHOD='POST')) + requests.append(req) + for req in requests: + self.assertEqual(req.get_response(self.app).status_int, 404) + + def verify(self, url, username, action, delay=None): + """Make sure that POSTing to the given url causes the given username + to perform the given action. Make the internal rate limiter return + delay and make sure that the WSGI app returns the correct response. + """ + req = webob.Request.blank(url, dict(REQUEST_METHOD='POST')) + self.limiter.mock(action, username, delay) + resp = req.get_response(self.app) + if not delay: + self.assertEqual(resp.status_int, 200) + else: + self.assertEqual(resp.status_int, 403) + self.assertEqual(resp.headers['X-Wait-Seconds'], delay) + + def test_good_urls(self): + self.verify('/limiter/michael/hoot', 'michael', 'hoot') + + def test_escaping(self): + self.verify('/limiter/michael/jump%20up', 'michael', 'jump up') + + def test_response_to_delays(self): + self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1) + self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1.56) + self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1000) + + if __name__ == '__main__': unittest.main() -- cgit From f200587ce068482ab94e777154de3ac777269fa0 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 15 Sep 2010 13:54:38 -0400 Subject: Add support for middleware proxying to a ratelimiting.WSGIApp, for deployments that use more than one API Server and thus can't store ratelimiting counters in memory. --- nova/api/rackspace/__init__.py | 29 ++++-- nova/api/rackspace/ratelimiting/__init__.py | 21 ++++- nova/api/rackspace/ratelimiting/tests.py | 140 +++++++++++++++++++++++++--- nova/tests/api/rackspace/__init__.py | 8 ++ 4 files changed, 173 insertions(+), 25 deletions(-) diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py index 66d80a5b7..ac5365310 100644 --- a/nova/api/rackspace/__init__.py +++ b/nova/api/rackspace/__init__.py @@ -69,17 +69,26 @@ class AuthMiddleware(wsgi.Middleware): class RateLimitingMiddleware(wsgi.Middleware): """Rate limit incoming requests according to the OpenStack rate limits.""" - def __init__(self, application): + def __init__(self, application, service_host=None): + """Create a rate limiting middleware that wraps the given application. + + By default, rate counters are stored in memory. If service_host is + specified, the middleware instead relies on the ratelimiting.WSGIApp + at the given host+port to keep rate counters. + """ super(RateLimitingMiddleware, self).__init__(application) - #TODO(gundlach): These limits were based on limitations of Cloud - #Servers. We should revisit them in Nova. - self.limiter = ratelimiting.Limiter(limits={ - 'DELETE': (100, ratelimiting.PER_MINUTE), - 'PUT': (10, ratelimiting.PER_MINUTE), - 'POST': (10, ratelimiting.PER_MINUTE), - 'POST servers': (50, ratelimiting.PER_DAY), - 'GET changes-since': (3, ratelimiting.PER_MINUTE), - }) + if not service_host: + #TODO(gundlach): These limits were based on limitations of Cloud + #Servers. We should revisit them in Nova. + self.limiter = ratelimiting.Limiter(limits={ + 'DELETE': (100, ratelimiting.PER_MINUTE), + 'PUT': (10, ratelimiting.PER_MINUTE), + 'POST': (10, ratelimiting.PER_MINUTE), + 'POST servers': (50, ratelimiting.PER_DAY), + 'GET changes-since': (3, ratelimiting.PER_MINUTE), + }) + else: + self.limiter = ratelimiting.WSGIAppProxy(service_host) @webob.dec.wsgify def __call__(self, req): diff --git a/nova/api/rackspace/ratelimiting/__init__.py b/nova/api/rackspace/ratelimiting/__init__.py index 64d5fff2c..f843bac0f 100644 --- a/nova/api/rackspace/ratelimiting/__init__.py +++ b/nova/api/rackspace/ratelimiting/__init__.py @@ -1,5 +1,6 @@ """Rate limiting of arbitrary actions.""" +import httplib import time import urllib import webob.dec @@ -98,6 +99,24 @@ class WSGIApp(object): delay = self.limiter.perform(action_name, username) if delay: return webob.exc.HTTPForbidden( - headers={'X-Wait-Seconds': delay}) + headers={'X-Wait-Seconds': "%.2f" % delay}) else: return '' # 200 OK + + +class WSGIAppProxy(object): + + """Limiter lookalike that proxies to a ratelimiting.WSGIApp.""" + + def __init__(self, service_host): + """Creates a proxy pointing to a ratelimiting.WSGIApp at the given + host.""" + self.service_host = service_host + + def perform(self, action, username='nobody'): + conn = httplib.HTTPConnection(self.service_host) + conn.request('POST', '/limiter/%s/%s' % (username, action)) + resp = conn.getresponse() + if resp.status == 200: + return None # no delay + return float(resp.getheader('X-Wait-Seconds')) diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py index f924e7805..13a47989b 100644 --- a/nova/api/rackspace/ratelimiting/tests.py +++ b/nova/api/rackspace/ratelimiting/tests.py @@ -1,3 +1,5 @@ +import httplib +import StringIO import time import unittest import webob @@ -62,22 +64,25 @@ class LimiterTest(unittest.TestCase): self.exhaust('c', 0, username='alice') +class FakeLimiter(object): + """Fake Limiter class that you can tell how to behave.""" + def __init__(self, test): + self._action = self._username = self._delay = None + self.test = test + def mock(self, action, username, delay): + self._action = action + self._username = username + self._delay = delay + def perform(self, action, username): + self.test.assertEqual(action, self._action) + self.test.assertEqual(username, self._username) + return self._delay + + class WSGIAppTest(unittest.TestCase): def setUp(self): - test = self - class FakeLimiter(object): - def __init__(self): - self._action = self._username = self._delay = None - def mock(self, action, username, delay): - self._action = action - self._username = username - self._delay = delay - def perform(self, action, username): - test.assertEqual(action, self._action) - test.assertEqual(username, self._username) - return self._delay - self.limiter = FakeLimiter() + self.limiter = FakeLimiter(self) self.app = ratelimiting.WSGIApp(self.limiter) def test_invalid_methods(self): @@ -110,7 +115,7 @@ class WSGIAppTest(unittest.TestCase): self.assertEqual(resp.status_int, 200) else: self.assertEqual(resp.status_int, 403) - self.assertEqual(resp.headers['X-Wait-Seconds'], delay) + self.assertEqual(resp.headers['X-Wait-Seconds'], "%.2f" % delay) def test_good_urls(self): self.verify('/limiter/michael/hoot', 'michael', 'hoot') @@ -124,5 +129,112 @@ class WSGIAppTest(unittest.TestCase): self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1000) +class FakeHttplibSocket(object): + """a fake socket implementation for httplib.HTTPResponse, trivial""" + + def __init__(self, response_string): + self._buffer = StringIO.StringIO(response_string) + + def makefile(self, _mode, _other): + """Returns the socket's internal buffer""" + return self._buffer + + +class FakeHttplibConnection(object): + """A fake httplib.HTTPConnection + + Requests made via this connection actually get translated and routed into + our WSGI app, we then wait for the response and turn it back into + an httplib.HTTPResponse. + """ + def __init__(self, app, host, is_secure=False): + self.app = app + self.host = host + + def request(self, method, path, data='', headers={}): + req = webob.Request.blank(path) + req.method = method + req.body = data + req.headers = headers + req.host = self.host + # Call the WSGI app, get the HTTP response + resp = str(req.get_response(self.app)) + # For some reason, the response doesn't have "HTTP/1.0 " prepended; I + # guess that's a function the web server usually provides. + resp = "HTTP/1.0 %s" % resp + sock = FakeHttplibSocket(resp) + self.http_response = httplib.HTTPResponse(sock) + self.http_response.begin() + + def getresponse(self): + return self.http_response + + +def wire_HTTPConnection_to_WSGI(host, app): + """Monkeypatches HTTPConnection so that if you try to connect to host, you + are instead routed straight to the given WSGI app. + + After calling this method, when any code calls + + httplib.HTTPConnection(host) + + the connection object will be a fake. Its requests will be sent directly + to the given WSGI app rather than through a socket. + + Code connecting to hosts other than host will not be affected. + + This method may be called multiple times to map different hosts to + different apps. + """ + class HTTPConnectionDecorator(object): + """Wraps the real HTTPConnection class so that when you instantiate + the class you might instead get a fake instance.""" + def __init__(self, wrapped): + self.wrapped = wrapped + def __call__(self, connection_host, *args, **kwargs): + if connection_host == host: + return FakeHttplibConnection(app, host) + else: + return self.wrapped(connection_host, *args, **kwargs) + httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) + + +class WSGIAppProxyTest(unittest.TestCase): + + def setUp(self): + """Our WSGIAppProxy is going to call across an HTTPConnection to a + WSGIApp running a limiter. The proxy will send input, and the proxy + should receive that same input, pass it to the limiter who gives a + result, and send the expected result back. + + The HTTPConnection isn't real -- it's monkeypatched to point straight + at the WSGIApp. And the limiter isn't real -- it's a fake that + behaves the way we tell it to. + """ + self.limiter = FakeLimiter(self) + app = ratelimiting.WSGIApp(self.limiter) + wire_HTTPConnection_to_WSGI('100.100.100.100:80', app) + self.proxy = ratelimiting.WSGIAppProxy('100.100.100.100:80') + + def test_200(self): + self.limiter.mock('conquer', 'caesar', None) + when = self.proxy.perform('conquer', 'caesar') + self.assertEqual(when, None) + + def test_403(self): + self.limiter.mock('grumble', 'proletariat', 1.5) + when = self.proxy.perform('grumble', 'proletariat') + self.assertEqual(when, 1.5) + + def test_failure(self): + self.limiter.mock('murder', 'brutus', None) + try: + when = self.proxy.perform('stab', 'brutus') + except AssertionError: + pass + else: + self.fail("I didn't perform the action I expected") + + if __name__ == '__main__': unittest.main() diff --git a/nova/tests/api/rackspace/__init__.py b/nova/tests/api/rackspace/__init__.py index 2fab1a4da..622cb4335 100644 --- a/nova/tests/api/rackspace/__init__.py +++ b/nova/tests/api/rackspace/__init__.py @@ -21,7 +21,9 @@ from nova.api.rackspace import RateLimitingMiddleware from nova.tests.api.test_helper import * from webob import Request + class RateLimitingMiddlewareTest(unittest.TestCase): + def test_get_action_name(self): middleware = RateLimitingMiddleware(APIStub()) def verify(method, url, action_name): @@ -69,3 +71,9 @@ class RateLimitingMiddlewareTest(unittest.TestCase): del middleware.limiter._levels['usr1:POST'] # All 50 daily "POST servers" actions should be all used up self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 0) + + def test_proxy_ctor_works(self): + middleware = RateLimitingMiddleware(APIStub()) + self.assertEqual(middleware.limiter.__class__.__name__, "Limiter") + middleware = RateLimitingMiddleware(APIStub(), service_host='foobar') + self.assertEqual(middleware.limiter.__class__.__name__, "WSGIAppProxy") -- cgit From 7437df558f3277e21a4c34a5b517a1cae5dd5a74 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 15 Sep 2010 17:17:20 -0400 Subject: Support querying version list --- nova/api/__init__.py | 13 +++++++++++++ nova/tests/api/__init__.py | 5 +++-- run_tests.py | 4 +++- tools/pip-requires | 4 ++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index b9b9e3988..9f116dada 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -21,6 +21,7 @@ Root WSGI middleware for all API controllers. """ import routes +import webob.dec from nova import wsgi from nova.api import ec2 @@ -32,6 +33,18 @@ class API(wsgi.Router): def __init__(self): mapper = routes.Mapper() + mapper.connect("/", controller=self.versions) mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API()) mapper.connect("/ec2/{path_info:.*}", controller=ec2.API()) super(API, self).__init__(mapper) + + @webob.dec.wsgify + def versions(self, req): + """Respond to a request for all OpenStack API versions.""" + response = { + "versions": [ + dict(status="CURRENT", id="v1.0")]} + metadata = { + "application/xml": { + "attributes": dict(version=["status", "id"])}} + return wsgi.Serializer(req.environ, metadata).to_content_type(response) diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index 59c4adc3d..4682c094e 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -52,8 +52,9 @@ class Test(unittest.TestCase): result = webob.Request.blank('/test/cloud').get_response(api.API()) self.assertNotEqual(result.body, "/cloud") - def test_query_api_version(self): - pass + def test_query_api_versions(self): + result = webob.Request.blank('/').get_response(api.API()) + self.assertTrue('CURRENT' in result.body) if __name__ == '__main__': unittest.main() diff --git a/run_tests.py b/run_tests.py index 77aa9088a..cf37b820e 100644 --- a/run_tests.py +++ b/run_tests.py @@ -50,8 +50,10 @@ from nova import flags from nova import twistd from nova.tests.access_unittest import * -from nova.tests.auth_unittest import * from nova.tests.api_unittest import * +from nova.tests.api import * +from nova.tests.api.rackspace import * +from nova.tests.auth_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * diff --git a/tools/pip-requires b/tools/pip-requires index 13e8e5f45..9b8027451 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -6,14 +6,14 @@ amqplib==0.6.1 anyjson==0.2.4 boto==2.0b1 carrot==0.10.5 -eventlet==0.9.10 +eventlet==0.9.12 lockfile==0.8 python-daemon==1.5.5 python-gflags==1.3 redis==2.0.0 routes==1.12.3 tornado==1.0 -webob==0.9.8 +WebOb==0.9.8 wsgiref==0.1.2 zope.interface==3.6.1 mox==0.5.0 -- cgit From 8e304fe0bf69fe5f6bad2fa3d5a71a93cb0612e8 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 16 Sep 2010 12:39:35 -0400 Subject: Fix things not quite merged perfectly -- all tests now pass --- nova/api/ec2/cloud.py | 2 +- nova/tests/network_unittest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 7a9b5f5cf..c04e722cc 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -556,7 +556,7 @@ class CloudController(object): # NOTE(vish): Right now we don't really care if the ip is # disassociated. We may need to worry about # checking this later. Perhaps in the scheduler? - network_topic = yield self._get_network_topic(context) + network_topic = self._get_network_topic(context) rpc.cast(network_topic, {"method": "disassociate_floating_ip", "args": {"context": None, diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index dc5277f02..da65b50a2 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -28,7 +28,7 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager -from nova.endpoint import api +from nova.api.ec2 import context FLAGS = flags.FLAGS @@ -49,7 +49,7 @@ class NetworkTestCase(test.TrialTestCase): self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] self.network = utils.import_object(FLAGS.network_manager) - self.context = api.APIRequestContext(None, project=None, user=self.user) + self.context = context.APIRequestContext(project=None, user=self.user) for i in range(5): name = 'project%s' % i self.projects.append(self.manager.create_project(name, -- cgit From 11b934f75ac4359b75f246fd9babfc3363a9a396 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 16 Sep 2010 14:41:51 -0500 Subject: Replaced the existing Rackspace Auth Mechanism with one that mirrors the implementation in the design document. --- nova/api/rackspace/__init__.py | 52 ++++++++++++++++----- nova/api/rackspace/auth.py | 37 +++++++++++++++ nova/tests/api/rackspace/auth.py | 81 +++++++++++++++++++++++++++++++++ nova/tests/api/rackspace/test_helper.py | 52 +++++++++++++++++++++ 4 files changed, 211 insertions(+), 11 deletions(-) create mode 100644 nova/api/rackspace/auth.py create mode 100644 nova/tests/api/rackspace/auth.py create mode 100644 nova/tests/api/rackspace/test_helper.py diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py index b4d666d63..dbba97107 100644 --- a/nova/api/rackspace/__init__.py +++ b/nova/api/rackspace/__init__.py @@ -26,8 +26,10 @@ import time import routes import webob.dec import webob.exc +import webob from nova import flags +from nova import utils from nova import wsgi from nova.api.rackspace import flavors from nova.api.rackspace import images @@ -36,6 +38,10 @@ from nova.api.rackspace import sharedipgroups from nova.auth import manager +FLAGS = flags.FLAGS +flags.DEFINE_string('nova_api_auth', 'nova.api.rackspace.auth.FakeAuth', + 'The auth mechanism to use for the Rackspace API implemenation') + class API(wsgi.Middleware): """WSGI entry point for all Rackspace API requests.""" @@ -47,23 +53,47 @@ class API(wsgi.Middleware): class AuthMiddleware(wsgi.Middleware): """Authorize the rackspace API request or return an HTTP Forbidden.""" - #TODO(gundlach): isn't this the old Nova API's auth? Should it be replaced - #with correct RS API auth? + def __init__(self, application): + self.auth_driver = utils.import_class(FLAGS.nova_api_auth)() + super(AuthMiddleware, self).__init__(application) @webob.dec.wsgify def __call__(self, req): - context = {} - if "HTTP_X_AUTH_TOKEN" in req.environ: - context['user'] = manager.AuthManager().get_user_from_access_key( - req.environ['HTTP_X_AUTH_TOKEN']) - if context['user']: - context['project'] = manager.AuthManager().get_project( - context['user'].name) - if "user" not in context: - return webob.exc.HTTPForbidden() + if not req.headers.has_key("X-Auth-Token"): + return self.authenticate(req) + + user = self.auth_driver.authorize_token(req.headers["X-Auth-Token"]) + + if not user: + return webob.exc.HTTPUnauthorized() + context = {'user':user} req.environ['nova.context'] = context return self.application + def authenticate(self, req): + # Unless the request is explicitly made against // don't + # honor it + path_info = req.environ['wsgiorg.routing_args'][1]['path_info'] + if path_info: + return webob.exc.HTTPUnauthorized() + + if req.headers.has_key("X-Auth-User") and \ + req.headers.has_key("X-Auth-Key"): + username, key = req.headers['X-Auth-User'], req.headers['X-Auth-Key'] + token, user = self.auth_driver.authorize_user(username, key) + if user and token: + res = webob.Response() + res.headers['X-Auth-Token'] = token + res.headers['X-Server-Management-Url'] = \ + user['server_management_url'] + res.headers['X-Storage-Url'] = user['storage_url'] + res.headers['X-CDN-Management-Url'] = user['cdn_management_url'] + res.content_type = 'text/plain' + res.status = '204' + return res + else: + return webob.exc.HTTPUnauthorized() + return webob.exc.HTTPUnauthorized() class APIRouter(wsgi.Router): """ diff --git a/nova/api/rackspace/auth.py b/nova/api/rackspace/auth.py new file mode 100644 index 000000000..d2b5193c3 --- /dev/null +++ b/nova/api/rackspace/auth.py @@ -0,0 +1,37 @@ +import json +from hashlib import sha1 +from nova import datastore + +class FakeAuth(object): + def __init__(self, store=datastore.Redis.instance): + self._store = store() + self.auth_hash = 'rs_fake_auth' + self._store.hsetnx(self.auth_hash, 'rs_last_id', 0) + + def authorize_token(self, token): + user = self._store.hget(self.auth_hash, token) + if user: + return json.loads(user) + return None + + def authorize_user(self, user, key): + token = sha1("%s_%s" % (user, key)).hexdigest() + user = self._store.hget(self.auth_hash, token) + if not user: + return None, None + else: + return token, json.loads(user) + + def add_user(self, user, key): + last_id = self._store.hget(self.auth_hash, 'rs_last_id') + token = sha1("%s_%s" % (user, key)).hexdigest() + user = { + 'id':last_id, + 'cdn_management_url':'cdn_management_url', + 'storage_url':'storage_url', + 'server_management_url':'server_management_url' + } + new_user = self._store.hsetnx(self.auth_hash, token, json.dumps(user)) + if new_user: + self._store.hincrby(self.auth_hash, 'rs_last_id') + diff --git a/nova/tests/api/rackspace/auth.py b/nova/tests/api/rackspace/auth.py new file mode 100644 index 000000000..65264fae9 --- /dev/null +++ b/nova/tests/api/rackspace/auth.py @@ -0,0 +1,81 @@ +import webob +import webob.dec +import unittest +import stubout +import nova.api +import nova.api.rackspace.auth +from nova.tests.api.rackspace import test_helper + +class Test(unittest.TestCase): + def setUp(self): + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.rackspace.auth.FakeAuth, '__init__', + test_helper.fake_auth_init) + ds = test_helper.FakeRedis() + ds.hset(test_helper.auth_hash, 'rs_last_id', 0) + + def tearDown(self): + self.stubs.UnsetAll() + test_helper.fake_data_store = {} + + def test_authorize_user(self): + auth = nova.api.rackspace.auth.FakeAuth() + auth.add_user('herp', 'derp') + + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'herp' + req.headers['X-Auth-Key'] = 'derp' + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '204 No Content') + self.assertEqual(len(result.headers['X-Auth-Token']), 40) + self.assertEqual(result.headers['X-Server-Management-Url'], + "server_management_url") + self.assertEqual(result.headers['X-CDN-Management-Url'], + "cdn_management_url") + self.assertEqual(result.headers['X-Storage-Url'], "storage_url") + + def test_authorize_token(self): + auth = nova.api.rackspace.auth.FakeAuth() + auth.add_user('herp', 'derp') + + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'herp' + req.headers['X-Auth-Key'] = 'derp' + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '204 No Content') + self.assertEqual(len(result.headers['X-Auth-Token']), 40) + self.assertEqual(result.headers['X-Server-Management-Url'], + "server_management_url") + self.assertEqual(result.headers['X-CDN-Management-Url'], + "cdn_management_url") + self.assertEqual(result.headers['X-Storage-Url'], "storage_url") + + token = result.headers['X-Auth-Token'] + self.stubs.Set(nova.api.rackspace, 'APIRouter', + test_helper.FakeRouter) + req = webob.Request.blank('/v1.0/fake') + req.headers['X-Auth-Token'] = token + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '200 OK') + self.assertEqual(result.headers['X-Test-Success'], 'True') + + def test_bad_user(self): + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'herp' + req.headers['X-Auth-Key'] = 'derp' + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_no_user(self): + req = webob.Request.blank('/v1.0/') + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_bad_token(self): + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-Token'] = 'baconbaconbacon' + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '401 Unauthorized') + +if __name__ == '__main__': + unittest.main() diff --git a/nova/tests/api/rackspace/test_helper.py b/nova/tests/api/rackspace/test_helper.py new file mode 100644 index 000000000..578b1e841 --- /dev/null +++ b/nova/tests/api/rackspace/test_helper.py @@ -0,0 +1,52 @@ +import webob +import webob.dec +from nova.wsgi import Router + +fake_data_store = {} +auth_hash = 'dummy_hash' + +class FakeRedis(object): + def __init__(self): + global fake_data_store + self.store = fake_data_store + + def hsetnx(self, hash_name, key, value): + if not self.store.has_key(hash_name): + self.store[hash_name] = {} + + if self.store[hash_name].has_key(key): + return 0 + self.store[hash_name][key] = value + return 1 + + def hset(self, hash_name, key, value): + if not self.store.has_key(hash_name): + self.store[hash_name] = {} + + self.store[hash_name][key] = value + return 1 + + def hget(self, hash_name, key): + if not self.store[hash_name].has_key(key): + return None + return self.store[hash_name][key] + + def hincrby(self, hash_name, key, amount=1): + self.store[hash_name][key] += amount + +class FakeRouter(Router): + def __init__(self): + pass + + @webob.dec.wsgify + def __call__(self, req): + res = webob.Response() + res.status = '200' + res.headers['X-Test-Success'] = 'True' + return res + +def fake_auth_init(self, store=FakeRedis): + global auth_hash + self._store = store() + self.auth_hash = auth_hash + -- cgit From 5ff47a4513c3b5a7f8f90c417e1e62113797de8c Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Sun, 19 Sep 2010 18:23:41 -0700 Subject: updated docstring --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index efa14f9d7..64dd9d456 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -169,7 +169,7 @@ def DECLARE(name, module_string, flag_values=FLAGS): DEFINE_list('region_list', [], - 'list of region,url pairs') + 'list of region|url pairs separated by commas') DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') -- cgit From ae760b13c5382f2f4719dde445235c156cc27d18 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 20 Sep 2010 14:49:05 -0400 Subject: Use assertRaises --- nova/api/rackspace/ratelimiting/tests.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py index 13a47989b..4c9510917 100644 --- a/nova/api/rackspace/ratelimiting/tests.py +++ b/nova/api/rackspace/ratelimiting/tests.py @@ -227,13 +227,10 @@ class WSGIAppProxyTest(unittest.TestCase): self.assertEqual(when, 1.5) def test_failure(self): - self.limiter.mock('murder', 'brutus', None) - try: - when = self.proxy.perform('stab', 'brutus') - except AssertionError: - pass - else: - self.fail("I didn't perform the action I expected") + def shouldRaise(): + self.limiter.mock('murder', 'brutus', None) + self.proxy.perform('stab', 'brutus') + self.assertRaises(AssertionError, shouldRaise) if __name__ == '__main__': -- cgit From fc93548e99dea561dbf2f198b0fccc84467dbf8b Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 20 Sep 2010 17:02:32 -0400 Subject: Undo run_tests.py modification in the hopes of making this merge --- run_tests.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run_tests.py b/run_tests.py index cf37b820e..77aa9088a 100644 --- a/run_tests.py +++ b/run_tests.py @@ -50,10 +50,8 @@ from nova import flags from nova import twistd from nova.tests.access_unittest import * -from nova.tests.api_unittest import * -from nova.tests.api import * -from nova.tests.api.rackspace import * from nova.tests.auth_unittest import * +from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * -- cgit From 64dd3000c4a9b88719e86d1090097e35398d3838 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 20 Sep 2010 18:04:57 -0500 Subject: Refactored the auth branch based on review feedback --- nova/api/rackspace/__init__.py | 33 ++------- nova/api/rackspace/auth.py | 117 +++++++++++++++++++++++--------- nova/db/api.py | 15 ++++ nova/db/sqlalchemy/api.py | 23 +++++++ nova/db/sqlalchemy/models.py | 14 +++- nova/tests/api/rackspace/auth.py | 84 +++++++++++++++-------- nova/tests/api/rackspace/test_helper.py | 66 +++++++++--------- 7 files changed, 225 insertions(+), 127 deletions(-) diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py index dbba97107..f62ddc1c7 100644 --- a/nova/api/rackspace/__init__.py +++ b/nova/api/rackspace/__init__.py @@ -39,7 +39,8 @@ from nova.auth import manager FLAGS = flags.FLAGS -flags.DEFINE_string('nova_api_auth', 'nova.api.rackspace.auth.FakeAuth', +flags.DEFINE_string('nova_api_auth', + 'nova.api.rackspace.auth.BasicApiAuthManager', 'The auth mechanism to use for the Rackspace API implemenation') class API(wsgi.Middleware): @@ -49,7 +50,6 @@ class API(wsgi.Middleware): app = AuthMiddleware(APIRouter()) super(API, self).__init__(app) - class AuthMiddleware(wsgi.Middleware): """Authorize the rackspace API request or return an HTTP Forbidden.""" @@ -60,41 +60,16 @@ class AuthMiddleware(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): if not req.headers.has_key("X-Auth-Token"): - return self.authenticate(req) + return self.auth_driver.authenticate(req) user = self.auth_driver.authorize_token(req.headers["X-Auth-Token"]) if not user: return webob.exc.HTTPUnauthorized() - context = {'user':user} + context = {'user': user} req.environ['nova.context'] = context return self.application - def authenticate(self, req): - # Unless the request is explicitly made against // don't - # honor it - path_info = req.environ['wsgiorg.routing_args'][1]['path_info'] - if path_info: - return webob.exc.HTTPUnauthorized() - - if req.headers.has_key("X-Auth-User") and \ - req.headers.has_key("X-Auth-Key"): - username, key = req.headers['X-Auth-User'], req.headers['X-Auth-Key'] - token, user = self.auth_driver.authorize_user(username, key) - if user and token: - res = webob.Response() - res.headers['X-Auth-Token'] = token - res.headers['X-Server-Management-Url'] = \ - user['server_management_url'] - res.headers['X-Storage-Url'] = user['storage_url'] - res.headers['X-CDN-Management-Url'] = user['cdn_management_url'] - res.content_type = 'text/plain' - res.status = '204' - return res - else: - return webob.exc.HTTPUnauthorized() - return webob.exc.HTTPUnauthorized() - class APIRouter(wsgi.Router): """ Routes requests on the Rackspace API to the appropriate controller diff --git a/nova/api/rackspace/auth.py b/nova/api/rackspace/auth.py index d2b5193c3..b29596880 100644 --- a/nova/api/rackspace/auth.py +++ b/nova/api/rackspace/auth.py @@ -1,37 +1,88 @@ +import datetime import json -from hashlib import sha1 -from nova import datastore - -class FakeAuth(object): - def __init__(self, store=datastore.Redis.instance): - self._store = store() - self.auth_hash = 'rs_fake_auth' - self._store.hsetnx(self.auth_hash, 'rs_last_id', 0) - - def authorize_token(self, token): - user = self._store.hget(self.auth_hash, token) - if user: - return json.loads(user) - return None +import time +import webob.exc +import webob.dec +import hashlib + +from nova import auth +from nova import manager +from nova import db + +class Context(object): + pass + +class BasicApiAuthManager(manager.Manager): + """ Implements a somewhat rudimentary version of Rackspace Auth""" + + def __init__(self): + self.auth = auth.manager.AuthManager() + self.context = Context() + super(BasicApiAuthManager, self).__init__() + + def authenticate(self, req): + # Unless the request is explicitly made against // don't + # honor it + path_info = req.path_info + if len(path_info) > 1: + return webob.exc.HTTPUnauthorized() + + try: + username, key = req.headers['X-Auth-User'], \ + req.headers['X-Auth-Key'] + except KeyError: + return webob.exc.HTTPUnauthorized() - def authorize_user(self, user, key): - token = sha1("%s_%s" % (user, key)).hexdigest() - user = self._store.hget(self.auth_hash, token) - if not user: - return None, None + username, key = req.headers['X-Auth-User'], req.headers['X-Auth-Key'] + token, user = self._authorize_user(username, key) + if user and token: + res = webob.Response() + res.headers['X-Auth-Token'] = token['token_hash'] + res.headers['X-Server-Management-Url'] = \ + token['server_management_url'] + res.headers['X-Storage-Url'] = token['storage_url'] + res.headers['X-CDN-Management-Url'] = token['cdn_management_url'] + res.content_type = 'text/plain' + res.status = '204' + return res else: - return token, json.loads(user) - - def add_user(self, user, key): - last_id = self._store.hget(self.auth_hash, 'rs_last_id') - token = sha1("%s_%s" % (user, key)).hexdigest() - user = { - 'id':last_id, - 'cdn_management_url':'cdn_management_url', - 'storage_url':'storage_url', - 'server_management_url':'server_management_url' - } - new_user = self._store.hsetnx(self.auth_hash, token, json.dumps(user)) - if new_user: - self._store.hincrby(self.auth_hash, 'rs_last_id') + return webob.exc.HTTPUnauthorized() + + def authorize_token(self, token_hash): + """ retrieves user information from the datastore given a token + + If the token has expired, returns None + If the token is not found, returns None + Otherwise returns the token + + This method will also remove the token if the timestamp is older than + 2 days ago. + """ + token = self.db.auth_get_token(self.context, token_hash) + if token: + delta = datetime.datetime.now() - token['created_at'] + if delta.days >= 2: + self.db.auth_destroy_token(self.context, token) + else: + user = self.auth.get_user(self.context, token['user_id']) + return { 'id':user['id'] } + return None + + def _authorize_user(self, username, key): + """ Generates a new token and assigns it to a user """ + user = self.auth.get_user_from_access_key(key) + if user and user['name'] == username: + token_hash = hashlib.sha1('%s%s%f' % (username, key, + time.time())).hexdigest() + token = {} + token['token_hash'] = token_hash + token['cdn_management_url'] = '' + token['server_management_url'] = self._get_server_mgmt_url() + token['storage_url'] = '' + self.db.auth_create_token(self.context, token, user['id']) + return token, user + return None, None + + def _get_server_mgmt_url(self): + return 'https://%s/v1.0/' % self.host diff --git a/nova/db/api.py b/nova/db/api.py index d749ae50a..80dde7a7a 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -374,6 +374,21 @@ def export_device_create(context, values): return IMPL.export_device_create(context, values) +################### + +def auth_destroy_token(context, token): + """Destroy an auth token""" + return IMPL.auth_destroy_token(context, token) + +def auth_get_token(context, token_hash): + """Retrieves a token given the hash representing it""" + return IMPL.auth_get_token(context, token_hash) + +def auth_create_token(context, token, user_id): + """Creates a new token""" + return IMPL.auth_create_token(context, token_hash, token, user_id) + + ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 485dca2b0..681dec15e 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -539,6 +539,29 @@ def export_device_create(_context, values): ################### +def auth_destroy_token(_context, token): + session = get_session() + session.delete(token) + +def auth_get_token(_context, token_hash): + session = get_session() + tk = session.query(models.AuthToken + ).filter_by(token_hash=token_hash) + if not tk: + raise exception.NotFound('Token %s does not exist' % token_hash) + return tk + +def auth_create_token(_context, token, user_id): + tk = models.AuthToken() + for k,v in token.iteritems(): + tk[k] = v + tk.save() + return tk + + +################### + + def volume_allocate_shelf_and_blade(_context, volume_id): session = get_session() with session.begin(): diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 6818f838c..df5848ec1 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -213,7 +213,7 @@ class Instance(BASE, NovaBase): image_id = Column(String(255)) kernel_id = Column(String(255)) - ramdisk_id = Column(String(255)) + # image_id = Column(Integer, ForeignKey('images.id'), nullable=True) # kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) # ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) @@ -321,6 +321,18 @@ class NetworkIndex(BASE, NovaBase): network = relationship(Network, backref=backref('network_index', uselist=False)) +class AuthToken(BASE, NovaBase): + """Represents an authorization token for all API transactions. Fields + are a string representing the actual token and a user id for mapping + to the actual user""" + __tablename__ = 'auth_tokens' + token_hash = Column(String(255)) + user_id = Column(Integer) + server_manageent_url = Column(String(255)) + storage_url = Column(String(255)) + cdn_management_url = Column(String(255)) + + # TODO(vish): can these both come from the same baseclass? class FixedIp(BASE, NovaBase): diff --git a/nova/tests/api/rackspace/auth.py b/nova/tests/api/rackspace/auth.py index 65264fae9..8ab10d94c 100644 --- a/nova/tests/api/rackspace/auth.py +++ b/nova/tests/api/rackspace/auth.py @@ -4,23 +4,24 @@ import unittest import stubout import nova.api import nova.api.rackspace.auth +from nova import auth from nova.tests.api.rackspace import test_helper +import datetime class Test(unittest.TestCase): def setUp(self): self.stubs = stubout.StubOutForTesting() - self.stubs.Set(nova.api.rackspace.auth.FakeAuth, '__init__', - test_helper.fake_auth_init) - ds = test_helper.FakeRedis() - ds.hset(test_helper.auth_hash, 'rs_last_id', 0) + self.stubs.Set(nova.api.rackspace.auth.BasicApiAuthManager, + '__init__', test_helper.fake_auth_init) + test_helper.auth_data = {} def tearDown(self): self.stubs.UnsetAll() test_helper.fake_data_store = {} def test_authorize_user(self): - auth = nova.api.rackspace.auth.FakeAuth() - auth.add_user('herp', 'derp') + f = test_helper.FakeAuthManager() + f.add_user('derp', { 'id': 1, 'name':'herp' } ) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -29,35 +30,58 @@ class Test(unittest.TestCase): self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], - "server_management_url") + "https://foo/v1.0/") self.assertEqual(result.headers['X-CDN-Management-Url'], - "cdn_management_url") - self.assertEqual(result.headers['X-Storage-Url'], "storage_url") + "") + self.assertEqual(result.headers['X-Storage-Url'], "") - def test_authorize_token(self): - auth = nova.api.rackspace.auth.FakeAuth() - auth.add_user('herp', 'derp') + #def test_authorize_token(self): + # auth = nova.api.rackspace.auth.FakeAuth() + # auth.add_user('herp', 'derp') - req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API()) - self.assertEqual(result.status, '204 No Content') - self.assertEqual(len(result.headers['X-Auth-Token']), 40) - self.assertEqual(result.headers['X-Server-Management-Url'], - "server_management_url") - self.assertEqual(result.headers['X-CDN-Management-Url'], - "cdn_management_url") - self.assertEqual(result.headers['X-Storage-Url'], "storage_url") + # req = webob.Request.blank('/v1.0/') + # req.headers['X-Auth-User'] = 'herp' + # req.headers['X-Auth-Key'] = 'derp' + # result = req.get_response(nova.api.API()) + # self.assertEqual(result.status, '204 No Content') + # self.assertEqual(len(result.headers['X-Auth-Token']), 40) + # self.assertEqual(result.headers['X-Server-Management-Url'], + # "server_management_url") + # self.assertEqual(result.headers['X-CDN-Management-Url'], + # "cdn_management_url") + # self.assertEqual(result.headers['X-Storage-Url'], "storage_url") + + # token = result.headers['X-Auth-Token'] + # self.stubs.Set(nova.api.rackspace, 'APIRouter', + # test_helper.FakeRouter) + # req = webob.Request.blank('/v1.0/fake') + # req.headers['X-Auth-Token'] = token + # result = req.get_response(nova.api.API()) + # self.assertEqual(result.status, '200 OK') + # self.assertEqual(result.headers['X-Test-Success'], 'True') + + def test_token_expiry(self): + self.destroy_called = False + token_hash = 'bacon' + + def destroy_token_mock(meh, context, token): + self.destroy_called = True + + def bad_token(meh, context, token_hash): + return { 'token_hash':token_hash, + 'created_at':datetime.datetime(1990, 1, 1) } + + self.stubs.Set(test_helper.FakeAuthDatabase, 'auth_destroy_token', + destroy_token_mock) - token = result.headers['X-Auth-Token'] - self.stubs.Set(nova.api.rackspace, 'APIRouter', - test_helper.FakeRouter) - req = webob.Request.blank('/v1.0/fake') - req.headers['X-Auth-Token'] = token + self.stubs.Set(test_helper.FakeAuthDatabase, 'auth_get_token', + bad_token) + + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-Token'] = 'bacon' result = req.get_response(nova.api.API()) - self.assertEqual(result.status, '200 OK') - self.assertEqual(result.headers['X-Test-Success'], 'True') + self.assertEqual(result.status, '401 Unauthorized') + self.assertEqual(self.destroy_called, True) def test_bad_user(self): req = webob.Request.blank('/v1.0/') diff --git a/nova/tests/api/rackspace/test_helper.py b/nova/tests/api/rackspace/test_helper.py index 578b1e841..8d784854f 100644 --- a/nova/tests/api/rackspace/test_helper.py +++ b/nova/tests/api/rackspace/test_helper.py @@ -1,38 +1,12 @@ import webob import webob.dec from nova.wsgi import Router +from nova import auth -fake_data_store = {} -auth_hash = 'dummy_hash' +auth_data = {} -class FakeRedis(object): - def __init__(self): - global fake_data_store - self.store = fake_data_store - - def hsetnx(self, hash_name, key, value): - if not self.store.has_key(hash_name): - self.store[hash_name] = {} - - if self.store[hash_name].has_key(key): - return 0 - self.store[hash_name][key] = value - return 1 - - def hset(self, hash_name, key, value): - if not self.store.has_key(hash_name): - self.store[hash_name] = {} - - self.store[hash_name][key] = value - return 1 - - def hget(self, hash_name, key): - if not self.store[hash_name].has_key(key): - return None - return self.store[hash_name][key] - - def hincrby(self, hash_name, key, amount=1): - self.store[hash_name][key] += amount +class Context(object): + pass class FakeRouter(Router): def __init__(self): @@ -45,8 +19,32 @@ class FakeRouter(Router): res.headers['X-Test-Success'] = 'True' return res -def fake_auth_init(self, store=FakeRedis): - global auth_hash - self._store = store() - self.auth_hash = auth_hash +def fake_auth_init(self): + self.db = FakeAuthDatabase() + self.context = Context() + self.auth = FakeAuthManager() + self.host = 'foo' + +class FakeAuthDatabase(object): + @staticmethod + def auth_get_token(context, token_hash): + pass + + @staticmethod + def auth_create_token(context, token, user_id): + pass + + @staticmethod + def auth_destroy_token(context, token): + pass + +class FakeAuthManager(object): + def __init__(self): + global auth_data + self.data = auth_data + + def add_user(self, key, user): + self.data[key] = user + def get_user_from_access_key(self, key): + return self.data.get(key, None) -- cgit From ce1a8086f7ec947dd148855910a1a5a9696e33f7 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 20 Sep 2010 23:56:17 -0400 Subject: Don't use something the shell will escape as a separator. | is now =. --- nova/endpoint/cloud.py | 2 +- nova/flags.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index b28bb0dc3..2b67af96f 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -158,7 +158,7 @@ class CloudController(object): if FLAGS.region_list: regions = [] for region in FLAGS.region_list: - name, _sep, url = region.partition('|') + name, _sep, url = region.partition('=') regions.append({'regionName': name, 'regionEndpoint': url}) else: diff --git a/nova/flags.py b/nova/flags.py index 64dd9d456..c5dee2855 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -169,7 +169,7 @@ def DECLARE(name, module_string, flag_values=FLAGS): DEFINE_list('region_list', [], - 'list of region|url pairs separated by commas') + 'list of region=url pairs separated by commas') DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') -- cgit From 0880e49a4e9c9a246e8f4d7cc805d79947de095a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 21 Sep 2010 10:07:59 -0500 Subject: Some more refactoring and another unit test --- nova/api/rackspace/auth.py | 17 +++++++---- nova/db/api.py | 4 +-- nova/db/sqlalchemy/api.py | 2 +- nova/tests/api/rackspace/auth.py | 51 ++++++++++++++++----------------- nova/tests/api/rackspace/test_helper.py | 29 ++++++++++++------- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/nova/api/rackspace/auth.py b/nova/api/rackspace/auth.py index b29596880..1ef90c324 100644 --- a/nova/api/rackspace/auth.py +++ b/nova/api/rackspace/auth.py @@ -4,7 +4,7 @@ import time import webob.exc import webob.dec import hashlib - +from nova import flags from nova import auth from nova import manager from nova import db @@ -12,10 +12,16 @@ from nova import db class Context(object): pass -class BasicApiAuthManager(manager.Manager): +class BasicApiAuthManager(object): """ Implements a somewhat rudimentary version of Rackspace Auth""" def __init__(self): + if not host: + host = FLAGS.host + self.host = host + if not db_driver: + db_driver = FLAGS.db_driver + self.db = utils.import_object(db_driver) self.auth = auth.manager.AuthManager() self.context = Context() super(BasicApiAuthManager, self).__init__() @@ -64,8 +70,8 @@ class BasicApiAuthManager(manager.Manager): if delta.days >= 2: self.db.auth_destroy_token(self.context, token) else: - user = self.auth.get_user(self.context, token['user_id']) - return { 'id':user['id'] } + user = self.auth.get_user(token['user_id']) + return { 'id':user['uid'] } return None def _authorize_user(self, username, key): @@ -79,7 +85,8 @@ class BasicApiAuthManager(manager.Manager): token['cdn_management_url'] = '' token['server_management_url'] = self._get_server_mgmt_url() token['storage_url'] = '' - self.db.auth_create_token(self.context, token, user['id']) + token['user_id'] = user['uid'] + self.db.auth_create_token(self.context, token) return token, user return None, None diff --git a/nova/db/api.py b/nova/db/api.py index 80dde7a7a..0f0549edf 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -384,9 +384,9 @@ def auth_get_token(context, token_hash): """Retrieves a token given the hash representing it""" return IMPL.auth_get_token(context, token_hash) -def auth_create_token(context, token, user_id): +def auth_create_token(context, token): """Creates a new token""" - return IMPL.auth_create_token(context, token_hash, token, user_id) + return IMPL.auth_create_token(context, token_hash, token) ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 681dec15e..78bc23b7b 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -551,7 +551,7 @@ def auth_get_token(_context, token_hash): raise exception.NotFound('Token %s does not exist' % token_hash) return tk -def auth_create_token(_context, token, user_id): +def auth_create_token(_context, token): tk = models.AuthToken() for k,v in token.iteritems(): tk[k] = v diff --git a/nova/tests/api/rackspace/auth.py b/nova/tests/api/rackspace/auth.py index 8ab10d94c..0f38ce79d 100644 --- a/nova/tests/api/rackspace/auth.py +++ b/nova/tests/api/rackspace/auth.py @@ -13,7 +13,8 @@ class Test(unittest.TestCase): self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.rackspace.auth.BasicApiAuthManager, '__init__', test_helper.fake_auth_init) - test_helper.auth_data = {} + test_helper.FakeAuthManager.auth_data = {} + test_helper.FakeAuthDatabase.data = {} def tearDown(self): self.stubs.UnsetAll() @@ -21,8 +22,22 @@ class Test(unittest.TestCase): def test_authorize_user(self): f = test_helper.FakeAuthManager() - f.add_user('derp', { 'id': 1, 'name':'herp' } ) + f.add_user('derp', { 'uid': 1, 'name':'herp' } ) + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'herp' + req.headers['X-Auth-Key'] = 'derp' + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '204 No Content') + self.assertEqual(len(result.headers['X-Auth-Token']), 40) + self.assertEqual(result.headers['X-CDN-Management-Url'], + "") + self.assertEqual(result.headers['X-Storage-Url'], "") + + def test_authorize_token(self): + f = test_helper.FakeAuthManager() + f.add_user('derp', { 'uid': 1, 'name':'herp' } ) + req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' @@ -35,30 +50,14 @@ class Test(unittest.TestCase): "") self.assertEqual(result.headers['X-Storage-Url'], "") - #def test_authorize_token(self): - # auth = nova.api.rackspace.auth.FakeAuth() - # auth.add_user('herp', 'derp') - - # req = webob.Request.blank('/v1.0/') - # req.headers['X-Auth-User'] = 'herp' - # req.headers['X-Auth-Key'] = 'derp' - # result = req.get_response(nova.api.API()) - # self.assertEqual(result.status, '204 No Content') - # self.assertEqual(len(result.headers['X-Auth-Token']), 40) - # self.assertEqual(result.headers['X-Server-Management-Url'], - # "server_management_url") - # self.assertEqual(result.headers['X-CDN-Management-Url'], - # "cdn_management_url") - # self.assertEqual(result.headers['X-Storage-Url'], "storage_url") - - # token = result.headers['X-Auth-Token'] - # self.stubs.Set(nova.api.rackspace, 'APIRouter', - # test_helper.FakeRouter) - # req = webob.Request.blank('/v1.0/fake') - # req.headers['X-Auth-Token'] = token - # result = req.get_response(nova.api.API()) - # self.assertEqual(result.status, '200 OK') - # self.assertEqual(result.headers['X-Test-Success'], 'True') + token = result.headers['X-Auth-Token'] + self.stubs.Set(nova.api.rackspace, 'APIRouter', + test_helper.FakeRouter) + req = webob.Request.blank('/v1.0/fake') + req.headers['X-Auth-Token'] = token + result = req.get_response(nova.api.API()) + self.assertEqual(result.status, '200 OK') + self.assertEqual(result.headers['X-Test-Success'], 'True') def test_token_expiry(self): self.destroy_called = False diff --git a/nova/tests/api/rackspace/test_helper.py b/nova/tests/api/rackspace/test_helper.py index 8d784854f..18d96d71e 100644 --- a/nova/tests/api/rackspace/test_helper.py +++ b/nova/tests/api/rackspace/test_helper.py @@ -1,10 +1,9 @@ import webob import webob.dec +import datetime from nova.wsgi import Router from nova import auth -auth_data = {} - class Context(object): pass @@ -26,25 +25,33 @@ def fake_auth_init(self): self.host = 'foo' class FakeAuthDatabase(object): + data = {} + @staticmethod def auth_get_token(context, token_hash): - pass + return FakeAuthDatabase.data.get(token_hash, None) @staticmethod - def auth_create_token(context, token, user_id): - pass + def auth_create_token(context, token): + token['created_at'] = datetime.datetime.now() + FakeAuthDatabase.data[token['token_hash']] = token @staticmethod def auth_destroy_token(context, token): - pass + if FakeAuthDatabase.data.has_key(token['token_hash']): + del FakeAuthDatabase.data['token_hash'] class FakeAuthManager(object): - def __init__(self): - global auth_data - self.data = auth_data + auth_data = {} def add_user(self, key, user): - self.data[key] = user + FakeAuthManager.auth_data[key] = user + + def get_user(self, uid): + for k, v in FakeAuthManager.auth_data.iteritems(): + if v['uid'] == uid: + return v + return None def get_user_from_access_key(self, key): - return self.data.get(key, None) + return FakeAuthManager.auth_data.get(key, None) -- cgit From dff6c134cb5b540ac1344faf9f0cbe7d19a8c9e7 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 21 Sep 2010 14:00:17 -0400 Subject: Fix quota unittest and don't run rbac unit tests for the moment --- nova/tests/api_unittest.py | 3 ++- nova/tests/cloud_unittest.py | 5 ++--- nova/tests/quota_unittest.py | 39 ++++++++++++++++++--------------------- run_tests.py | 3 ++- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 3d2724d12..0732c39bb 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -26,8 +26,9 @@ import StringIO import webob from nova import test -from nova.auth import manager from nova import api +from nova.api.ec2 import cloud +from nova.auth import manager class FakeHttplibSocket(object): diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 247ccc093..2f22982eb 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -63,9 +63,8 @@ class CloudTestCase(test.BaseTestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('proj', 'admin', 'proj') - self.context = context.APIRequestContext(handler=None, - user=self.user, - project=self.project) + self.context = context.APIRequestContext(user=self.user, + project=self.project) def tearDown(self): self.manager.delete_project(self.project) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index cab9f663d..370ccd506 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -25,8 +25,8 @@ from nova import quota from nova import test from nova import utils from nova.auth import manager -from nova.endpoint import cloud -from nova.endpoint import api +from nova.api.ec2 import cloud +from nova.api.ec2 import context FLAGS = flags.FLAGS @@ -48,9 +48,8 @@ class QuotaTestCase(test.TrialTestCase): self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('admin', 'admin', 'admin') self.network = utils.import_object(FLAGS.network_manager) - self.context = api.APIRequestContext(handler=None, - project=self.project, - user=self.user) + self.context = context.APIRequestContext(project=self.project, + user=self.user) def tearDown(self): # pylint: disable-msg=C0103 manager.AuthManager().delete_project(self.project) @@ -95,11 +94,11 @@ class QuotaTestCase(test.TrialTestCase): for i in range(FLAGS.quota_instances): instance_id = self._create_instance() instance_ids.append(instance_id) - self.assertFailure(self.cloud.run_instances(self.context, - min_count=1, - max_count=1, - instance_type='m1.small'), - cloud.QuotaError) + self.assertRaises(cloud.QuotaError, self.cloud.run_instances, + self.context, + min_count=1, + max_count=1, + instance_type='m1.small') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -107,11 +106,11 @@ class QuotaTestCase(test.TrialTestCase): instance_ids = [] instance_id = self._create_instance(cores=4) instance_ids.append(instance_id) - self.assertFailure(self.cloud.run_instances(self.context, - min_count=1, - max_count=1, - instance_type='m1.small'), - cloud.QuotaError) + self.assertRaises(cloud.QuotaError, self.cloud.run_instances, + self.context, + min_count=1, + max_count=1, + instance_type='m1.small') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -120,10 +119,9 @@ class QuotaTestCase(test.TrialTestCase): for i in range(FLAGS.quota_volumes): volume_id = self._create_volume() volume_ids.append(volume_id) - self.assertRaises(cloud.QuotaError, - self.cloud.create_volume, - self.context, - size=10) + self.assertRaises(cloud.QuotaError, self.cloud.create_volume, + self.context, + size=10) for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -151,5 +149,4 @@ class QuotaTestCase(test.TrialTestCase): # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertFailure(self.cloud.allocate_address(self.context), - cloud.QuotaError) + self.assertRaises(cloud.QuotaError, self.cloud.allocate_address, self.context) diff --git a/run_tests.py b/run_tests.py index 4121f4c06..bea97c0b3 100644 --- a/run_tests.py +++ b/run_tests.py @@ -49,7 +49,8 @@ from nova import datastore from nova import flags from nova import twistd -from nova.tests.access_unittest import * +#TODO(gundlach): rewrite and readd this after merge +#from nova.tests.access_unittest import * from nova.tests.auth_unittest import * from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * -- cgit From e027342cc647db080ee77de53b22126caf958339 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 21 Sep 2010 14:34:40 -0400 Subject: In desperation, I'm raising eventlet.__version__ so I can see why the trunk tests are failing. --- nova/tests/api_unittest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 0732c39bb..7e81d3dc8 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -107,6 +107,8 @@ class ApiEc2TestCase(test.BaseTestCase): def test_describe_instances(self): """Test that, after creating a user and a project, the describe instances call to the API works properly""" + import eventlet + raise KeyError(eventlet.__version__) self.expect_http() self.mox.ReplayAll() user = self.manager.create_user('fake', 'fake', 'fake') -- cgit From b82a9e3d3ca46e69a1583dea51a474456b867e6f Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 21 Sep 2010 15:00:43 -0400 Subject: Remove eventlet test, now that eventlet 0.9.10 has indeed been replaced by 0.9.12 per mtaylor --- nova/tests/api_unittest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 7e81d3dc8..0732c39bb 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -107,8 +107,6 @@ class ApiEc2TestCase(test.BaseTestCase): def test_describe_instances(self): """Test that, after creating a user and a project, the describe instances call to the API works properly""" - import eventlet - raise KeyError(eventlet.__version__) self.expect_http() self.mox.ReplayAll() user = self.manager.create_user('fake', 'fake', 'fake') -- cgit From 7a19f6f3978fc0942d5bc51a1ad3299968a4d215 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 21 Sep 2010 15:46:19 -0500 Subject: Missed the model include, and fixed a broke test after the merge --- nova/db/sqlalchemy/models.py | 3 ++- nova/tests/api/rackspace/auth.py | 2 ++ nova/tests/api/rackspace/test_helper.py | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index bd1e9164e..6e1c0ce16 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -475,7 +475,8 @@ def register_models(): """Register Models and create metadata""" from sqlalchemy import create_engine models = (Service, Instance, Volume, ExportDevice, - FixedIp, FloatingIp, Network, NetworkIndex) # , Image, Host) + FixedIp, FloatingIp, Network, NetworkIndex, + AuthToken) # , Image, Host) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/tests/api/rackspace/auth.py b/nova/tests/api/rackspace/auth.py index 0f38ce79d..429c22ad2 100644 --- a/nova/tests/api/rackspace/auth.py +++ b/nova/tests/api/rackspace/auth.py @@ -15,6 +15,8 @@ class Test(unittest.TestCase): '__init__', test_helper.fake_auth_init) test_helper.FakeAuthManager.auth_data = {} test_helper.FakeAuthDatabase.data = {} + self.stubs.Set(nova.api.rackspace, 'RateLimitingMiddleware', + test_helper.FakeRateLimiter) def tearDown(self): self.stubs.UnsetAll() diff --git a/nova/tests/api/rackspace/test_helper.py b/nova/tests/api/rackspace/test_helper.py index 18d96d71e..be14e2de8 100644 --- a/nova/tests/api/rackspace/test_helper.py +++ b/nova/tests/api/rackspace/test_helper.py @@ -55,3 +55,11 @@ class FakeAuthManager(object): def get_user_from_access_key(self, key): return FakeAuthManager.auth_data.get(key, None) + +class FakeRateLimiter(object): + def __init__(self, application): + self.application = application + + @webob.dec.wsgify + def __call__(self, req): + return self.application -- cgit From 84fbfe09e10b330a5668e99422247801f370d0f9 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 21 Sep 2010 16:57:08 -0400 Subject: Rewrite rbac tests to use Authorizer middleware --- doc/source/auth.rst | 8 ---- nova/api/ec2/__init__.py | 3 ++ nova/auth/manager.py | 4 +- nova/tests/access_unittest.py | 108 ++++++++++++++++-------------------------- 4 files changed, 47 insertions(+), 76 deletions(-) diff --git a/doc/source/auth.rst b/doc/source/auth.rst index 70aca704a..3fcb309cd 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -172,14 +172,6 @@ Further Challenges -The :mod:`rbac` Module --------------------------- - -.. automodule:: nova.auth.rbac - :members: - :undoc-members: - :show-inheritance: - The :mod:`signer` Module ------------------------ diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index a7b10e428..b041787c2 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -25,6 +25,7 @@ import webob.dec import webob.exc from nova import exception +from nova import flags from nova import wsgi from nova.api.ec2 import apirequest from nova.api.ec2 import context @@ -33,6 +34,7 @@ from nova.api.ec2 import cloud from nova.auth import manager +FLAGS = flags.FLAGS _log = logging.getLogger("api") _log.setLevel(logging.DEBUG) @@ -176,6 +178,7 @@ class Authorizer(wsgi.Middleware): controller_name = req.environ['ec2.controller'].__class__.__name__ action = req.environ['ec2.action'] allowed_roles = self.action_roles[controller_name].get(action, []) + allowed_roles.extend(FLAGS.superuser_roles) if self._matches_any_role(context, allowed_roles): return self.application else: diff --git a/nova/auth/manager.py b/nova/auth/manager.py index bc3a8a12e..928e0fd69 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -44,7 +44,7 @@ flags.DEFINE_list('allowed_roles', # NOTE(vish): a user with one of these roles will be a superuser and # have access to all api commands flags.DEFINE_list('superuser_roles', ['cloudadmin'], - 'Roles that ignore rbac checking completely') + 'Roles that ignore authorization checking completely') # NOTE(vish): a user with one of these roles will have it for every # project, even if he or she is not a member of the project @@ -304,7 +304,7 @@ class AuthManager(object): return "%s:%s" % (user.access, Project.safe_id(project)) def is_superuser(self, user): - """Checks for superuser status, allowing user to bypass rbac + """Checks for superuser status, allowing user to bypass authorization @type user: User or uid @param user: User to check. diff --git a/nova/tests/access_unittest.py b/nova/tests/access_unittest.py index 59e1683db..d85f559f3 100644 --- a/nova/tests/access_unittest.py +++ b/nova/tests/access_unittest.py @@ -18,12 +18,13 @@ import unittest import logging +import webob from nova import exception from nova import flags from nova import test +from nova.api import ec2 from nova.auth import manager -from nova.auth import rbac FLAGS = flags.FLAGS @@ -72,9 +73,14 @@ class AccessTestCase(test.BaseTestCase): try: self.project.add_role(self.testsys, 'sysadmin') except: pass - self.context = Context() - self.context.project = self.project #user is set in each test + self.mw = ec2.Authorizer(lambda x,y: y('200 OK', []) and '') + self.mw.action_roles = {'str': { + '_allow_all': ['all'], + '_allow_none': [], + '_allow_project_manager': ['projectmanager'], + '_allow_sys_and_net': ['sysadmin', 'netadmin'], + '_allow_sysadmin': ['sysadmin']}} def tearDown(self): um = manager.AuthManager() @@ -87,76 +93,46 @@ class AccessTestCase(test.BaseTestCase): um.delete_user('testsys') super(AccessTestCase, self).tearDown() + def response_status(self, user, methodName): + context = Context() + context.project = self.project + context.user = user + environ = {'ec2.context' : context, + 'ec2.controller': 'some string', + 'ec2.action': methodName} + req = webob.Request.blank('/', environ) + resp = req.get_response(self.mw) + return resp.status_int + + def shouldAllow(self, user, methodName): + self.assertEqual(200, self.response_status(user, methodName)) + + def shouldDeny(self, user, methodName): + self.assertEqual(401, self.response_status(user, methodName)) + def test_001_allow_all(self): - self.context.user = self.testadmin - self.assertTrue(self._allow_all(self.context)) - self.context.user = self.testpmsys - self.assertTrue(self._allow_all(self.context)) - self.context.user = self.testnet - self.assertTrue(self._allow_all(self.context)) - self.context.user = self.testsys - self.assertTrue(self._allow_all(self.context)) + users = [self.testadmin, self.testpmsys, self.testnet, self.testsys] + for user in users: + self.shouldAllow(user, '_allow_all') def test_002_allow_none(self): - self.context.user = self.testadmin - self.assertTrue(self._allow_none(self.context)) - self.context.user = self.testpmsys - self.assertRaises(exception.NotAuthorized, self._allow_none, self.context) - self.context.user = self.testnet - self.assertRaises(exception.NotAuthorized, self._allow_none, self.context) - self.context.user = self.testsys - self.assertRaises(exception.NotAuthorized, self._allow_none, self.context) + self.shouldAllow(self.testadmin, '_allow_none') + users = [self.testpmsys, self.testnet, self.testsys] + for user in users: + self.shouldDeny(user, '_allow_none') def test_003_allow_project_manager(self): - self.context.user = self.testadmin - self.assertTrue(self._allow_project_manager(self.context)) - self.context.user = self.testpmsys - self.assertTrue(self._allow_project_manager(self.context)) - self.context.user = self.testnet - self.assertRaises(exception.NotAuthorized, self._allow_project_manager, self.context) - self.context.user = self.testsys - self.assertRaises(exception.NotAuthorized, self._allow_project_manager, self.context) + for user in [self.testadmin, self.testpmsys]: + self.shouldAllow(user, '_allow_project_manager') + for user in [self.testnet, self.testsys]: + self.shouldDeny(user, '_allow_project_manager') def test_004_allow_sys_and_net(self): - self.context.user = self.testadmin - self.assertTrue(self._allow_sys_and_net(self.context)) - self.context.user = self.testpmsys # doesn't have the per project sysadmin - self.assertRaises(exception.NotAuthorized, self._allow_sys_and_net, self.context) - self.context.user = self.testnet - self.assertTrue(self._allow_sys_and_net(self.context)) - self.context.user = self.testsys - self.assertTrue(self._allow_sys_and_net(self.context)) - - def test_005_allow_sys_no_pm(self): - self.context.user = self.testadmin - self.assertTrue(self._allow_sys_no_pm(self.context)) - self.context.user = self.testpmsys - self.assertRaises(exception.NotAuthorized, self._allow_sys_no_pm, self.context) - self.context.user = self.testnet - self.assertRaises(exception.NotAuthorized, self._allow_sys_no_pm, self.context) - self.context.user = self.testsys - self.assertTrue(self._allow_sys_no_pm(self.context)) - - @rbac.allow('all') - def _allow_all(self, context): - return True - - @rbac.allow('none') - def _allow_none(self, context): - return True - - @rbac.allow('projectmanager') - def _allow_project_manager(self, context): - return True - - @rbac.allow('sysadmin', 'netadmin') - def _allow_sys_and_net(self, context): - return True - - @rbac.allow('sysadmin') - @rbac.deny('projectmanager') - def _allow_sys_no_pm(self, context): - return True + for user in [self.testadmin, self.testnet, self.testsys]: + self.shouldAllow(user, '_allow_sys_and_net') + # denied because it doesn't have the per project sysadmin + for user in [self.testpmsys]: + self.shouldDeny(user, '_allow_sys_and_net') if __name__ == "__main__": # TODO: Implement use_fake as an option -- cgit From ffa426d68bfb3d1c2acaeef4c48d2662e88fc878 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 21 Sep 2010 16:58:08 -0400 Subject: Reenable access_unittest now that it works with new rbac --- run_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run_tests.py b/run_tests.py index bea97c0b3..4121f4c06 100644 --- a/run_tests.py +++ b/run_tests.py @@ -49,8 +49,7 @@ from nova import datastore from nova import flags from nova import twistd -#TODO(gundlach): rewrite and readd this after merge -#from nova.tests.access_unittest import * +from nova.tests.access_unittest import * from nova.tests.auth_unittest import * from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * -- cgit From a8c5901faaa98b7f0c06db086a03a0d38a210986 Mon Sep 17 00:00:00 2001 From: mdietz Date: Wed, 22 Sep 2010 18:46:55 +0000 Subject: Added a primary_key to AuthToken, fixed some unbound variables, and now all unit tests pass --- nova/api/rackspace/auth.py | 5 ++++- nova/db/sqlalchemy/models.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/api/rackspace/auth.py b/nova/api/rackspace/auth.py index 1ef90c324..ce5a967eb 100644 --- a/nova/api/rackspace/auth.py +++ b/nova/api/rackspace/auth.py @@ -8,6 +8,9 @@ from nova import flags from nova import auth from nova import manager from nova import db +from nova import utils + +FLAGS = flags.FLAGS class Context(object): pass @@ -15,7 +18,7 @@ class Context(object): class BasicApiAuthManager(object): """ Implements a somewhat rudimentary version of Rackspace Auth""" - def __init__(self): + def __init__(self, host=None, db_driver=None): if not host: host = FLAGS.host self.host = host diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 6e1c0ce16..161c5f1bc 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -401,7 +401,7 @@ class AuthToken(BASE, NovaBase): are a string representing the actual token and a user id for mapping to the actual user""" __tablename__ = 'auth_tokens' - token_hash = Column(String(255)) + token_hash = Column(String(255), primary_key=True) user_id = Column(Integer) server_manageent_url = Column(String(255)) storage_url = Column(String(255)) -- cgit From f3f271644eac4ec74ce3786840a7743aac4f6032 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 22 Sep 2010 15:57:24 -0400 Subject: Responding to eday's feedback -- make a clearer inner wsgi app --- nova/tests/access_unittest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/tests/access_unittest.py b/nova/tests/access_unittest.py index d85f559f3..c8a49d2ca 100644 --- a/nova/tests/access_unittest.py +++ b/nova/tests/access_unittest.py @@ -74,7 +74,10 @@ class AccessTestCase(test.BaseTestCase): self.project.add_role(self.testsys, 'sysadmin') except: pass #user is set in each test - self.mw = ec2.Authorizer(lambda x,y: y('200 OK', []) and '') + def noopWSGIApp(environ, start_response): + start_response('200 OK', []) + return [''] + self.mw = ec2.Authorizer(noopWSGIApp) self.mw.action_roles = {'str': { '_allow_all': ['all'], '_allow_none': [], -- cgit From 6f82d0f84c9474e72ef70c9ff568d68031191e0a Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 22 Sep 2010 17:35:02 -0400 Subject: Soren's patch to fix part of ec2 --- nova/api/ec2/apirequest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index a3b20118f..a87c21fb3 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -68,10 +68,8 @@ class APIRequest(object): key = _camelcase_to_underscore(parts[0]) if len(parts) > 1: d = args.get(key, {}) - d[parts[1]] = value[0] + d[parts[1]] = value value = d - else: - value = value[0] args[key] = value for key in args.keys(): -- cgit From 9e12753508474b430c1b87fd7d59dcbc2d096042 Mon Sep 17 00:00:00 2001 From: mdietz Date: Wed, 22 Sep 2010 21:57:34 +0000 Subject: Re-added the ramdisk line I accidentally removed --- nova/db/sqlalchemy/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 161c5f1bc..f6ba7953f 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -213,7 +213,8 @@ class Instance(BASE, NovaBase): image_id = Column(String(255)) kernel_id = Column(String(255)) - + ramdisk_id = Column(String(255)) + # image_id = Column(Integer, ForeignKey('images.id'), nullable=True) # kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) # ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) -- cgit From f188b5a02d34751e89fae60b4d3b1ef144f138d7 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 22 Sep 2010 19:11:04 -0400 Subject: Re-add root and metadata request handlers to EC2 API --- nova/api/__init__.py | 56 +++++++++++++++++++++++++-- nova/api/ec2/metadatarequesthandler.py | 71 ++++++++++++++++++++++++++++++++++ nova/tests/api/__init__.py | 30 ++++++++++++-- 3 files changed, 149 insertions(+), 8 deletions(-) create mode 100644 nova/api/ec2/metadatarequesthandler.py diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 821f1deea..a0be05d86 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -23,9 +23,18 @@ Root WSGI middleware for all API controllers. import routes import webob.dec +from nova import flags from nova import wsgi from nova.api import ec2 from nova.api import rackspace +from nova.api.ec2 import metadatarequesthandler + + +flags.DEFINE_string('rsapi_subdomain', 'rs', + 'subdomain running the RS API') +flags.DEFINE_string('ec2api_subdomain', 'ec2', + 'subdomain running the EC2 API') +FLAGS = flags.FLAGS class API(wsgi.Router): @@ -33,13 +42,33 @@ class API(wsgi.Router): def __init__(self): mapper = routes.Mapper() - mapper.connect("/", controller=self.versions) - mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API()) - mapper.connect("/services/{path_info:.*}", controller=ec2.API()) + mapper.sub_domains = True + mapper.connect("/", controller=self.rsapi_versions, + conditions={'sub_domain': [FLAGS.rsapi_subdomain]}) + mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API(), + conditions={'sub_domain': [FLAGS.rsapi_subdomain]}) + + mapper.connect("/", controller=self.ec2api_versions, + conditions={'sub_domain': [FLAGS.ec2api_subdomain]}) + mapper.connect("/services/{path_info:.*}", controller=ec2.API(), + conditions={'sub_domain': [FLAGS.ec2api_subdomain]}) + mrh = metadatarequesthandler.MetadataRequestHandler() + for s in ['/latest', + '/2009-04-04', + '/2008-09-01', + '/2008-02-01', + '/2007-12-15', + '/2007-10-10', + '/2007-08-29', + '/2007-03-01', + '/2007-01-19', + '/1.0']: + mapper.connect('%s/{path_info:.*}' % s, controller=mrh, + conditions={'subdomain': FLAGS.ec2api_subdomain}) super(API, self).__init__(mapper) @webob.dec.wsgify - def versions(self, req): + def rsapi_versions(self, req): """Respond to a request for all OpenStack API versions.""" response = { "versions": [ @@ -48,3 +77,22 @@ class API(wsgi.Router): "application/xml": { "attributes": dict(version=["status", "id"])}} return wsgi.Serializer(req.environ, metadata).to_content_type(response) + + @webob.dec.wsgify + def ec2api_versions(self, req): + """Respond to a request for all EC2 versions.""" + # available api versions + versions = [ + '1.0', + '2007-01-19', + '2007-03-01', + '2007-08-29', + '2007-10-10', + '2007-12-15', + '2008-02-01', + '2008-09-01', + '2009-04-04', + ] + return ''.join('%s\n' % v for v in versions) + + diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py new file mode 100644 index 000000000..229e5a78d --- /dev/null +++ b/nova/api/ec2/metadatarequesthandler.py @@ -0,0 +1,71 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Metadata request handler.""" + +import webob.dec +import webob.exc + +from nova.api.ec2 import cloud + + +class MetadataRequestHandler(object): + + """Serve metadata from the EC2 API.""" + + def print_data(self, data): + if isinstance(data, dict): + output = '' + for key in data: + if key == '_name': + continue + output += key + if isinstance(data[key], dict): + if '_name' in data[key]: + output += '=' + str(data[key]['_name']) + else: + output += '/' + output += '\n' + return output[:-1] # cut off last \n + elif isinstance(data, list): + return '\n'.join(data) + else: + return str(data) + + def lookup(self, path, data): + items = path.split('/') + for item in items: + if item: + if not isinstance(data, dict): + return data + if not item in data: + return None + data = data[item] + return data + + @webob.dec.wsgify + def __call__(self, req): + cc = cloud.CloudController() + meta_data = cc.get_metadata(req.remote_addr) + if meta_data is None: + _log.error('Failed to get metadata for ip: %s' % req.remote_addr) + raise webob.exc.HTTPNotFound() + data = self.lookup(path, meta_data) + if data is None: + raise webob.exc.HTTPNotFound() + return self.print_data(data) diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index 4682c094e..fc1ab9ae2 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -25,6 +25,7 @@ import stubout import webob import webob.dec +import nova.exception from nova import api from nova.tests.api.test_helper import * @@ -36,25 +37,46 @@ class Test(unittest.TestCase): def tearDown(self): # pylint: disable-msg=C0103 self.stubs.UnsetAll() + def _request(self, url, subdomain, **kwargs): + environ_keys = {'HTTP_HOST': '%s.example.com' % subdomain} + environ_keys.update(kwargs) + req = webob.Request.blank(url, environ_keys) + return req.get_response(api.API()) + def test_rackspace(self): self.stubs.Set(api.rackspace, 'API', APIStub) - result = webob.Request.blank('/v1.0/cloud').get_response(api.API()) + result = self._request('/v1.0/cloud', 'rs') self.assertEqual(result.body, "/cloud") def test_ec2(self): self.stubs.Set(api.ec2, 'API', APIStub) - result = webob.Request.blank('/ec2/cloud').get_response(api.API()) + result = self._request('/services/cloud', 'ec2') self.assertEqual(result.body, "/cloud") def test_not_found(self): self.stubs.Set(api.ec2, 'API', APIStub) self.stubs.Set(api.rackspace, 'API', APIStub) - result = webob.Request.blank('/test/cloud').get_response(api.API()) + result = self._request('/test/cloud', 'ec2') self.assertNotEqual(result.body, "/cloud") def test_query_api_versions(self): - result = webob.Request.blank('/').get_response(api.API()) + result = self._request('/', 'rs') self.assertTrue('CURRENT' in result.body) + def test_metadata(self): + def go(url): + result = self._request(url, 'ec2', + REMOTE_ADDR='128.192.151.2') + # Each should get to the ORM layer and fail to find the IP + self.assertRaises(nova.exception.NotFound, go, '/latest/') + self.assertRaises(nova.exception.NotFound, go, '/2009-04-04/') + self.assertRaises(nova.exception.NotFound, go, '/1.0/') + + def test_ec2_root(self): + result = self._request('/', 'ec2') + self.assertTrue('2007-12-15\n' in result.body) + + + if __name__ == '__main__': unittest.main() -- cgit From 54122c0a156d1562be76dfde41bd62006f9ed426 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 22 Sep 2010 17:54:57 -0700 Subject: Various loose ends for endpoint and tornado removal cleanup, including cloudpipe API addition, rpc.call() cleanup by removing tornado ioloop, and fixing bin/* programs. Tornado still exists as part of some test cases and those should be reworked to not require it. --- bin/nova-api | 47 +++++++++-------------------- bin/nova-api-new | 45 ---------------------------- bin/nova-manage | 3 +- nova/api/__init__.py | 2 ++ nova/api/cloudpipe/__init__.py | 68 ++++++++++++++++++++++++++++++++++++++++++ nova/cloudpipe/api.py | 59 ------------------------------------ nova/cloudpipe/pipelib.py | 3 +- nova/rpc.py | 34 +++++++++++---------- nova/tests/cloud_unittest.py | 1 - 9 files changed, 105 insertions(+), 157 deletions(-) delete mode 100755 bin/nova-api-new create mode 100644 nova/api/cloudpipe/__init__.py delete mode 100644 nova/cloudpipe/api.py diff --git a/bin/nova-api b/bin/nova-api index ede09d38c..8625c487f 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,31 +1,28 @@ #!/usr/bin/env python +# pylint: disable-msg=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ -Tornado daemon for the main API endpoint. +Nova API daemon. """ -import logging import os import sys -from tornado import httpserver -from tornado import ioloop # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -35,30 +32,14 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +from nova import api from nova import flags -from nova import server from nova import utils -from nova.endpoint import admin -from nova.endpoint import api -from nova.endpoint import cloud +from nova import wsgi FLAGS = flags.FLAGS - - -def main(_argv): - """Load the controllers and start the tornado I/O loop.""" - controllers = { - 'Cloud': cloud.CloudController(), - 'Admin': admin.AdminController()} - _app = api.APIServerApplication(controllers) - - io_inst = ioloop.IOLoop.instance() - http_server = httpserver.HTTPServer(_app) - http_server.listen(FLAGS.cc_port) - logging.debug('Started HTTP server on %s', FLAGS.cc_port) - io_inst.start() - +flags.DEFINE_integer('api_port', 8773, 'API port') if __name__ == '__main__': utils.default_flagfile() - server.serve('nova-api', main) + wsgi.run_server(api.API(), FLAGS.api_port) diff --git a/bin/nova-api-new b/bin/nova-api-new deleted file mode 100755 index 8625c487f..000000000 --- a/bin/nova-api-new +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# pylint: disable-msg=C0103 -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Nova API daemon. -""" - -import os -import sys - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -from nova import api -from nova import flags -from nova import utils -from nova import wsgi - -FLAGS = flags.FLAGS -flags.DEFINE_integer('api_port', 8773, 'API port') - -if __name__ == '__main__': - utils.default_flagfile() - wsgi.run_server(api.API(), FLAGS.api_port) diff --git a/bin/nova-manage b/bin/nova-manage index 824e00ac5..baa1cb4db 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -73,7 +73,6 @@ from nova import quota from nova import utils from nova.auth import manager from nova.cloudpipe import pipelib -from nova.endpoint import cloud FLAGS = flags.FLAGS @@ -84,7 +83,7 @@ class VpnCommands(object): def __init__(self): self.manager = manager.AuthManager() - self.pipe = pipelib.CloudPipe(cloud.CloudController()) + self.pipe = pipelib.CloudPipe() def list(self): """Print a listing of the VPNs for all projects.""" diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 821f1deea..ff9b94de9 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -24,6 +24,7 @@ import routes import webob.dec from nova import wsgi +from nova.api import cloudpipe from nova.api import ec2 from nova.api import rackspace @@ -36,6 +37,7 @@ class API(wsgi.Router): mapper.connect("/", controller=self.versions) mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API()) mapper.connect("/services/{path_info:.*}", controller=ec2.API()) + mapper.connect("/cloudpipe/{path_info:.*}", controller=cloudpipe.API()) super(API, self).__init__(mapper) @webob.dec.wsgify diff --git a/nova/api/cloudpipe/__init__.py b/nova/api/cloudpipe/__init__.py new file mode 100644 index 000000000..642f8ef6c --- /dev/null +++ b/nova/api/cloudpipe/__init__.py @@ -0,0 +1,68 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +REST API Request Handlers for CloudPipe +""" + +import logging +import urllib +import webob +import webob.dec +import webob.exc + +from nova import crypto +from nova import wsgi +from nova.auth import manager +from nova.api.ec2 import cloud + + +_log = logging.getLogger("api") +_log.setLevel(logging.DEBUG) + + +class API(wsgi.Application): + + def __init__(self): + self.controller = cloud.CloudController() + + @webob.dec.wsgify + def __call__(self, req): + if req.method == 'POST': + return self.sign_csr(req) + _log.debug("Cloudpipe path is %s" % req.path_info) + if req.path_info.endswith("/getca/"): + return self.send_root_ca(req) + return webob.exc.HTTPNotFound() + + def get_project_id_from_ip(self, ip): + instance = self.controller.get_instance_by_ip(ip) + return instance['project_id'] + + def send_root_ca(self, req): + _log.debug("Getting root ca") + project_id = self.get_project_id_from_ip(req.remote_addr) + res = webob.Response() + res.headers["Content-Type"] = "text/plain" + res.body = crypto.fetch_ca(project_id) + return res + + def sign_csr(self, req): + project_id = self.get_project_id_from_ip(req.remote_addr) + cert = self.str_params['cert'] + return crypto.sign_csr(urllib.unquote(cert), project_id) diff --git a/nova/cloudpipe/api.py b/nova/cloudpipe/api.py deleted file mode 100644 index 56aa89834..000000000 --- a/nova/cloudpipe/api.py +++ /dev/null @@ -1,59 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Tornado REST API Request Handlers for CloudPipe -""" - -import logging -import urllib - -import tornado.web - -from nova import crypto -from nova.auth import manager - - -_log = logging.getLogger("api") -_log.setLevel(logging.DEBUG) - - -class CloudPipeRequestHandler(tornado.web.RequestHandler): - def get(self, path): - path = self.request.path - _log.debug( "Cloudpipe path is %s" % path) - if path.endswith("/getca/"): - self.send_root_ca() - self.finish() - - def get_project_id_from_ip(self, ip): - cc = self.application.controllers['Cloud'] - instance = cc.get_instance_by_ip(ip) - instance['project_id'] - - def send_root_ca(self): - _log.debug( "Getting root ca") - project_id = self.get_project_id_from_ip(self.request.remote_ip) - self.set_header("Content-Type", "text/plain") - self.write(crypto.fetch_ca(project_id)) - - def post(self, *args, **kwargs): - project_id = self.get_project_id_from_ip(self.request.remote_ip) - cert = self.get_argument('cert', '') - self.write(crypto.sign_csr(urllib.unquote(cert), project_id)) - self.finish() diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index b13a60292..abc14bbb6 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -32,6 +32,7 @@ from nova import exception from nova import flags from nova import utils from nova.auth import manager +from nova.api.ec2 import cloud from nova.api.ec2 import context @@ -43,7 +44,7 @@ flags.DEFINE_string('boot_script_template', class CloudPipe(object): def __init__(self, cloud_controller): - self.controller = cloud_controller + self.controller = cloud.CloudController() self.manager = manager.AuthManager() def launch_vpn_instance(self, project_id): diff --git a/nova/rpc.py b/nova/rpc.py index 84a9b5590..7e4d91a03 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -94,8 +94,6 @@ class Consumer(messaging.Consumer): injected.start() return injected - attachToTornado = attach_to_tornado - def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" # TODO(vish): the logic for failed connections and logging should be @@ -266,27 +264,31 @@ def call(topic, msg): LOG.debug("MSG_ID is %s" % (msg_id)) conn = Connection.instance() - d = defer.Deferred() - consumer = DirectConsumer(connection=conn, msg_id=msg_id) - def deferred_receive(data, message): - """Acks message and callbacks or errbacks""" - message.ack() - if data['failure']: - return d.errback(RemoteError(*data['failure'])) - else: - return d.callback(data['result']) + class WaitMessage(object): - consumer.register_callback(deferred_receive) - injected = consumer.attach_to_tornado() + def __call__(self, data, message): + """Acks message and sets result.""" + message.ack() + if data['failure']: + self.result = RemoteError(*data['failure']) + else: + self.result = data['result'] - # clean up after the injected listened and return x - d.addCallback(lambda x: injected.stop() and x or x) + wait_msg = WaitMessage() + consumer = DirectConsumer(connection=conn, msg_id=msg_id) + consumer.register_callback(wait_msg) publisher = TopicPublisher(connection=conn, topic=topic) publisher.send(msg) publisher.close() - return d + + try: + consumer.wait(limit=1) + except StopIteration: + pass + consumer.close() + return wait_msg.result def cast(topic, msg): diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 2f22982eb..756ce519e 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -22,7 +22,6 @@ from M2Crypto import RSA import StringIO import time -from tornado import ioloop from twisted.internet import defer import unittest from xml.etree import ElementTree -- cgit From 378970b1495840a2a193dbecc3f9bb8701237744 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 23 Sep 2010 11:06:49 +0200 Subject: Compare project_id to '' using == (equality) rather than 'is' (identity). This is needed because '' isn't the same as u''. --- nova/auth/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index bc3a8a12e..2ec586419 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -266,7 +266,7 @@ class AuthManager(object): # NOTE(vish): if we stop using project name as id we need better # logic to find a default project for user - if project_id is '': + if project_id == '': project_id = user.name project = self.get_project(project_id) -- cgit From 08622cb48c200aa27e214fb14e47a741069b9bb0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 23 Sep 2010 04:24:54 -0500 Subject: All timestamps should be in UTC. Without this patch, the scheduler unit tests fail for anyone sufficiently East of Greenwich. --- nova/scheduler/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 2e6a5a835..c89d25a47 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -42,7 +42,8 @@ class Scheduler(object): def service_is_up(service): """Check whether a service is up based on last heartbeat.""" last_heartbeat = service['updated_at'] or service['created_at'] - elapsed = datetime.datetime.now() - last_heartbeat + # Timestamps in DB are UTC. + elapsed = datetime.datetime.utcnow() - last_heartbeat return elapsed < datetime.timedelta(seconds=FLAGS.service_down_time) def hosts_up(self, context, topic): -- cgit From d98c663d3e521d45586ed3922d93e0ca612a5639 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 23 Sep 2010 09:06:45 -0400 Subject: Added FLAGS.FAKE_subdomain letting you manually set the subdomain for testing on localhost. --- nova/api/__init__.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index a0be05d86..8e4d844b2 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -34,6 +34,8 @@ flags.DEFINE_string('rsapi_subdomain', 'rs', 'subdomain running the RS API') flags.DEFINE_string('ec2api_subdomain', 'ec2', 'subdomain running the EC2 API') +flags.DEFINE_string('FAKE_subdomain', None, + 'set to rs or ec2 to fake the subdomain of the host for testing') FLAGS = flags.FLAGS @@ -41,17 +43,26 @@ class API(wsgi.Router): """Routes top-level requests to the appropriate controller.""" def __init__(self): + rsdomain = {'sub_domain': [FLAGS.rsapi_subdomain]} + ec2domain = {'sub_domain': [FLAGS.ec2api_subdomain]} + # If someone wants to pretend they're hitting the RS subdomain + # on their local box, they can set FAKE_subdomain to 'rs', which + # removes subdomain restrictions from the RS routes below. + if FLAGS.FAKE_subdomain == 'rs': + rsdomain = {} + elif FLAGS.FAKE_subdomain == 'ec2': + ec2domain = {} mapper = routes.Mapper() mapper.sub_domains = True mapper.connect("/", controller=self.rsapi_versions, - conditions={'sub_domain': [FLAGS.rsapi_subdomain]}) + conditions=rsdomain) mapper.connect("/v1.0/{path_info:.*}", controller=rackspace.API(), - conditions={'sub_domain': [FLAGS.rsapi_subdomain]}) + conditions=rsdomain) mapper.connect("/", controller=self.ec2api_versions, - conditions={'sub_domain': [FLAGS.ec2api_subdomain]}) + conditions=ec2domain) mapper.connect("/services/{path_info:.*}", controller=ec2.API(), - conditions={'sub_domain': [FLAGS.ec2api_subdomain]}) + conditions=ec2domain) mrh = metadatarequesthandler.MetadataRequestHandler() for s in ['/latest', '/2009-04-04', @@ -64,7 +75,7 @@ class API(wsgi.Router): '/2007-01-19', '/1.0']: mapper.connect('%s/{path_info:.*}' % s, controller=mrh, - conditions={'subdomain': FLAGS.ec2api_subdomain}) + conditions=ec2domain) super(API, self).__init__(mapper) @webob.dec.wsgify -- cgit From 24f589d421be9a15ad941c34128b4fa0bdc28db4 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 23 Sep 2010 09:13:27 -0400 Subject: Apply vish's patch --- bin/nova-api-new | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/nova-api-new b/bin/nova-api-new index 8625c487f..6f25ad8c7 100755 --- a/bin/nova-api-new +++ b/bin/nova-api-new @@ -34,12 +34,11 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import api from nova import flags -from nova import utils from nova import wsgi FLAGS = flags.FLAGS flags.DEFINE_integer('api_port', 8773, 'API port') if __name__ == '__main__': - utils.default_flagfile() + FLAGS(sys.argv) wsgi.run_server(api.API(), FLAGS.api_port) -- cgit From ebf71b08efc6ab3c590f71715aa16b925f17c38e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 23 Sep 2010 15:47:29 +0200 Subject: Wrap WSGI container in server.serve to make it properly handle command line arguments as well as daemonise properly. Moved api and wsgi imports in the main() function to delay their inclusion until after python-daemon has closed all the file descriptors. Without this, eventlet's epoll fd gets opened before daemonize is called and thus its fd gets closed leading to very, very, very confusing errors. --- bin/nova-api-new | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/nova-api-new b/bin/nova-api-new index 6f25ad8c7..a5027700b 100755 --- a/bin/nova-api-new +++ b/bin/nova-api-new @@ -32,13 +32,18 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import api from nova import flags -from nova import wsgi +from nova import utils +from nova import server FLAGS = flags.FLAGS flags.DEFINE_integer('api_port', 8773, 'API port') -if __name__ == '__main__': - FLAGS(sys.argv) +def main(_args): + from nova import api + from nova import wsgi wsgi.run_server(api.API(), FLAGS.api_port) + +if __name__ == '__main__': + utils.default_flagfile() + server.serve('nova-api', main) -- cgit From 90669318581554a72890a6fd9c6837deb86c7e4c Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 23 Sep 2010 10:19:27 -0400 Subject: Spot-fix endpoint reference --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 824e00ac5..e9219c515 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -73,7 +73,7 @@ from nova import quota from nova import utils from nova.auth import manager from nova.cloudpipe import pipelib -from nova.endpoint import cloud +from nova.api.ec2 import cloud FLAGS = flags.FLAGS -- cgit From c9ac49b2425b932f60a87da80887d4556806ca60 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 23 Sep 2010 11:21:14 -0700 Subject: Fixed cloudpipe lib init. --- nova/api/cloudpipe/__init__.py | 1 + nova/cloudpipe/pipelib.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/api/cloudpipe/__init__.py b/nova/api/cloudpipe/__init__.py index 642f8ef6c..6d40990a8 100644 --- a/nova/api/cloudpipe/__init__.py +++ b/nova/api/cloudpipe/__init__.py @@ -51,6 +51,7 @@ class API(wsgi.Application): return webob.exc.HTTPNotFound() def get_project_id_from_ip(self, ip): + # TODO(eday): This was removed with the ORM branch, fix! instance = self.controller.get_instance_by_ip(ip) return instance['project_id'] diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index abc14bbb6..706a175d9 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -32,6 +32,7 @@ from nova import exception from nova import flags from nova import utils from nova.auth import manager +# TODO(eday): Eventually changes these to something not ec2-specific from nova.api.ec2 import cloud from nova.api.ec2 import context @@ -43,7 +44,7 @@ flags.DEFINE_string('boot_script_template', class CloudPipe(object): - def __init__(self, cloud_controller): + def __init__(self): self.controller = cloud.CloudController() self.manager = manager.AuthManager() -- cgit From 94b9d491d4f691f7ede3c0d5d8ca98288af1646f Mon Sep 17 00:00:00 2001 From: mdietz Date: Thu, 23 Sep 2010 18:28:49 +0000 Subject: Missed the model include, and fixed a broken test after the merge --- nova/db/sqlalchemy/models.py | 3 ++- nova/tests/api/rackspace/auth.py | 2 ++ nova/tests/api/rackspace/test_helper.py | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index bd1e9164e..6e1c0ce16 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -475,7 +475,8 @@ def register_models(): """Register Models and create metadata""" from sqlalchemy import create_engine models = (Service, Instance, Volume, ExportDevice, - FixedIp, FloatingIp, Network, NetworkIndex) # , Image, Host) + FixedIp, FloatingIp, Network, NetworkIndex, + AuthToken) # , Image, Host) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/tests/api/rackspace/auth.py b/nova/tests/api/rackspace/auth.py index 0f38ce79d..429c22ad2 100644 --- a/nova/tests/api/rackspace/auth.py +++ b/nova/tests/api/rackspace/auth.py @@ -15,6 +15,8 @@ class Test(unittest.TestCase): '__init__', test_helper.fake_auth_init) test_helper.FakeAuthManager.auth_data = {} test_helper.FakeAuthDatabase.data = {} + self.stubs.Set(nova.api.rackspace, 'RateLimitingMiddleware', + test_helper.FakeRateLimiter) def tearDown(self): self.stubs.UnsetAll() diff --git a/nova/tests/api/rackspace/test_helper.py b/nova/tests/api/rackspace/test_helper.py index 18d96d71e..be14e2de8 100644 --- a/nova/tests/api/rackspace/test_helper.py +++ b/nova/tests/api/rackspace/test_helper.py @@ -55,3 +55,11 @@ class FakeAuthManager(object): def get_user_from_access_key(self, key): return FakeAuthManager.auth_data.get(key, None) + +class FakeRateLimiter(object): + def __init__(self, application): + self.application = application + + @webob.dec.wsgify + def __call__(self, req): + return self.application -- cgit From 020f1a304c15db3086169efe67994ca59ca04e0c Mon Sep 17 00:00:00 2001 From: mdietz Date: Thu, 23 Sep 2010 18:29:17 +0000 Subject: Added a primary_key to AuthToken, fixed some unbound variables, and now all unit tests pass --- nova/api/rackspace/auth.py | 5 ++++- nova/db/sqlalchemy/models.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/api/rackspace/auth.py b/nova/api/rackspace/auth.py index 1ef90c324..ce5a967eb 100644 --- a/nova/api/rackspace/auth.py +++ b/nova/api/rackspace/auth.py @@ -8,6 +8,9 @@ from nova import flags from nova import auth from nova import manager from nova import db +from nova import utils + +FLAGS = flags.FLAGS class Context(object): pass @@ -15,7 +18,7 @@ class Context(object): class BasicApiAuthManager(object): """ Implements a somewhat rudimentary version of Rackspace Auth""" - def __init__(self): + def __init__(self, host=None, db_driver=None): if not host: host = FLAGS.host self.host = host diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 6e1c0ce16..161c5f1bc 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -401,7 +401,7 @@ class AuthToken(BASE, NovaBase): are a string representing the actual token and a user id for mapping to the actual user""" __tablename__ = 'auth_tokens' - token_hash = Column(String(255)) + token_hash = Column(String(255), primary_key=True) user_id = Column(Integer) server_manageent_url = Column(String(255)) storage_url = Column(String(255)) -- cgit From ca854c764a21985fd07becf7b0686f5d00125851 Mon Sep 17 00:00:00 2001 From: mdietz Date: Thu, 23 Sep 2010 18:29:40 +0000 Subject: Re-added the ramdisk line I accidentally removed --- nova/db/sqlalchemy/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 161c5f1bc..f6ba7953f 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -213,7 +213,8 @@ class Instance(BASE, NovaBase): image_id = Column(String(255)) kernel_id = Column(String(255)) - + ramdisk_id = Column(String(255)) + # image_id = Column(Integer, ForeignKey('images.id'), nullable=True) # kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) # ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) -- cgit From a6954efa3155868d31163236aa9e44f693f51b30 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 23 Sep 2010 11:56:44 -0700 Subject: Fixed rpc consumer to use unique return connection to prevent overlap. This could be reworked to share a connection, but it should be a wait operation and not a fast poll like it was before. We could also keep a cache of opened connections to be used between requests. --- nova/rpc.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 7e4d91a03..6363335ea 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -46,9 +46,9 @@ LOG.setLevel(logging.DEBUG) class Connection(carrot_connection.BrokerConnection): """Connection instance object""" @classmethod - def instance(cls): + def instance(cls, new=False): """Returns the instance""" - if not hasattr(cls, '_instance'): + if new or not hasattr(cls, '_instance'): params = dict(hostname=FLAGS.rabbit_host, port=FLAGS.rabbit_port, userid=FLAGS.rabbit_userid, @@ -60,7 +60,10 @@ class Connection(carrot_connection.BrokerConnection): # NOTE(vish): magic is fun! # pylint: disable-msg=W0142 - cls._instance = cls(**params) + if new: + return cls(**params) + else: + cls._instance = cls(**params) return cls._instance @classmethod @@ -263,8 +266,6 @@ def call(topic, msg): msg.update({'_msg_id': msg_id}) LOG.debug("MSG_ID is %s" % (msg_id)) - conn = Connection.instance() - class WaitMessage(object): def __call__(self, data, message): @@ -276,9 +277,11 @@ def call(topic, msg): self.result = data['result'] wait_msg = WaitMessage() + conn = Connection.instance(True) consumer = DirectConsumer(connection=conn, msg_id=msg_id) consumer.register_callback(wait_msg) + conn = Connection.instance() publisher = TopicPublisher(connection=conn, topic=topic) publisher.send(msg) publisher.close() -- cgit From 2b30ffe2f3c79e3701487d18fe1d4eef671aa335 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 23 Sep 2010 13:18:40 -0700 Subject: Applied vish's fixes. --- nova/api/ec2/metadatarequesthandler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 229e5a78d..08a8040ca 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -18,6 +18,8 @@ """Metadata request handler.""" +import logging + import webob.dec import webob.exc @@ -63,9 +65,9 @@ class MetadataRequestHandler(object): cc = cloud.CloudController() meta_data = cc.get_metadata(req.remote_addr) if meta_data is None: - _log.error('Failed to get metadata for ip: %s' % req.remote_addr) + logging.error('Failed to get metadata for ip: %s' % req.remote_addr) raise webob.exc.HTTPNotFound() - data = self.lookup(path, meta_data) + data = self.lookup(req.path_info, meta_data) if data is None: raise webob.exc.HTTPNotFound() return self.print_data(data) -- cgit From a70632890c610ece766bfd3c31eea4bc6eb4a316 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 23 Sep 2010 17:06:23 -0400 Subject: Apply vish's patch --- nova/api/ec2/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index b041787c2..f0aa57ee4 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -166,8 +166,8 @@ class Authorizer(wsgi.Middleware): 'ModifyImageAttribute': ['projectmanager', 'sysadmin'], }, 'AdminController': { - # All actions have the same permission: [] (the default) - # admins will be allowed to run them + # All actions have the same permission: ['none'] (the default) + # superusers will be allowed to run them # all others will get HTTPUnauthorized. }, } @@ -177,8 +177,7 @@ class Authorizer(wsgi.Middleware): context = req.environ['ec2.context'] controller_name = req.environ['ec2.controller'].__class__.__name__ action = req.environ['ec2.action'] - allowed_roles = self.action_roles[controller_name].get(action, []) - allowed_roles.extend(FLAGS.superuser_roles) + allowed_roles = self.action_roles[controller_name].get(action, ['none']) if self._matches_any_role(context, allowed_roles): return self.application else: @@ -186,6 +185,8 @@ class Authorizer(wsgi.Middleware): def _matches_any_role(self, context, roles): """Return True if any role in roles is allowed in context.""" + if context.user.is_superuser(): + return True if 'all' in roles: return True if 'none' in roles: -- cgit