From 0ed410621b3c2d621aa3fa52ca7ac46c6a5f0b70 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 23 May 2011 16:19:12 -0700 Subject: getting closer to working select call --- nova/compute/api.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index 785aff397..301c777bb 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -257,19 +257,21 @@ class API(base.Base): # we'll be ripping this whole for-loop out and deferring the # creation of the Instance record. At that point all this will # change. + filter_driver = 'nova.scheduler.host_filter.InstanceTypeFilter' + request_spec = { + 'instance_properties': base_options, + 'instance_type': instance_type, + 'filter_driver': filter_driver, + 'blob': zone_blob + } + LOG.debug(_("**** REQUEST SPEC: %(request_spec)s") % locals()) + rpc.cast(context, FLAGS.scheduler_topic, {"method": "run_instance", "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, - "request_spec": { - 'instance_properties': instance, - 'instance_type': instance_type, - 'filter_driver': - 'nova.scheduler.host_filter.' - 'InstanceTypeFilter', - 'blob': zone_blob - }, + "request_spec": request_spec, "availability_zone": availability_zone, "injected_files": injected_files}}) -- cgit From 48a3ec6e55f029578d5dc8ef7fe2e9fbe0de1b81 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 24 May 2011 12:05:46 -0700 Subject: more fix up --- nova/compute/api.py | 1 - 1 file changed, 1 deletion(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index 301c777bb..d66ee7920 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -264,7 +264,6 @@ class API(base.Base): 'filter_driver': filter_driver, 'blob': zone_blob } - LOG.debug(_("**** REQUEST SPEC: %(request_spec)s") % locals()) rpc.cast(context, FLAGS.scheduler_topic, -- cgit From 660d1802a6c202465af585a059930113de5ae646 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 25 May 2011 13:20:34 -0700 Subject: starting breakdown of nova.compute.api.create() --- nova/compute/api.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 7 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index 57f8c421a..1d75dbc80 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -54,6 +54,7 @@ def generate_default_hostname(instance_id): class API(base.Base): """API for interacting with the compute manager.""" + scheduler_rules = None def __init__(self, image_service=None, network_api=None, volume_api=None, hostname_factory=generate_default_hostname, @@ -128,18 +129,15 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - def create(self, context, instance_type, + def _check_create_parameters(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, injected_files=None, zone_blob=None): - """Create the number and type of instances requested. - - Verifies that quota and other arguments are valid. - - """ + """Verify all the input parameters regardless of the provisioning + strategy being performed.""" if not instance_type: instance_type = instance_types.get_default_instance_type() @@ -220,7 +218,46 @@ class API(base.Base): 'metadata': metadata, 'availability_zone': availability_zone, 'os_type': os_type} - elevated = context.elevated() + + return (num_instances, base_options) + + def create_all_at_once(self, context, instance_type, + image_id, kernel_id=None, ramdisk_id=None, + min_count=1, max_count=1, + display_name='', display_description='', + key_name=None, key_data=None, security_group='default', + availability_zone=None, user_data=None, metadata={}, + injected_files=None, zone_blob=None): + """Provision the instances by passing the whole request to + the Scheduler for execution.""" + self._check_create_parameters(self, context, instance_type, + image_id, kernel_id, ramdisk_id, min_count=1, max_count=1, + display_name, display_description, + key_name, key_data, security_group, + availability_zone, user_data, metadata, + injected_files, zone_blob) + + def create(self, context, instance_type, + image_id, kernel_id=None, ramdisk_id=None, + min_count=1, max_count=1, + display_name='', display_description='', + key_name=None, key_data=None, security_group='default', + availability_zone=None, user_data=None, metadata={}, + injected_files=None, zone_blob=None): + """Provision the instances by sending off a series of single + instance requests to the Schedulers. This is fine for trival + Scheduler drivers, but may remove the effectiveness of the + more complicated drivers.""" + + num_instances, base_options = self._check_create_parameters( + context, instance_type, + image_id, kernel_id, ramdisk_id, + min_count=1, max_count=1, + display_name, display_description, + key_name, key_data, security_group, + availability_zone, user_data, metadata, + injected_files, zone_blob) + instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): @@ -279,6 +316,22 @@ class API(base.Base): return [dict(x.iteritems()) for x in instances] + def smart_create(self, *args, **kwargs): + """Ask the scheduler if we should: 1. do single shot instance + requests or all-at-once, and 2. defer the DB work until + a suitable host has been selected (if at all). Cache this + information and act accordingly.""" + + if API.scheduler_rules == None: + API.scheduler_rules = scheduler_api.get_scheduler_rules(context) + + should_create_all_at_once, should_defer_database_create = \ + API.scheduler_rules + + if should_create_all_at_once: + return self.create_all_at_once(*args, **kwargs) + return self.create(*args, **kwargs) + def has_finished_migration(self, context, instance_id): """Returns true if an instance has a finished migration.""" try: -- cgit From bc176751de7f55d22d1bb04552bbff9c496979ed Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 26 May 2011 08:28:57 -0700 Subject: refactoring compute.api.create() --- nova/compute/api.py | 193 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 118 insertions(+), 75 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index 1d75dbc80..cd4d7ca47 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -54,7 +54,11 @@ def generate_default_hostname(instance_id): class API(base.Base): """API for interacting with the compute manager.""" - scheduler_rules = None + + # Should we create instances all-at-once or as single-shot requests. + # Different schedulers use different approaches. + # This is cached across all API instances. + should_create_all_at_once = None # None implies uninitialized. def __init__(self, image_service=None, network_api=None, volume_api=None, hostname_factory=generate_default_hostname, @@ -219,8 +223,74 @@ class API(base.Base): 'availability_zone': availability_zone, 'os_type': os_type} - return (num_instances, base_options) + return (num_instances, base_options, security_groups) + + def create_db_entry_for_new_instance(self, context, base_options, + security_groups, num=1): + """Create an entry in the DB for this new instance, + including any related table updates (such as security + groups, MAC address, etc). This will called by create() + in the majority of situations, but all-at-once style + Schedulers may initiate the call.""" + instance = dict(mac_address=utils.generate_mac(), + launch_index=num, + **base_options) + instance = self.db.instance_create(context, instance) + instance_id = instance['id'] + + elevated = context.elevated() + if not security_groups: + security_groups = [] + for security_group_id in security_groups: + self.db.instance_add_security_group(elevated, + instance_id, + security_group_id) + + # Set sane defaults if not specified + updates = dict(hostname=self.hostname_factory(instance_id)) + if (not hasattr(instance, 'display_name') or + instance.display_name is None): + updates['display_name'] = "Server %s" % instance_id + + instance = self.update(context, instance_id, **updates) + + for group_id in security_groups: + self.trigger_security_group_members_refresh(elevated, group_id) + + return instance + def _ask_scheduler_to_create_instance(self, context, base_options, + instance_type, zone_blob, + availability_zone, injected_files, + instance_id=None, num_instances=1): + """Send the run_instance request to the schedulers for processing.""" + pid = context.project_id + uid = context.user_id + if instance_id: + LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's" + " instance %(instance_id)s (single-shot)") % locals()) + else: + LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's" + " (all-at-once)") % locals()) + + filter_class = 'nova.scheduler.host_filter.InstanceTypeFilter' + request_spec = { + 'instance_properties': base_options, + 'instance_type': instance_type, + 'filter': filter_class, + 'blob': zone_blob, + 'num_instances': num_instances + } + + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "run_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id, + "request_spec": request_spec, + "availability_zone": availability_zone, + "injected_files": injected_files}}) + def create_all_at_once(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, @@ -229,13 +299,24 @@ class API(base.Base): availability_zone=None, user_data=None, metadata={}, injected_files=None, zone_blob=None): """Provision the instances by passing the whole request to - the Scheduler for execution.""" - self._check_create_parameters(self, context, instance_type, - image_id, kernel_id, ramdisk_id, min_count=1, max_count=1, - display_name, display_description, - key_name, key_data, security_group, - availability_zone, user_data, metadata, - injected_files, zone_blob) + the Scheduler for execution. Returns a Reservation ID + related to the creation of all of these instances.""" + num_instances, base_options, security_groups = \ + self._check_create_parameters( + context, instance_type, + image_id, kernel_id, ramdisk_id, + min_count, max_count, + display_name, display_description, + key_name, key_data, security_group, + availability_zone, user_data, metadata, + injected_files, zone_blob) + + self._ask_scheduler_to_create_instance(context, base_options, + instance_type, zone_blob, + availability_zone, injected_files, + num_instances=num_instances) + + return base_options['reservation_id'] def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, @@ -244,15 +325,20 @@ class API(base.Base): key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, injected_files=None, zone_blob=None): - """Provision the instances by sending off a series of single + """ + Provision the instances by sending off a series of single instance requests to the Schedulers. This is fine for trival Scheduler drivers, but may remove the effectiveness of the - more complicated drivers.""" + more complicated drivers. + + Returns a list of instance dicts. + """ - num_instances, base_options = self._check_create_parameters( + num_instances, base_options, security_groups = \ + self._check_create_parameters( context, instance_type, image_id, kernel_id, ramdisk_id, - min_count=1, max_count=1, + min_count, max_count, display_name, display_description, key_name, key_data, security_group, availability_zone, user_data, metadata, @@ -261,74 +347,31 @@ class API(base.Base): instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): - instance = dict(mac_address=utils.generate_mac(), - launch_index=num, - **base_options) - instance = self.db.instance_create(context, instance) - instance_id = instance['id'] - - elevated = context.elevated() - if not security_groups: - security_groups = [] - for security_group_id in security_groups: - self.db.instance_add_security_group(elevated, - instance_id, - security_group_id) - - # Set sane defaults if not specified - updates = dict(hostname=self.hostname_factory(instance_id)) - if (not hasattr(instance, 'display_name') or - instance.display_name is None): - updates['display_name'] = "Server %s" % instance_id - - instance = self.update(context, instance_id, **updates) + instance = self.create_db_entry_for_new_instance(context, + base_options, security_groups, num=num) instances.append(instance) - - pid = context.project_id - uid = context.user_id - LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's" - " instance %(instance_id)s") % locals()) - - # NOTE(sandy): For now we're just going to pass in the - # instance_type record to the scheduler. In a later phase - # we'll be ripping this whole for-loop out and deferring the - # creation of the Instance record. At that point all this will - # change. - filter_driver = 'nova.scheduler.host_filter.InstanceTypeFilter' - request_spec = { - 'instance_properties': base_options, - 'instance_type': instance_type, - 'filter_driver': filter_driver, - 'blob': zone_blob - } - - rpc.cast(context, - FLAGS.scheduler_topic, - {"method": "run_instance", - "args": {"topic": FLAGS.compute_topic, - "instance_id": instance_id, - "request_spec": request_spec, - "availability_zone": availability_zone, - "injected_files": injected_files}}) - - for group_id in security_groups: - self.trigger_security_group_members_refresh(elevated, group_id) - - return [dict(x.iteritems()) for x in instances] + instance_id = instance['id'] + + self._ask_scheduler_to_create_instance(context, base_options, + instance_type, zone_blob, + availability_zone, injected_files, + instance_id=instance_id) + + return [x.items() for x in instances] def smart_create(self, *args, **kwargs): - """Ask the scheduler if we should: 1. do single shot instance - requests or all-at-once, and 2. defer the DB work until - a suitable host has been selected (if at all). Cache this - information and act accordingly.""" + """ + Ask the scheduler if we should do single shot instance requests + or all-at-once. - if API.scheduler_rules == None: - API.scheduler_rules = scheduler_api.get_scheduler_rules(context) + Cache this information on first request and act accordingly. + """ - should_create_all_at_once, should_defer_database_create = \ - API.scheduler_rules + if API.should_create_all_at_once == None: + API.should_create_all_at_once = \ + scheduler_api.should_create_all_at_once(context) - if should_create_all_at_once: + if API.should_create_all_at_once: return self.create_all_at_once(*args, **kwargs) return self.create(*args, **kwargs) -- cgit From 9a9dc80bcb47db5864b0c35fe1dd1a636b0a933e Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 26 May 2011 11:34:53 -0700 Subject: tests pass and pep8'ed --- nova/compute/api.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index cd4d7ca47..032ef7469 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -225,9 +225,9 @@ class API(base.Base): return (num_instances, base_options, security_groups) - def create_db_entry_for_new_instance(self, context, base_options, + def create_db_entry_for_new_instance(self, context, base_options, security_groups, num=1): - """Create an entry in the DB for this new instance, + """Create an entry in the DB for this new instance, including any related table updates (such as security groups, MAC address, etc). This will called by create() in the majority of situations, but all-at-once style @@ -272,7 +272,7 @@ class API(base.Base): else: LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's" " (all-at-once)") % locals()) - + filter_class = 'nova.scheduler.host_filter.InstanceTypeFilter' request_spec = { 'instance_properties': base_options, @@ -290,7 +290,7 @@ class API(base.Base): "request_spec": request_spec, "availability_zone": availability_zone, "injected_files": injected_files}}) - + def create_all_at_once(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, @@ -298,10 +298,10 @@ class API(base.Base): key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, injected_files=None, zone_blob=None): - """Provision the instances by passing the whole request to - the Scheduler for execution. Returns a Reservation ID - related to the creation of all of these instances.""" - num_instances, base_options, security_groups = \ + """Provision the instances by passing the whole request to + the Scheduler for execution. Returns a Reservation ID + related to the creation of all of these instances.""" + num_instances, base_options, security_groups = \ self._check_create_parameters( context, instance_type, image_id, kernel_id, ramdisk_id, @@ -311,12 +311,12 @@ class API(base.Base): availability_zone, user_data, metadata, injected_files, zone_blob) - self._ask_scheduler_to_create_instance(context, base_options, + self._ask_scheduler_to_create_instance(context, base_options, instance_type, zone_blob, availability_zone, injected_files, num_instances=num_instances) - return base_options['reservation_id'] + return base_options['reservation_id'] def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, @@ -330,7 +330,7 @@ class API(base.Base): instance requests to the Schedulers. This is fine for trival Scheduler drivers, but may remove the effectiveness of the more complicated drivers. - + Returns a list of instance dicts. """ @@ -343,7 +343,7 @@ class API(base.Base): key_name, key_data, security_group, availability_zone, user_data, metadata, injected_files, zone_blob) - + instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): @@ -351,13 +351,13 @@ class API(base.Base): base_options, security_groups, num=num) instances.append(instance) instance_id = instance['id'] - + self._ask_scheduler_to_create_instance(context, base_options, instance_type, zone_blob, availability_zone, injected_files, instance_id=instance_id) - - return [x.items() for x in instances] + + return [dict(x.iteritems()) for x in instances] def smart_create(self, *args, **kwargs): """ -- cgit From 727317333978ac5cf0fb1cd3f86e49e9868f1e19 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 6 Jun 2011 17:58:40 -0700 Subject: fixed up tests after trunk merge --- nova/compute/api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index 24f04f226..e09127d5c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -134,7 +134,7 @@ class API(base.Base): raise quota.QuotaError(msg, "MetadataLimitExceeded") def _check_create_parameters(self, context, instance_type, - image_id, kernel_id=None, ramdisk_id=None, + image_href, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', @@ -300,7 +300,7 @@ class API(base.Base): "injected_files": injected_files}}) def create_all_at_once(self, context, instance_type, - image_id, kernel_id=None, ramdisk_id=None, + image_href, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', @@ -312,7 +312,7 @@ class API(base.Base): num_instances, base_options, security_groups = \ self._check_create_parameters( context, instance_type, - image_id, kernel_id, ramdisk_id, + image_href, kernel_id, ramdisk_id, min_count, max_count, display_name, display_description, key_name, key_data, security_group, @@ -328,7 +328,7 @@ class API(base.Base): return base_options['reservation_id'] def create(self, context, instance_type, - image_id, kernel_id=None, ramdisk_id=None, + image_href, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', @@ -346,7 +346,7 @@ class API(base.Base): num_instances, base_options, security_groups = \ self._check_create_parameters( context, instance_type, - image_id, kernel_id, ramdisk_id, + image_href, kernel_id, ramdisk_id, min_count, max_count, display_name, display_description, key_name, key_data, security_group, -- cgit From dcb0d38aa829e1e2492defffaf6ad393b809289b Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 8 Jun 2011 08:13:23 -0700 Subject: removed straggler code --- nova/compute/api.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'nova/compute') diff --git a/nova/compute/api.py b/nova/compute/api.py index e09127d5c..b0949a729 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -55,11 +55,6 @@ def generate_default_hostname(instance_id): class API(base.Base): """API for interacting with the compute manager.""" - # Should we create instances all-at-once or as single-shot requests. - # Different schedulers use different approaches. - # This is cached across all API instances. - should_create_all_at_once = None # None implies uninitialized. - def __init__(self, image_service=None, network_api=None, volume_api=None, hostname_factory=generate_default_hostname, **kwargs): -- cgit