From 04b50db56ee90c0f4dd685a8f45883522260164f Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 11 Jul 2011 14:27:01 -0700 Subject: Replace 'like' support with 'regexp' matching done in python. Since 'like' would result in a full table scan anyway, this is a bit more flexible. Make search options and matching a little more generic Return 404 when --fixed_ip doesn't match any instance, instead of a 500 only when the IP isn't in the FixedIps table. --- nova/db/api.py | 44 ++++++-- nova/db/sqlalchemy/api.py | 239 ++++++++++++++++++++++++++++++++----------- nova/db/sqlalchemy/models.py | 1 + 3 files changed, 214 insertions(+), 70 deletions(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index b7c5700e5..c0f49d98d 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -381,15 +381,6 @@ def fixed_ip_get_by_virtual_interface(context, vif_id): return IMPL.fixed_ip_get_by_virtual_interface(context, vif_id) -def fixed_ip_get_instance(context, address): - """Get an instance for a fixed ip by address.""" - return IMPL.fixed_ip_get_instance(context, address) - - -def fixed_ip_get_instance_v6(context, address): - return IMPL.fixed_ip_get_instance_v6(context, address) - - def fixed_ip_get_network(context, address): """Get a network for a fixed ip by address.""" return IMPL.fixed_ip_get_network(context, address) @@ -515,10 +506,43 @@ def instance_get_all_by_host(context, host): def instance_get_all_by_reservation(context, reservation_id): - """Get all instance belonging to a reservation.""" + """Get all instances belonging to a reservation.""" return IMPL.instance_get_all_by_reservation(context, reservation_id) +def instance_get_by_fixed_ip(context, address): + """Get an instance for a fixed ip by address.""" + return IMPL.instance_get_by_fixed_ip(context, address) + + +def instance_get_by_fixed_ipv6(context, address): + """Get an instance for a fixed ip by IPv6 address.""" + return IMPL.instance_get_by_fixed_ipv6(context, address) + + +def instance_get_all_by_column_regexp(context, column, column_regexp): + """Get all instances by using regular expression matching against + a particular DB column + """ + return IMPL.instance_get_all_by_column_regexp(context, + column, + column_regexp) + + +def instance_get_all_by_ip_regexp(context, ip_regexp): + """Get all instances by using regular expression matching against + Floating and Fixed IP Addresses + """ + return IMPL.instance_get_all_by_ip_regexp(context, ip_regexp) + + +def instance_get_all_by_ipv6_regexp(context, ipv6_regexp): + """Get all instances by using regular expression matching against + IPv6 Addresses + """ + return IMPL.instance_get_all_by_ipv6_regexp(context, ipv6_regexp) + + def instance_get_fixed_addresses(context, instance_id): """Get the fixed ip address of an instance.""" return IMPL.instance_get_fixed_addresses(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index ffd009513..af2acbcb3 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -18,6 +18,7 @@ """ Implementation of SQLAlchemy backend. """ +import re import traceback import warnings @@ -797,28 +798,6 @@ def fixed_ip_get_by_virtual_interface(context, vif_id): return rv -@require_context -def fixed_ip_get_instance(context, address): - fixed_ip_ref = fixed_ip_get_by_address(context, address) - return fixed_ip_ref.instance - - -@require_context -def fixed_ip_get_instance_v6(context, address): - session = get_session() - - # convert IPv6 address to mac - mac = ipv6.to_mac(address) - - # get virtual interface - vif_ref = virtual_interface_get_by_address(context, mac) - - # look up instance based on instance_id from vif row - result = session.query(models.Instance).\ - filter_by(id=vif_ref['instance_id']) - return result - - @require_admin_context def fixed_ip_get_network(context, address): fixed_ip_ref = fixed_ip_get_by_address(context, address) @@ -1204,30 +1183,166 @@ def instance_get_all_by_project(context, project_id): @require_context def instance_get_all_by_reservation(context, reservation_id): session = get_session() + query = session.query(models.Instance).\ + filter_by(reservation_id=reservation_id).\ + options(joinedload_all('fixed_ips.floating_ips')).\ + options(joinedload('virtual_interfaces')).\ + options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ips.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')) if is_admin_context(context): - return session.query(models.Instance).\ - options(joinedload_all('fixed_ips.floating_ips')).\ - options(joinedload('virtual_interfaces')).\ - options(joinedload('security_groups')).\ - options(joinedload_all('fixed_ips.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(reservation_id=reservation_id).\ - filter_by(deleted=can_read_deleted(context)).\ - all() + return query.\ + filter_by(deleted=can_read_deleted(context)).\ + all() elif is_user_context(context): - return session.query(models.Instance).\ - options(joinedload_all('fixed_ips.floating_ips')).\ - options(joinedload('virtual_interfaces')).\ - options(joinedload('security_groups')).\ - options(joinedload_all('fixed_ips.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(project_id=context.project_id).\ - filter_by(reservation_id=reservation_id).\ - filter_by(deleted=False).\ - all() + return query.\ + filter_by(project_id=context.project_id).\ + filter_by(deleted=False).\ + all() + + +@require_context +def instance_get_by_fixed_ip(context, address): + fixed_ip_ref = fixed_ip_get_by_address(context, address) + return fixed_ip_ref.instance + + +@require_context +def instance_get_by_fixed_ipv6(context, address): + session = get_session() + + # convert IPv6 address to mac + mac = ipv6.to_mac(address) + + # get virtual interface + vif_ref = virtual_interface_get_by_address(context, mac) + + # look up instance based on instance_id from vif row + result = session.query(models.Instance).\ + filter_by(id=vif_ref['instance_id']) + return result + + +@require_context +def instance_get_all_by_column_regexp(context, column, column_regexp): + """Get all instances by using regular expression matching against + a particular DB column + """ + session = get_session() + + # MySQL 'regexp' is not portable, so we must do our own matching. + # First... grab all Instances. + query = session.query(models.Instance).\ + options(joinedload('metadata')) + if is_admin_context(context): + all_instances = query.\ + filter_by(deleted=can_read_deleted(context)).\ + all() + elif is_user_context(context): + all_instances = query.\ + filter_by(project_id=context.project_id).\ + filter_by(deleted=False).\ + all() + else: + return [] + + if all_instances is None: + all_instances = [] + + # Now do the regexp matching + compiled_regexp = re.compile(column_regexp) + instances = [] + + for instance in all_instances: + v = getattr(instance, column) + if v and compiled_regexp.match(v): + instances.append(instance) + return instances + + +@require_context +def instance_get_all_by_ip_regexp(context, ip_regexp): + """Get all instances by using regular expression matching against + Floating and Fixed IP Addresses + """ + session = get_session() + + fixed_ip_query = session.query(models.FixedIp).\ + options(joinedload('instance.metadata')) + floating_ip_query = session.query(models.FloatingIp).\ + options(joinedload_all('fixed_ip.instance.metadata')) + + # Query both FixedIp and FloatingIp tables to get matches. + # Since someone could theoretically search for something that matches + # instances in both tables... we need to use a dictionary keyed + # on instance ID to make sure we return only 1. We can't key off + # of 'instance' because it's just a reference and will be different + # addresses even though they might point to the same instance ID. + instances = {} + + # MySQL 'regexp' is not portable, so we must do our own matching. + # First... grab all of the IP entries. + if is_admin_context(context): + fixed_ips = fixed_ip_query.\ + filter_by(deleted=can_read_deleted(context)).\ + all() + floating_ips = floating_ip_query.\ + filter_by(deleted=can_read_deleted(context)).\ + all() + elif is_user_context(context): + fixed_ips = fixed_ip_query.filter_by(deleted=False).all() + floating_ips = floating_ip_query.filter_by(deleted=False).all() + else: + return None + + if fixed_ips is None: + fixed_ips = [] + if floating_ips is None: + floating_ips = [] + + compiled_regexp = re.compile(ip_regexp) + instances = {} + + # Now do the regexp matching + for fixed_ip in fixed_ips: + if fixed_ip.instance and compiled_regexp.match(fixed_ip.address): + instances[fixed_ip.instance.uuid] = fixed_ip.instance + for floating_ip in floating_ips: + fixed_ip = floating_ip.fixed_ip + if fixed_ip and fixed_ip.instance and\ + compiled_regexp.match(floating_ip.address): + instances[fixed_ip.instance.uuid] = fixed_ip.instance + + return instances.values() + +@require_context +def instance_get_all_by_ipv6_regex(context, ipv6_regexp): + """Get all instances by using regular expression matching against + IPv6 Addresses + """ + + session = get_session() + with session.begin(): + # get instances + + all_instances = session.query(models.Instance).\ + options(joinedload('metadata')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + if not all_instances: + return [] + + instances = [] + compiled_regexp = re.compile(ipv6_regexp) + for instance in all_instances: + ipv6_addrs = _ipv6_get_by_instance_ref(context, instance) + for ipv6 in ipv6_addrs: + if compiled_regexp.match(ipv6): + instances.append(instance) + break + return instances @require_admin_context @@ -1258,29 +1373,33 @@ def instance_get_fixed_addresses(context, instance_id): return [fixed_ip.address for fixed_ip in fixed_ips] +def _ipv6_get_by_instance_ref(context, instance_ref): + # assume instance has 1 mac for each network associated with it + # get networks associated with instance + network_refs = network_get_all_by_instance(context, instance_id) + # compile a list of cidr_v6 prefixes sorted by network id + prefixes = [ref.cidr_v6 for ref in + sorted(network_refs, key=lambda ref: ref.id)] + # get vifs associated with instance + vif_refs = virtual_interface_get_by_instance(context, instance_ref.id) + # compile list of the mac_addresses for vifs sorted by network id + macs = [vif_ref['address'] for vif_ref in + sorted(vif_refs, key=lambda vif_ref: vif_ref['network_id'])] + # get project id from instance + project_id = instance_ref.project_id + # combine prefixes, macs, and project_id into (prefix,mac,p_id) tuples + prefix_mac_tuples = zip(prefixes, macs, [project_id for m in macs]) + # return list containing ipv6 address for each tuple + return [ipv6.to_global_ipv6(*t) for t in prefix_mac_tuples] + + @require_context def instance_get_fixed_addresses_v6(context, instance_id): session = get_session() with session.begin(): # get instance instance_ref = instance_get(context, instance_id, session=session) - # assume instance has 1 mac for each network associated with it - # get networks associated with instance - network_refs = network_get_all_by_instance(context, instance_id) - # compile a list of cidr_v6 prefixes sorted by network id - prefixes = [ref.cidr_v6 for ref in - sorted(network_refs, key=lambda ref: ref.id)] - # get vifs associated with instance - vif_refs = virtual_interface_get_by_instance(context, instance_ref.id) - # compile list of the mac_addresses for vifs sorted by network id - macs = [vif_ref['address'] for vif_ref in - sorted(vif_refs, key=lambda vif_ref: vif_ref['network_id'])] - # get project id from instance - project_id = instance_ref.project_id - # combine prefixes, macs, and project_id into (prefix,mac,p_id) tuples - prefix_mac_tuples = zip(prefixes, macs, [project_id for m in macs]) - # return list containing ipv6 address for each tuple - return [ipv6.to_global_ipv6(*t) for t in prefix_mac_tuples] + return _ipv6_get_by_instance_ref(context, instance_ref) @require_context diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index d29d3d6f1..e42da193f 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -187,6 +187,7 @@ class Instance(BASE, NovaBase): image_ref = Column(String(255)) kernel_id = Column(String(255)) ramdisk_id = Column(String(255)) + server_name = Column(String(255)) # image_ref = Column(Integer, ForeignKey('images.id'), nullable=True) # kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) -- cgit From 04804aba3c995260cf376b8d979f032942cd0988 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 11 Jul 2011 14:45:27 -0700 Subject: pep8 fixes --- nova/db/sqlalchemy/api.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index af2acbcb3..99e96e679 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1317,6 +1317,7 @@ def instance_get_all_by_ip_regexp(context, ip_regexp): return instances.values() + @require_context def instance_get_all_by_ipv6_regex(context, ipv6_regexp): """Get all instances by using regular expression matching against -- cgit From a3096d593fbe21625e3c4102e69d12950e9d2ef2 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Tue, 12 Jul 2011 02:01:09 -0700 Subject: added searching by instance name added unit tests --- nova/db/api.py | 7 +++++++ nova/db/sqlalchemy/api.py | 24 +++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index c0f49d98d..6aa44f06b 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -529,6 +529,13 @@ def instance_get_all_by_column_regexp(context, column, column_regexp): column_regexp) +def instance_get_all_by_name_regexp(context, name_regexp): + """Get all instances by using regular expression matching against + its name + """ + return IMPL.instance_get_all_by_name_regexp(context, name_regexp) + + def instance_get_all_by_ip_regexp(context, ip_regexp): """Get all instances by using regular expression matching against Floating and Fixed IP Addresses diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 99e96e679..93614a307 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1262,6 +1262,28 @@ def instance_get_all_by_column_regexp(context, column, column_regexp): return instances +@require_context +def instance_get_all_by_name_regexp(context, ipv6_regexp): + """Get all instances by using regular expression matching against + its name + """ + + session = get_session() + with session.begin(): + # get instances + + all_instances = session.query(models.Instance).\ + options(joinedload('metadata')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + if not all_instances: + return [] + + compiled_regexp = re.compile(ipv6_regexp) + return [instance for instance in all_instances + if compiled_regexp.match(instance.name)] + + @require_context def instance_get_all_by_ip_regexp(context, ip_regexp): """Get all instances by using regular expression matching against @@ -1319,7 +1341,7 @@ def instance_get_all_by_ip_regexp(context, ip_regexp): @require_context -def instance_get_all_by_ipv6_regex(context, ipv6_regexp): +def instance_get_all_by_ipv6_regexp(context, ipv6_regexp): """Get all instances by using regular expression matching against IPv6 Addresses """ -- cgit From edccef06c24df2fa785005f7a3c1f52a45bfc071 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Tue, 12 Jul 2011 03:13:43 -0700 Subject: fix bugs with fixed_ip returning a 404 instance searching needs to joinload more stuff --- nova/db/sqlalchemy/api.py | 102 +++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 50 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 93614a307..59db56a5c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1234,27 +1234,20 @@ def instance_get_all_by_column_regexp(context, column, column_regexp): # MySQL 'regexp' is not portable, so we must do our own matching. # First... grab all Instances. - query = session.query(models.Instance).\ - options(joinedload('metadata')) - if is_admin_context(context): - all_instances = query.\ - filter_by(deleted=can_read_deleted(context)).\ - all() - elif is_user_context(context): - all_instances = query.\ - filter_by(project_id=context.project_id).\ - filter_by(deleted=False).\ - all() - else: + all_instances = session.query(models.Instance).\ + options(joinedload_all('fixed_ips.floating_ips')).\ + options(joinedload('virtual_interfaces')).\ + options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ips.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + if not all_instances: return [] - - if all_instances is None: - all_instances = [] - # Now do the regexp matching compiled_regexp = re.compile(column_regexp) instances = [] - for instance in all_instances: v = getattr(instance, column) if v and compiled_regexp.match(v): @@ -1269,19 +1262,24 @@ def instance_get_all_by_name_regexp(context, ipv6_regexp): """ session = get_session() - with session.begin(): - # get instances - - all_instances = session.query(models.Instance).\ - options(joinedload('metadata')).\ - filter_by(deleted=can_read_deleted(context)).\ - all() - if not all_instances: - return [] - compiled_regexp = re.compile(ipv6_regexp) - return [instance for instance in all_instances - if compiled_regexp.match(instance.name)] + # MySQL 'regexp' is not portable, so we must do our own matching. + # First... grab all Instances. + all_instances = session.query(models.Instance).\ + options(joinedload_all('fixed_ips.floating_ips')).\ + options(joinedload('virtual_interfaces')).\ + options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ips.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + if not all_instances: + return [] + # Now do the regexp matching + compiled_regexp = re.compile(ipv6_regexp) + return [instance for instance in all_instances + if compiled_regexp.match(instance.name)] @require_context @@ -1291,11 +1289,6 @@ def instance_get_all_by_ip_regexp(context, ip_regexp): """ session = get_session() - fixed_ip_query = session.query(models.FixedIp).\ - options(joinedload('instance.metadata')) - floating_ip_query = session.query(models.FloatingIp).\ - options(joinedload_all('fixed_ip.instance.metadata')) - # Query both FixedIp and FloatingIp tables to get matches. # Since someone could theoretically search for something that matches # instances in both tables... we need to use a dictionary keyed @@ -1304,20 +1297,26 @@ def instance_get_all_by_ip_regexp(context, ip_regexp): # addresses even though they might point to the same instance ID. instances = {} - # MySQL 'regexp' is not portable, so we must do our own matching. - # First... grab all of the IP entries. - if is_admin_context(context): - fixed_ips = fixed_ip_query.\ - filter_by(deleted=can_read_deleted(context)).\ - all() - floating_ips = floating_ip_query.\ - filter_by(deleted=can_read_deleted(context)).\ - all() - elif is_user_context(context): - fixed_ips = fixed_ip_query.filter_by(deleted=False).all() - floating_ips = floating_ip_query.filter_by(deleted=False).all() - else: - return None + fixed_ips = session.query(models.FixedIp).\ + options(joinedload_all('instance.fixed_ips.floating_ips')).\ + options(joinedload('instance.virtual_interfaces')).\ + options(joinedload('instance.security_groups')).\ + options(joinedload_all('instance.fixed_ips.network')).\ + options(joinedload('instance.metadata')).\ + options(joinedload('instance.instance_type')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + floating_ips = session.query(models.FloatingIp).\ + options(joinedload_all( + 'fixed_ip.instance.fixed_ips.floating_ips')).\ + options(joinedload('fixed_ip.instance.virtual_interfaces')).\ + options(joinedload('fixed_ip.instance.security_groups')).\ + options(joinedload_all( + 'fixed_ip.instance.fixed_ips.network')).\ + options(joinedload('fixed_ip.instance.metadata')).\ + options(joinedload('fixed_ip.instance.instance_type')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() if fixed_ips is None: fixed_ips = [] @@ -1348,10 +1347,13 @@ def instance_get_all_by_ipv6_regexp(context, ipv6_regexp): session = get_session() with session.begin(): - # get instances - all_instances = session.query(models.Instance).\ + options(joinedload_all('fixed_ips.floating_ips')).\ + options(joinedload('virtual_interfaces')).\ + options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ips.network')).\ options(joinedload('metadata')).\ + options(joinedload('instance_type')).\ filter_by(deleted=can_read_deleted(context)).\ all() if not all_instances: -- cgit From 1dec3d7c3380d83398be0588b58c1cad13252807 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 14 Jul 2011 12:13:13 -0700 Subject: clean up checking for exclusive search options fix a cut n paste error with instance_get_all_by_name_regexp --- nova/db/sqlalchemy/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 59db56a5c..735d63be9 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1256,7 +1256,7 @@ def instance_get_all_by_column_regexp(context, column, column_regexp): @require_context -def instance_get_all_by_name_regexp(context, ipv6_regexp): +def instance_get_all_by_name_regexp(context, name_regexp): """Get all instances by using regular expression matching against its name """ @@ -1277,7 +1277,7 @@ def instance_get_all_by_name_regexp(context, ipv6_regexp): if not all_instances: return [] # Now do the regexp matching - compiled_regexp = re.compile(ipv6_regexp) + compiled_regexp = re.compile(name_regexp) return [instance for instance in all_instances if compiled_regexp.match(instance.name)] -- cgit From 491c90924ac87e533ce61e3bf949a50bfdd6a31d Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Sun, 17 Jul 2011 16:35:11 -0700 Subject: compute's get_all should accept 'name' not 'display_name' for searching Instance.display_name. Removed 'server_name' searching.. Fixed DB calls for searching to filter results based on context --- nova/db/sqlalchemy/api.py | 58 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 735d63be9..feccba389 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1234,15 +1234,25 @@ def instance_get_all_by_column_regexp(context, column, column_regexp): # MySQL 'regexp' is not portable, so we must do our own matching. # First... grab all Instances. - all_instances = session.query(models.Instance).\ + prefix = session.query(models.Instance).\ options(joinedload_all('fixed_ips.floating_ips')).\ options(joinedload('virtual_interfaces')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ips.network')).\ options(joinedload('metadata')).\ options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)).\ - all() + filter_by(deleted=can_read_deleted(context)) + + if context.is_admin: + all_instances = prefix.all() + elif context.project: + all_instances = prefix.\ + filter_by(project_id=context.project_id).\ + all() + else: + all_instances = prefix.\ + filter_by(user_id=context.user_id).\ + all() if not all_instances: return [] # Now do the regexp matching @@ -1265,15 +1275,25 @@ def instance_get_all_by_name_regexp(context, name_regexp): # MySQL 'regexp' is not portable, so we must do our own matching. # First... grab all Instances. - all_instances = session.query(models.Instance).\ + prefix = session.query(models.Instance).\ options(joinedload_all('fixed_ips.floating_ips')).\ options(joinedload('virtual_interfaces')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ips.network')).\ options(joinedload('metadata')).\ options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)).\ - all() + filter_by(deleted=can_read_deleted(context)) + + if context.is_admin: + all_instances = prefix.all() + elif context.project: + all_instances = prefix.\ + filter_by(project_id=context.project_id).\ + all() + else: + all_instances = prefix.\ + filter_by(user_id=context.user_id).\ + all() if not all_instances: return [] # Now do the regexp matching @@ -1336,6 +1356,15 @@ def instance_get_all_by_ip_regexp(context, ip_regexp): compiled_regexp.match(floating_ip.address): instances[fixed_ip.instance.uuid] = fixed_ip.instance + if context.is_admin: + return instances.values() + elif context.project: + return [instance for instance in instances.values() + if instance.project_id == context.project_id] + else: + return [instance for instance in instances.values() + if instance.user_id == context.user_id] + return instances.values() @@ -1347,15 +1376,26 @@ def instance_get_all_by_ipv6_regexp(context, ipv6_regexp): session = get_session() with session.begin(): - all_instances = session.query(models.Instance).\ + prefix = session.query(models.Instance).\ options(joinedload_all('fixed_ips.floating_ips')).\ options(joinedload('virtual_interfaces')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ips.network')).\ options(joinedload('metadata')).\ options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)).\ - all() + filter_by(deleted=can_read_deleted(context)) + + if context.is_admin: + all_instances = prefix.all() + elif context.project: + all_instances = prefix.\ + filter_by(project_id=context.project_id).\ + all() + else: + all_instances = prefix.\ + filter_by(user_id=context.user_id).\ + all() + if not all_instances: return [] -- cgit From 102a0e5b9d6ce22a5fc5a00fc260bbe1e3592222 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 18 Jul 2011 02:45:10 -0700 Subject: added searching by 'image', 'flavor', and 'status' reverted ip/ip6 searching to be admin only --- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index 6aa44f06b..b1b9b4544 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -520,6 +520,11 @@ def instance_get_by_fixed_ipv6(context, address): return IMPL.instance_get_by_fixed_ipv6(context, address) +def instance_get_all_by_column(context, column, column_data): + """Get all instances by exact match against the specified DB column""" + return IMPL.instance_get_all_by_column(context, column, column_data) + + def instance_get_all_by_column_regexp(context, column, column_regexp): """Get all instances by using regular expression matching against a particular DB column diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index feccba389..5209e9c8d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1225,6 +1225,48 @@ def instance_get_by_fixed_ipv6(context, address): return result +@require_context +def instance_get_all_by_column(context, column, column_data): + """Get all instances by exact match against the specified DB column + 'column_data' can be a list. + """ + session = get_session() + + + prefix = session.query(models.Instance).\ + options(joinedload_all('fixed_ips.floating_ips')).\ + options(joinedload('virtual_interfaces')).\ + options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ips.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')).\ + filter_by(deleted=can_read_deleted(context)) + + if isinstance(column_data, list): + column_attr = getattr(models.Instance, column) + prefix = prefix.filter(column_attr.in_(column_data)) + else: + # Set up the dictionary for filter_by() + query_filter = {} + query_filter[column] = column_data + prefix = prefix.filter_by(**query_filter) + + if context.is_admin: + all_instances = prefix.all() + elif context.project: + all_instances = prefix.\ + filter_by(project_id=context.project_id).\ + all() + else: + all_instances = prefix.\ + filter_by(user_id=context.user_id).\ + all() + if not all_instances: + return [] + + return all_instances + + @require_context def instance_get_all_by_column_regexp(context, column, column_regexp): """Get all instances by using regular expression matching against -- cgit From bc2a2f30e4b8ab92d6893ec333e756c92e96a932 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 20 Jul 2011 11:48:52 -0700 Subject: pep8 fixes --- nova/db/sqlalchemy/api.py | 1 - 1 file changed, 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 331c58fa3..7eb724785 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1248,7 +1248,6 @@ def instance_get_all_by_column(context, column, column_data): """ session = get_session() - prefix = session.query(models.Instance).\ options(joinedload_all('fixed_ips.floating_ips')).\ options(joinedload('virtual_interfaces')).\ -- cgit From 970b37ff9e9aef987f6e87df7d2c2e73c484e439 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 20 Jul 2011 12:35:42 -0700 Subject: missing doc strings for fixed_ip calls I renamed --- nova/db/sqlalchemy/api.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 43b9e195b..a3fc6d733 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1221,12 +1221,14 @@ def instance_get_all_by_reservation(context, reservation_id): @require_context def instance_get_by_fixed_ip(context, address): + """Return instance ref by exact match of FixedIP""" fixed_ip_ref = fixed_ip_get_by_address(context, address) return fixed_ip_ref.instance @require_context def instance_get_by_fixed_ipv6(context, address): + """Return instance ref by exact match of IPv6""" session = get_session() # convert IPv6 address to mac -- cgit From b5ac286fade15a61326068e5ef0959352f885efe Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 3 Aug 2011 23:08:42 -0700 Subject: a lot of major re-work.. still things to finish up --- nova/db/api.py | 39 +----- nova/db/sqlalchemy/api.py | 328 ++++++++++++++-------------------------------- 2 files changed, 106 insertions(+), 261 deletions(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index 22dd8a4b4..c7d5420e1 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -491,6 +491,10 @@ def instance_get_all(context): return IMPL.instance_get_all(context) +def instance_get_all_by-filters(context, filters): + """Get all instances that match all filters.""" + return IMPL.instance_get_all_by_filters(context, filters) + def instance_get_active_by_window(context, begin, end=None): """Get instances active during a certain time window.""" return IMPL.instance_get_active_by_window(context, begin, end) @@ -526,41 +530,6 @@ def instance_get_by_fixed_ipv6(context, address): return IMPL.instance_get_by_fixed_ipv6(context, address) -def instance_get_all_by_column(context, column, column_data): - """Get all instances by exact match against the specified DB column""" - return IMPL.instance_get_all_by_column(context, column, column_data) - - -def instance_get_all_by_column_regexp(context, column, column_regexp): - """Get all instances by using regular expression matching against - a particular DB column - """ - return IMPL.instance_get_all_by_column_regexp(context, - column, - column_regexp) - - -def instance_get_all_by_name_regexp(context, name_regexp): - """Get all instances by using regular expression matching against - its name - """ - return IMPL.instance_get_all_by_name_regexp(context, name_regexp) - - -def instance_get_all_by_ip_regexp(context, ip_regexp): - """Get all instances by using regular expression matching against - Floating and Fixed IP Addresses - """ - return IMPL.instance_get_all_by_ip_regexp(context, ip_regexp) - - -def instance_get_all_by_ipv6_regexp(context, ipv6_regexp): - """Get all instances by using regular expression matching against - IPv6 Addresses - """ - return IMPL.instance_get_all_by_ipv6_regexp(context, ipv6_regexp) - - def instance_get_fixed_addresses(context, instance_id): """Get the fixed ip address of an instance.""" return IMPL.instance_get_fixed_addresses(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index e4250a6cb..25a486b9c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1147,6 +1147,108 @@ def instance_get_all(context): all() +@require_context +def instance_get_all_by_filters(context, filters): + """Return instances the match all filters""" + + def _filter_by_ipv6(instance, filter_re): + for interface in instance['virtual_interfaces']: + fixed_ipv6 = interface.get('fixed_ipv6') + if fixed_ipv6 and filter_re.match(fixed_ipv6): + return True + return False + + def _filter_by_ip(instance, filter_re): + for interface in instance['virtual_interfaces']: + for fixed_ip in interface['fixed_ips']: + if not fixed_ip or not fixed_ip['address']: + continue + if filter_re.match(fixed_ip['address']): + return True + for floating_ip in fixed_ip.get('floating_ips', []): + if not floating_ip or not floating_ip['address']: + continue + if filter_re.match(floating_ip['address']): + return True + return False + + def _filter_by_display_name(instance, filter_re): + if filter_re.match(instance.display_name): + return True + return False + + def _filter_by_column(instance, filter_name, filter_re): + try: + v = getattr(instance, filter_name) + except AttributeError: + return True + if v and filter_re.match(str(v)): + return True + return False + + session = get_session() + query_prefix = session.query(models.Instance).\ + options(joinedload_all('fixed_ips.floating_ips')).\ + options(joinedload_all('virtual_interfaces.network')).\ + options(joinedload_all( + 'virtual_interfaces.fixed_ips.floating_ips')).\ + options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ips.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')).\ + filter_by(deleted=can_read_deleted(context)) + + filters = filters.copy() + + if not context.is_admin: + # If we're not admin context, add appropriate filter.. + if context.project_id: + filters['project_id'] = context.project_id + else: + filters['user_id'] = context.user_id + + # Filters that we can do along with the SQL query... + query_filter_funcs = { + 'project_id': lambda query, value: query.filter_by( + project_id=value), + 'user_id': lambda query, value: query.filter_by( + user_id=value), + 'reservation_id': lambda query, value: query.filter_by( + reservation_id=value), + 'state': lambda query, value: query.filter_by(state=value)} + + query_filters = [key for key in filters.iterkeys() + if key in query_filter_funcs] + + for filter_name in query_filters: + query_prefix = query_filter_funcs[filter_name](query_prefix, + filters[filter_name]) + # Remove this from filters, so it doesn't get tried below + del filters[filter_name] + + instances = query_prefix.all() + + if not instances: + return [] + + # Now filter on everything else for regexp matching.. + filter_funcs = {'ip6': _filter_by_ipv6, + 'ip': _filter_by_ip, + 'name': _filter_by_display_name} + + for filter_name in filters.iterkeys(): + filter_func = filter_funcs.get(filter_name, None) + filter_re = re.compile(filters[filter_name]) + if filter_func: + filter_l = lambda instance: filter_func(instance, filter_re) + else: + filter_l = lambda instance: _filter_by_column(instance, + filter_name, filter_re) + instances = filter(filter_l, instances) + + return instances + + @require_admin_context def instance_get_active_by_window(context, begin, end=None): """Return instances that were continuously active over the given window""" @@ -1259,232 +1361,6 @@ def instance_get_by_fixed_ipv6(context, address): return result -@require_context -def instance_get_all_by_column(context, column, column_data): - """Get all instances by exact match against the specified DB column - 'column_data' can be a list. - """ - session = get_session() - - prefix = session.query(models.Instance).\ - options(joinedload_all('fixed_ips.floating_ips')).\ - options(joinedload('virtual_interfaces')).\ - options(joinedload('security_groups')).\ - options(joinedload_all('fixed_ips.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)) - - if isinstance(column_data, list): - column_attr = getattr(models.Instance, column) - prefix = prefix.filter(column_attr.in_(column_data)) - else: - # Set up the dictionary for filter_by() - query_filter = {} - query_filter[column] = column_data - prefix = prefix.filter_by(**query_filter) - - if context.is_admin: - all_instances = prefix.all() - elif context.project: - all_instances = prefix.\ - filter_by(project_id=context.project_id).\ - all() - else: - all_instances = prefix.\ - filter_by(user_id=context.user_id).\ - all() - if not all_instances: - return [] - - return all_instances - - -@require_context -def instance_get_all_by_column_regexp(context, column, column_regexp): - """Get all instances by using regular expression matching against - a particular DB column - """ - session = get_session() - - # MySQL 'regexp' is not portable, so we must do our own matching. - # First... grab all Instances. - prefix = session.query(models.Instance).\ - options(joinedload_all('fixed_ips.floating_ips')).\ - options(joinedload('virtual_interfaces')).\ - options(joinedload('security_groups')).\ - options(joinedload_all('fixed_ips.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)) - - if context.is_admin: - all_instances = prefix.all() - elif context.project: - all_instances = prefix.\ - filter_by(project_id=context.project_id).\ - all() - else: - all_instances = prefix.\ - filter_by(user_id=context.user_id).\ - all() - if not all_instances: - return [] - # Now do the regexp matching - compiled_regexp = re.compile(column_regexp) - instances = [] - for instance in all_instances: - v = getattr(instance, column) - if v and compiled_regexp.match(v): - instances.append(instance) - return instances - - -@require_context -def instance_get_all_by_name_regexp(context, name_regexp): - """Get all instances by using regular expression matching against - its name - """ - - session = get_session() - - # MySQL 'regexp' is not portable, so we must do our own matching. - # First... grab all Instances. - prefix = session.query(models.Instance).\ - options(joinedload_all('fixed_ips.floating_ips')).\ - options(joinedload('virtual_interfaces')).\ - options(joinedload('security_groups')).\ - options(joinedload_all('fixed_ips.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)) - - if context.is_admin: - all_instances = prefix.all() - elif context.project: - all_instances = prefix.\ - filter_by(project_id=context.project_id).\ - all() - else: - all_instances = prefix.\ - filter_by(user_id=context.user_id).\ - all() - if not all_instances: - return [] - # Now do the regexp matching - compiled_regexp = re.compile(name_regexp) - return [instance for instance in all_instances - if compiled_regexp.match(instance.name)] - - -@require_context -def instance_get_all_by_ip_regexp(context, ip_regexp): - """Get all instances by using regular expression matching against - Floating and Fixed IP Addresses - """ - session = get_session() - - # Query both FixedIp and FloatingIp tables to get matches. - # Since someone could theoretically search for something that matches - # instances in both tables... we need to use a dictionary keyed - # on instance ID to make sure we return only 1. We can't key off - # of 'instance' because it's just a reference and will be different - # addresses even though they might point to the same instance ID. - instances = {} - - fixed_ips = session.query(models.FixedIp).\ - options(joinedload_all('instance.fixed_ips.floating_ips')).\ - options(joinedload('instance.virtual_interfaces')).\ - options(joinedload('instance.security_groups')).\ - options(joinedload_all('instance.fixed_ips.network')).\ - options(joinedload('instance.metadata')).\ - options(joinedload('instance.instance_type')).\ - filter_by(deleted=can_read_deleted(context)).\ - all() - floating_ips = session.query(models.FloatingIp).\ - options(joinedload_all( - 'fixed_ip.instance.fixed_ips.floating_ips')).\ - options(joinedload('fixed_ip.instance.virtual_interfaces')).\ - options(joinedload('fixed_ip.instance.security_groups')).\ - options(joinedload_all( - 'fixed_ip.instance.fixed_ips.network')).\ - options(joinedload('fixed_ip.instance.metadata')).\ - options(joinedload('fixed_ip.instance.instance_type')).\ - filter_by(deleted=can_read_deleted(context)).\ - all() - - if fixed_ips is None: - fixed_ips = [] - if floating_ips is None: - floating_ips = [] - - compiled_regexp = re.compile(ip_regexp) - instances = {} - - # Now do the regexp matching - for fixed_ip in fixed_ips: - if fixed_ip.instance and compiled_regexp.match(fixed_ip.address): - instances[fixed_ip.instance.uuid] = fixed_ip.instance - for floating_ip in floating_ips: - fixed_ip = floating_ip.fixed_ip - if fixed_ip and fixed_ip.instance and\ - compiled_regexp.match(floating_ip.address): - instances[fixed_ip.instance.uuid] = fixed_ip.instance - - if context.is_admin: - return instances.values() - elif context.project: - return [instance for instance in instances.values() - if instance.project_id == context.project_id] - else: - return [instance for instance in instances.values() - if instance.user_id == context.user_id] - - return instances.values() - - -@require_context -def instance_get_all_by_ipv6_regexp(context, ipv6_regexp): - """Get all instances by using regular expression matching against - IPv6 Addresses - """ - - session = get_session() - with session.begin(): - prefix = session.query(models.Instance).\ - options(joinedload_all('fixed_ips.floating_ips')).\ - options(joinedload('virtual_interfaces')).\ - options(joinedload('security_groups')).\ - options(joinedload_all('fixed_ips.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)) - - if context.is_admin: - all_instances = prefix.all() - elif context.project: - all_instances = prefix.\ - filter_by(project_id=context.project_id).\ - all() - else: - all_instances = prefix.\ - filter_by(user_id=context.user_id).\ - all() - - if not all_instances: - return [] - - instances = [] - compiled_regexp = re.compile(ipv6_regexp) - for instance in all_instances: - ipv6_addrs = _ipv6_get_by_instance_ref(context, instance) - for ipv6 in ipv6_addrs: - if compiled_regexp.match(ipv6): - instances.append(instance) - break - return instances - - @require_admin_context def instance_get_project_vpn(context, project_id): session = get_session() -- cgit From c0851f2ec5be12c43cc96367e22220d25589e4ae Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 4 Aug 2011 01:36:12 -0700 Subject: cleanup checking of options in the API before calling compute_api's get_all() --- nova/db/sqlalchemy/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 25a486b9c..04c9273b0 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1149,7 +1149,9 @@ def instance_get_all(context): @require_context def instance_get_all_by_filters(context, filters): - """Return instances the match all filters""" + """Return instances the match all filters. Deleted instances + will be returned by default, unless there's a filter that says + otherwise""" def _filter_by_ipv6(instance, filter_re): for interface in instance['virtual_interfaces']: @@ -1195,8 +1197,7 @@ def instance_get_all_by_filters(context, filters): options(joinedload('security_groups')).\ options(joinedload_all('fixed_ips.network')).\ options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(deleted=can_read_deleted(context)) + options(joinedload('instance_type')) filters = filters.copy() -- cgit From b9ecf869761ee0506872b0d44d93d453be4c3477 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 4 Aug 2011 01:45:42 -0700 Subject: typos --- nova/db/api.py | 2 +- nova/db/sqlalchemy/api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index c7d5420e1..23fac9921 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -491,7 +491,7 @@ def instance_get_all(context): return IMPL.instance_get_all(context) -def instance_get_all_by-filters(context, filters): +def instance_get_all_by_filters(context, filters): """Get all instances that match all filters.""" return IMPL.instance_get_all_by_filters(context, filters) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 04c9273b0..55c804ae9 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1149,7 +1149,7 @@ def instance_get_all(context): @require_context def instance_get_all_by_filters(context, filters): - """Return instances the match all filters. Deleted instances + """Return instances that match all filters. Deleted instances will be returned by default, unless there's a filter that says otherwise""" -- cgit From b1b919d42d8c359fc9ae981b44466d269fc688a6 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 4 Aug 2011 13:59:57 -0700 Subject: test fixes and typos --- nova/db/sqlalchemy/api.py | 49 ++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 20 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 55c804ae9..f8920e62c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1153,14 +1153,14 @@ def instance_get_all_by_filters(context, filters): will be returned by default, unless there's a filter that says otherwise""" - def _filter_by_ipv6(instance, filter_re): + def _regexp_filter_by_ipv6(instance, filter_re): for interface in instance['virtual_interfaces']: fixed_ipv6 = interface.get('fixed_ipv6') if fixed_ipv6 and filter_re.match(fixed_ipv6): return True return False - def _filter_by_ip(instance, filter_re): + def _regexp_filter_by_ip(instance, filter_re): for interface in instance['virtual_interfaces']: for fixed_ip in interface['fixed_ips']: if not fixed_ip or not fixed_ip['address']: @@ -1174,12 +1174,12 @@ def instance_get_all_by_filters(context, filters): return True return False - def _filter_by_display_name(instance, filter_re): + def _regexp_filter_by_display_name(instance, filter_re): if filter_re.match(instance.display_name): return True return False - def _filter_by_column(instance, filter_name, filter_re): + def _regexp_filter_by_column(instance, filter_name, filter_re): try: v = getattr(instance, filter_name) except AttributeError: @@ -1188,6 +1188,18 @@ def instance_get_all_by_filters(context, filters): return True return False + def _exact_match_filter(query, column, value): + """Do exact match against a column. value to match can be a list + so you can match any value in the list. + """ + if isinstance(value, list): + column_attr = getattr(models.Instance, column) + return query.filter(column_attr.in_(value)) + else: + filter_dict = {} + filter_dict[column] = value + return query.filter_by(**filter_dict) + session = get_session() query_prefix = session.query(models.Instance).\ options(joinedload_all('fixed_ips.floating_ips')).\ @@ -1208,21 +1220,16 @@ def instance_get_all_by_filters(context, filters): else: filters['user_id'] = context.user_id - # Filters that we can do along with the SQL query... - query_filter_funcs = { - 'project_id': lambda query, value: query.filter_by( - project_id=value), - 'user_id': lambda query, value: query.filter_by( - user_id=value), - 'reservation_id': lambda query, value: query.filter_by( - reservation_id=value), - 'state': lambda query, value: query.filter_by(state=value)} + # Filters for exact matches that we can do along with the SQL query... + # For other filters that don't match this, we will do regexp matching + exact_match_filter_names = ['project_id', 'user_id', 'image_ref', + 'state', 'instance_type_id', 'deleted'] query_filters = [key for key in filters.iterkeys() - if key in query_filter_funcs] + if key in exact_match_filter_names] for filter_name in query_filters: - query_prefix = query_filter_funcs[filter_name](query_prefix, + query_prefix = _exact_match_filter(query_prefix, filter_name, filters[filter_name]) # Remove this from filters, so it doesn't get tried below del filters[filter_name] @@ -1233,17 +1240,19 @@ def instance_get_all_by_filters(context, filters): return [] # Now filter on everything else for regexp matching.. - filter_funcs = {'ip6': _filter_by_ipv6, - 'ip': _filter_by_ip, - 'name': _filter_by_display_name} + # For filters not in the list, we'll attempt to use the filter_name + # as a column name in Instance.. + regexp_filter_funcs = {'ip6': _regexp_filter_by_ipv6, + 'ip': _regexp_filter_by_ip, + 'name': _regexp_filter_by_display_name} for filter_name in filters.iterkeys(): - filter_func = filter_funcs.get(filter_name, None) + filter_func = regexp_filter_funcs.get(filter_name, None) filter_re = re.compile(filters[filter_name]) if filter_func: filter_l = lambda instance: filter_func(instance, filter_re) else: - filter_l = lambda instance: _filter_by_column(instance, + filter_l = lambda instance: _regexp_filter_by_column(instance, filter_name, filter_re) instances = filter(filter_l, instances) -- cgit From 6e791e8b773565b62c4b8ba35cec455cb8c13ac8 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 4 Aug 2011 16:30:55 -0700 Subject: test fixes.. one more to go --- nova/db/sqlalchemy/api.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index f8920e62c..4d724dbdb 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1174,11 +1174,6 @@ def instance_get_all_by_filters(context, filters): return True return False - def _regexp_filter_by_display_name(instance, filter_re): - if filter_re.match(instance.display_name): - return True - return False - def _regexp_filter_by_column(instance, filter_name, filter_re): try: v = getattr(instance, filter_name) @@ -1243,8 +1238,7 @@ def instance_get_all_by_filters(context, filters): # For filters not in the list, we'll attempt to use the filter_name # as a column name in Instance.. regexp_filter_funcs = {'ip6': _regexp_filter_by_ipv6, - 'ip': _regexp_filter_by_ip, - 'name': _regexp_filter_by_display_name} + 'ip': _regexp_filter_by_ip} for filter_name in filters.iterkeys(): filter_func = regexp_filter_funcs.get(filter_name, None) -- cgit From 64966cbd83cbde6a240dad4ac786fe7a6a116f2f Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 4 Aug 2011 17:30:07 -0700 Subject: pep8 fixes --- nova/db/api.py | 1 + nova/db/sqlalchemy/api.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index 23fac9921..4c8f25f5d 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -495,6 +495,7 @@ def instance_get_all_by_filters(context, filters): """Get all instances that match all filters.""" return IMPL.instance_get_all_by_filters(context, filters) + def instance_get_active_by_window(context, begin, end=None): """Get instances active during a certain time window.""" return IMPL.instance_get_active_by_window(context, begin, end) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 4d724dbdb..36fae1be1 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1205,7 +1205,7 @@ def instance_get_all_by_filters(context, filters): options(joinedload_all('fixed_ips.network')).\ options(joinedload('metadata')).\ options(joinedload('instance_type')) - + filters = filters.copy() if not context.is_admin: -- cgit From 4d0125b34a4796fd6d3312a4144a0834ba318469 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 4 Aug 2011 17:50:59 -0700 Subject: convert filter value to a string just in case before running re.compile --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 36fae1be1..d6e7204b4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1242,7 +1242,7 @@ def instance_get_all_by_filters(context, filters): for filter_name in filters.iterkeys(): filter_func = regexp_filter_funcs.get(filter_name, None) - filter_re = re.compile(filters[filter_name]) + filter_re = re.compile(str(filters[filter_name])) if filter_func: filter_l = lambda instance: filter_func(instance, filter_re) else: -- cgit From b2fe710c59ba266b9afd67db1cae60a6db5c71e3 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 5 Aug 2011 15:06:07 -0700 Subject: rename _check_servers_options, add some comments and small cleanup in the db get_by_filters call --- nova/db/sqlalchemy/api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index d6e7204b4..65a1c19a1 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1206,6 +1206,8 @@ def instance_get_all_by_filters(context, filters): options(joinedload('metadata')).\ options(joinedload('instance_type')) + # Make a copy of the filters dictionary to use going forward, as we'll + # be modifying it and we shouldn't affect the caller's use of it. filters = filters.copy() if not context.is_admin: @@ -1224,10 +1226,10 @@ def instance_get_all_by_filters(context, filters): if key in exact_match_filter_names] for filter_name in query_filters: + # Do the matching and remove the filter from the dictionary + # so we don't try it again below.. query_prefix = _exact_match_filter(query_prefix, filter_name, - filters[filter_name]) - # Remove this from filters, so it doesn't get tried below - del filters[filter_name] + filters.pop(filter_name)) instances = query_prefix.all() -- cgit From 65fcbc8cf51cc02071d1d9cd60cf0eb59c2bcce0 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 6 Jul 2011 06:44:50 -0700 Subject: merge code i'd split from instance_get_fixed_addresses_v6 that's no longer needed to be split --- nova/db/sqlalchemy/api.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 65a1c19a1..503c526f0 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1395,33 +1395,29 @@ def instance_get_fixed_addresses(context, instance_id): return [fixed_ip.address for fixed_ip in fixed_ips] -def _ipv6_get_by_instance_ref(context, instance_ref): - # assume instance has 1 mac for each network associated with it - # get networks associated with instance - network_refs = network_get_all_by_instance(context, instance_id) - # compile a list of cidr_v6 prefixes sorted by network id - prefixes = [ref.cidr_v6 for ref in - sorted(network_refs, key=lambda ref: ref.id)] - # get vifs associated with instance - vif_refs = virtual_interface_get_by_instance(context, instance_ref.id) - # compile list of the mac_addresses for vifs sorted by network id - macs = [vif_ref['address'] for vif_ref in - sorted(vif_refs, key=lambda vif_ref: vif_ref['network_id'])] - # get project id from instance - project_id = instance_ref.project_id - # combine prefixes, macs, and project_id into (prefix,mac,p_id) tuples - prefix_mac_tuples = zip(prefixes, macs, [project_id for m in macs]) - # return list containing ipv6 address for each tuple - return [ipv6.to_global(*t) for t in prefix_mac_tuples] - - @require_context def instance_get_fixed_addresses_v6(context, instance_id): session = get_session() with session.begin(): # get instance instance_ref = instance_get(context, instance_id, session=session) - return _ipv6_get_by_instance_ref(context, instance_ref) + # assume instance has 1 mac for each network associated with it + # get networks associated with instance + network_refs = network_get_all_by_instance(context, instance_id) + # compile a list of cidr_v6 prefixes sorted by network id + prefixes = [ref.cidr_v6 for ref in + sorted(network_refs, key=lambda ref: ref.id)] + # get vifs associated with instance + vif_refs = virtual_interface_get_by_instance(context, instance_ref.id) + # compile list of the mac_addresses for vifs sorted by network id + macs = [vif_ref['address'] for vif_ref in + sorted(vif_refs, key=lambda vif_ref: vif_ref['network_id'])] + # get project id from instance + project_id = instance_ref.project_id + # combine prefixes, macs, and project_id into (prefix,mac,p_id) tuples + prefix_mac_tuples = zip(prefixes, macs, [project_id for m in macs]) + # return list containing ipv6 address for each tuple + return [ipv6.to_global(*t) for t in prefix_mac_tuples] @require_context -- cgit From 9be2793c2e057a5e4f8c8c4dd2131ddcc3b11608 Mon Sep 17 00:00:00 2001 From: Isaku Yamahata Date: Sat, 23 Jul 2011 16:55:25 +0900 Subject: db/api: block_device_mapping_update_or_create() It is possible to have same virtual device name. So eliminate old entries whose entry has same virtual device name. --- nova/db/sqlalchemy/api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index ba03cabbc..ad51f5192 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -20,6 +20,7 @@ Implementation of SQLAlchemy backend. """ import warnings +from nova import block_device from nova import db from nova import exception from nova import flags @@ -2264,6 +2265,20 @@ def block_device_mapping_update_or_create(context, values): else: result.update(values) + # NOTE(yamahata): same virtual device name can be specified multiple + # times. So delete the existing ones. + virtual_name = values['virtual_name'] + if (virtual_name is not None and + block_device.is_swap_or_ephemeral(virtual_name)): + session.query(models.BlockDeviceMapping).\ + filter_by(instance_id=values['instance_id']).\ + filter_by(virtual_name=virtual_name).\ + filter(models.BlockDeviceMapping.device_name != + values['device_name']).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + @require_context def block_device_mapping_get_all_by_instance(context, instance_id): -- cgit From 24b6597035c4393383ed1bdc2a6e52830743a7ea Mon Sep 17 00:00:00 2001 From: Isaku Yamahata Date: Sat, 23 Jul 2011 16:57:04 +0900 Subject: db/api: fix network_get_by_cidr() User of the function is only 'nova-manage network delete'. It doesn't check deleted flag which must be checked. Otherwise some it might pick up deleted column depending on query result, and tries to delete already deleted columns and results in exception. --- nova/db/sqlalchemy/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index ad51f5192..abfa6a3b7 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1681,7 +1681,9 @@ def network_get_by_bridge(context, bridge): def network_get_by_cidr(context, cidr): session = get_session() result = session.query(models.Network).\ - filter_by(cidr=cidr).first() + filter_by(cidr=cidr).\ + filter_by(deleted=False).\ + first() if not result: raise exception.NetworkNotFoundForCidr(cidr=cidr) -- cgit From 19e50320e36f02ce11a6aaae8f88a6ddbc132859 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 5 Aug 2011 07:21:55 -0700 Subject: pep8 violations sneaking into trunk? --- nova/db/sqlalchemy/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index ce12ba4e0..8cfa94ed7 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -3301,7 +3301,8 @@ def instance_type_extra_specs_delete(context, instance_type_id, key): @require_context -def instance_type_extra_specs_get_item(context, instance_type_id, key, session=None): +def instance_type_extra_specs_get_item(context, instance_type_id, key, + session=None): if not session: session = get_session() -- cgit From 61cf3721ce94d7f2458e4e469cbee3333f954588 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 8 Aug 2011 16:38:14 -0400 Subject: cleaning up instance metadata api code --- nova/db/api.py | 6 +++--- nova/db/sqlalchemy/api.py | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 14 deletions(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index 47308bdba..90003cf86 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -1381,9 +1381,9 @@ def instance_metadata_delete(context, instance_id, key): IMPL.instance_metadata_delete(context, instance_id, key) -def instance_metadata_update_or_create(context, instance_id, metadata): - """Create or update instance metadata.""" - IMPL.instance_metadata_update_or_create(context, instance_id, metadata) +def instance_metadata_update(context, instance_id, metadata, delete): + """Update metadata if it exists, otherwise create it.""" + IMPL.instance_metadata_update(context, instance_id, metadata) #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 003eb8bcb..5ae14261c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1345,9 +1345,10 @@ def instance_update(context, instance_id, values): session = get_session() metadata = values.get('metadata') if metadata is not None: - instance_metadata_delete_all(context, instance_id) - instance_metadata_update_or_create(context, instance_id, - values.pop('metadata')) + instance_metadata_update(context, + instance_id, + values.pop('metadata'), + delete=True) with session.begin(): if utils.is_uuid_like(instance_id): instance_ref = instance_get_by_uuid(context, instance_id, @@ -3201,21 +3202,37 @@ def instance_metadata_get_item(context, instance_id, key, session=None): @require_context @require_instance_exists -def instance_metadata_update_or_create(context, instance_id, metadata): +def instance_metadata_update(context, instance_id, metadata, delete): session = get_session() - original_metadata = instance_metadata_get(context, instance_id) + # Set all metadata that isn't passed in if delete kwarg is True + if delete: + original_metadata = instance_metadata_get(context, instance_id) + for meta in original_metadata.iteritems(): + if meta.key not in metadata: + meta.update({"deleted": True}) + meta.save(session=session) meta_ref = None - for key, value in metadata.iteritems(): + + # Now update all existing items with new values, or create new meta objects + for meta_key, meta_value in metadata.iteritems(): + + # update the value whether it exists or not + item = {"value": meta_value} + + # if the metadata item exists, make sure it is not delete try: - meta_ref = instance_metadata_get_item(context, instance_id, key, - session) + meta_ref = instance_metadata_get_item(context, instance_id, + meta_key, session) + item.update({"deleted": False}) + + # if the item doesn't exist, we also need to set key and instance_id except exception.InstanceMetadataNotFound, e: meta_ref = models.InstanceMetadata() - meta_ref.update({"key": key, "value": value, - "instance_id": instance_id, - "deleted": False}) + item.update({"key": meta_key, "instance_id": instance_id}) + + meta_ref.update(item) meta_ref.save(session=session) return metadata -- cgit From 4de24a4d44040ba38a474cd789b95a2b59d494ff Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 8 Aug 2011 17:33:03 -0400 Subject: making server metadata work functionally --- nova/db/api.py | 2 +- nova/db/sqlalchemy/api.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'nova/db') diff --git a/nova/db/api.py b/nova/db/api.py index 90003cf86..0516c683f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -1383,7 +1383,7 @@ def instance_metadata_delete(context, instance_id, key): def instance_metadata_update(context, instance_id, metadata, delete): """Update metadata if it exists, otherwise create it.""" - IMPL.instance_metadata_update(context, instance_id, metadata) + IMPL.instance_metadata_update(context, instance_id, metadata, delete) #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 5ae14261c..2e29f34ac 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -3208,10 +3208,12 @@ def instance_metadata_update(context, instance_id, metadata, delete): # Set all metadata that isn't passed in if delete kwarg is True if delete: original_metadata = instance_metadata_get(context, instance_id) - for meta in original_metadata.iteritems(): - if meta.key not in metadata: - meta.update({"deleted": True}) - meta.save(session=session) + for meta_key, meta_value in original_metadata.iteritems(): + if meta_key not in metadata: + meta_ref = instance_metadata_get_item(context, instance_id, + meta_key, session) + meta_ref.update({'deleted': True}) + meta_ref.save(session=session) meta_ref = None @@ -3221,11 +3223,9 @@ def instance_metadata_update(context, instance_id, metadata, delete): # update the value whether it exists or not item = {"value": meta_value} - # if the metadata item exists, make sure it is not delete try: meta_ref = instance_metadata_get_item(context, instance_id, meta_key, session) - item.update({"deleted": False}) # if the item doesn't exist, we also need to set key and instance_id except exception.InstanceMetadataNotFound, e: -- cgit From ed4a3b33647d3cbf5b1733596c1e180078e23cb0 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 9 Aug 2011 10:29:07 -0400 Subject: updating tests; fixing create output; review fixes --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/db') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2e29f34ac..24ea23611 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -3205,7 +3205,7 @@ def instance_metadata_get_item(context, instance_id, key, session=None): def instance_metadata_update(context, instance_id, metadata, delete): session = get_session() - # Set all metadata that isn't passed in if delete kwarg is True + # Set existing metadata to deleted if delete argument is True if delete: original_metadata = instance_metadata_get(context, instance_id) for meta_key, meta_value in original_metadata.iteritems(): -- cgit