From c94ec9a5bab6c07b402b68e2f4ff081247a27cda Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 14:17:58 -0700 Subject: Initial implementation of refresh instance states --- nova/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 87e726394..e93f489be 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -585,3 +585,12 @@ def get_from_path(items, path): return results else: return get_from_path(results, remainder) + + +def check_instance(obj, cls): + """Checks that obj is of type cls, and lets PyLint infer types""" + if isinstance(obj, cls): + return obj + raise Exception(_("Expected object of type: %s") % (str(cls))) + #TODO(justinsb): Can we make this better?? + return cls() # Ugly PyLint hack -- cgit From d1860ce5d26fbbadb2310e8225e924879cde9a6c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 10:35:43 +0100 Subject: Make synchronized support both external (file based) locks as well as internal (semaphore based) locks. Attempt to make it native thread safe at the expense of never cleaning up semaphores. --- nova/utils.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 7 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 499af2039..8936614cc 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -41,6 +41,7 @@ from xml.sax import saxutils from eventlet import event from eventlet import greenthread +from eventlet import semaphore from eventlet.green import subprocess None from nova import exception @@ -531,17 +532,69 @@ def loads(s): return json.loads(s) -def synchronized(name): +_semaphores_semaphore = semaphore.Semaphore() +_semaphores = {} + + +class _NoopContextManager(object): + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + +def synchronized(name, external=False): + """Synchronization decorator + + Decorating a method like so: + @synchronized('mylock') + def foo(self, *args): + ... + + ensures that only one thread will execute the bar method at a time. + + Different methods can share the same lock: + @synchronized('mylock') + def foo(self, *args): + ... + + @synchronized('mylock') + def bar(self, *args): + ... + + This way only one of either foo or bar can be executing at a time. + + The external keyword argument denotes whether this lock should work across + multiple processes. This means that if two different workers both run a + a method decorated with @synchronized('mylock', external=True), only one + of them will execute at a time. + """ + def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): - LOG.debug(_("Attempting to grab %(lock)s for method " - "%(method)s..." % {"lock": name, + with _semaphores_semaphore: + if name not in _semaphores: + _semaphores[name] = semaphore.Semaphore() + sem = _semaphores[name] + LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method ' + '"%(method)s"...' % {"lock": name, "method": f.__name__})) - lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, - 'nova-%s.lock' % name)) - with lock: - return f(*args, **kwargs) + with sem: + if external: + LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' + 'method "%(method)s"...' % + {"lock": name, "method": f.__name__})) + lock_file_path = os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name) + lock = lockfile.FileLock(lock_file_path) + else: + lock = _NoopContextManager() + + with lock: + return f(*args, **kwargs) + return inner return wrap -- cgit From 60a3aa86db1d0e1ea2f680c9587881e45fa99336 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 14:14:47 +0100 Subject: Make synchronized decorator not leak semaphores, at the expense of not being truly thread safe (but safe enough for Eventlet style green threads). --- nova/utils.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 8936614cc..c580e805a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -574,10 +574,12 @@ def synchronized(name, external=False): def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): - with _semaphores_semaphore: - if name not in _semaphores: - _semaphores[name] = semaphore.Semaphore() - sem = _semaphores[name] + # NOTE(soren): If we ever go natively threaded, this will be racy. + # See http://stackoverflow.com/questions/5390569/dyn\ + # amically-allocating-and-destroying-mutexes + if name not in _semaphores: + _semaphores[name] = semaphore.Semaphore() + sem = _semaphores[name] LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method ' '"%(method)s"...' % {"lock": name, "method": f.__name__})) @@ -593,8 +595,14 @@ def synchronized(name, external=False): lock = _NoopContextManager() with lock: - return f(*args, **kwargs) + retval = f(*args, **kwargs) + # If no-one else is waiting for it, delete it. + # See note about possible raciness above. + if not sem.balance < 1: + del _semaphores[name] + + return retval return inner return wrap -- cgit From 06815cb729d8687403fc736ae6125c26867f42b3 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 17:13:48 +0100 Subject: Remove unused global semaphore. --- nova/utils.py | 1 - 1 file changed, 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index c580e805a..8b9ce4734 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -532,7 +532,6 @@ def loads(s): return json.loads(s) -_semaphores_semaphore = semaphore.Semaphore() _semaphores = {} -- cgit From 789fcb46915dce5fa533357ac462040ec6aa8968 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 22 Mar 2011 20:26:45 +0000 Subject: Adding BASE_IMAGE_ATTRS to ImageService --- nova/utils.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 199ee8701..96a51b425 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -597,3 +597,39 @@ def get_from_path(items, path): return results else: return get_from_path(results, remainder) + + +def flatten_dict(dict_, flattened=None): + """Recursively flatten a nested dictionary""" + flattened = flattened or {} + for key, value in dict_.iteritems(): + if hasattr(value, 'iteritems'): + flatten_dict(value, flattened) + else: + flattened[key] = value + return flattened + + +def partition_dict(dict_, keys): + """Return two dicts, one containing only `keys` the other containing + everything but `keys` + """ + intersection = {} + difference = {} + for key, value in dict_.iteritems(): + if key in keys: + intersection[key] = value + else: + difference[key] = value + return intersection, difference + + +def map_dict_keys(dict_, key_map): + """Return a dictionary in which the dictionaries keys are mapped to + new keys. + """ + mapped = {} + for key, value in dict_.iteritems(): + mapped_key = key_map[key] if key in key_map else key + mapped[mapped_key] = value + return mapped -- cgit From d8176eda3f31973a8718b98f35e202ff61c48bbc Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 22 Mar 2011 20:34:00 +0000 Subject: Pep8 fix --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index f43ba2e7e..7b96a0daf 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -621,7 +621,7 @@ def partition_dict(dict_, keys): def map_dict_keys(dict_, key_map): - """Return a dictionary in which the dictionaries keys are mapped to + """Return a dictionary in which the dictionaries keys are mapped to new keys. """ mapped = {} -- cgit From 365b98f4d52740ef85f8a8f098a32e441d7ac168 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Mar 2011 21:42:17 -0700 Subject: Renamed check_instance -> check_isinstance to make intent clearer --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index e93f489be..2e653bda0 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -587,7 +587,7 @@ def get_from_path(items, path): return get_from_path(results, remainder) -def check_instance(obj, cls): +def check_isinstance(obj, cls): """Checks that obj is of type cls, and lets PyLint infer types""" if isinstance(obj, cls): return obj -- cgit From a4d78e44d7ca35a6cca4454667cab743409fd95a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Mar 2011 22:45:15 -0700 Subject: Added space in between # and TODO in #TODO --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 2e653bda0..36b384f4f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -592,5 +592,5 @@ def check_isinstance(obj, cls): if isinstance(obj, cls): return obj raise Exception(_("Expected object of type: %s") % (str(cls))) - #TODO(justinsb): Can we make this better?? + # TODO(justinsb): Can we make this better?? return cls() # Ugly PyLint hack -- cgit From a7c9ad393f72b49515a445504a5bc87f8a26932c Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Wed, 23 Mar 2011 05:50:53 +0000 Subject: Filtering images by user_id now --- nova/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 7b96a0daf..d114cb14f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -629,3 +629,9 @@ def map_dict_keys(dict_, key_map): mapped_key = key_map[key] if key in key_map else key mapped[mapped_key] = value return mapped + + +def subset_dict(dict_, keys): + """Return a dict that only contains a subset of keys""" + subset = partition_dict(dict_, keys)[0] + return subset -- cgit From abb764f51385a0b811b23379d78f7db027d4cca5 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 14:41:35 -0500 Subject: Automatically unrescue instances after a given timeout --- nova/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 499af2039..38cdb8021 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -334,6 +334,13 @@ def utcnow(): utcnow.override_time = None +def is_then_greater(then, seconds): + if utcnow() - then > datetime.timedelta(seconds=seconds): + return True + else: + return False + + def utcnow_ts(): """Timestamp version of our utcnow function.""" return time.mktime(utcnow().timetuple()) -- cgit From 3c0fcc47be08ac4f3d508fd46f3b95036899aaad Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Mar 2011 13:39:01 -0700 Subject: fix utils.execute retries for osx also some minor misc cleanups --- nova/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 499af2039..249470636 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -170,10 +170,6 @@ def execute(*cmd, **kwargs): stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess - # call clean something up in between calls, without - # it two execute calls in a row hangs the second one - greenthread.sleep(0) return result except ProcessExecutionError: if not attempts: @@ -182,6 +178,11 @@ def execute(*cmd, **kwargs): LOG.debug(_("%r failed. Retrying."), cmd) if delay_on_retry: greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) def ssh_execute(ssh, cmd, process_input=None, -- cgit From 95fa499f1a7718694e37a747a6a5a0e309ce877d Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Thu, 24 Mar 2011 00:36:07 +0300 Subject: migration gateway_v6 to network_info --- nova/utils.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 499af2039..44234813f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -309,11 +309,15 @@ def get_my_linklocal(interface): def to_global_ipv6(prefix, mac): - mac64 = netaddr.EUI(mac).eui64().words - int_addr = int(''.join(['%02x' % i for i in mac64]), 16) - mac64_addr = netaddr.IPAddress(int_addr) - maskIP = netaddr.IPNetwork(prefix).ip - return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).format() + try: + mac64 = netaddr.EUI(mac).eui64().words + int_addr = int(''.join(['%02x' % i for i in mac64]), 16) + mac64_addr = netaddr.IPAddress(int_addr) + maskIP = netaddr.IPNetwork(prefix).ip + return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).\ + format() + except TypeError: + raise TypeError(_("Bad mac for to_global_ipv6: %s" % mac)) def to_mac(ipv6_address): -- cgit From b3a8c70304672abe9b461c6cfeed3e8b517ca0b6 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 16:56:54 -0500 Subject: Added docstring --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 38cdb8021..bf1aa4a91 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -335,6 +335,7 @@ utcnow.override_time = None def is_then_greater(then, seconds): + """Return True of 'then' is greater than 'seconds'""" if utcnow() - then > datetime.timedelta(seconds=seconds): return True else: -- cgit From 5170e8b5dd96cf8c7bb91e84203cfaebb099af46 Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Thu, 24 Mar 2011 00:56:56 +0300 Subject: small fix --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 44234813f..fabc01532 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -317,7 +317,7 @@ def to_global_ipv6(prefix, mac): return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).\ format() except TypeError: - raise TypeError(_("Bad mac for to_global_ipv6: %s" % mac)) + raise TypeError(_("Bad mac for to_global_ipv6: %s") % mac) def to_mac(ipv6_address): -- cgit From a12b6f0a0808fba5541723a537118447b55b69ad Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 17:15:41 -0500 Subject: Better method name --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index bf1aa4a91..04b6c9778 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -334,9 +334,9 @@ def utcnow(): utcnow.override_time = None -def is_then_greater(then, seconds): - """Return True of 'then' is greater than 'seconds'""" - if utcnow() - then > datetime.timedelta(seconds=seconds): +def is_older_than(before, seconds): + """Return True if before is older than 'seconds'""" + if utcnow() - before > datetime.timedelta(seconds=seconds): return True else: return False -- cgit From cf70a1a76dfa0456ea6230eaa014fa98e7ddd464 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 24 Mar 2011 10:30:09 -0500 Subject: Small refactor --- nova/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 03a6e8095..6042a5332 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -336,11 +336,8 @@ utcnow.override_time = None def is_older_than(before, seconds): - """Return True if before is older than 'seconds'""" - if utcnow() - before > datetime.timedelta(seconds=seconds): - return True - else: - return False + """Return True if before is older than seconds""" + return utcnow() - before > datetime.timedelta(seconds=seconds) def utcnow_ts(): -- cgit From c3b98443263de944aa54ae4948330b6cfb9a02a6 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Thu, 24 Mar 2011 23:00:16 +0300 Subject: style and spacing fixed --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index d3375abc3..29e33ad9e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -316,7 +316,7 @@ def to_global_ipv6(prefix, mac): mac64_addr = netaddr.IPAddress(int_addr) maskIP = netaddr.IPNetwork(prefix).ip return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).\ - format() + format() except TypeError: raise TypeError(_("Bad mac for to_global_ipv6: %s") % mac) -- cgit