From 2337fab0979b72bbc7e7730e94518a0e835a2751 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Mon, 25 Oct 2010 03:45:19 +0900 Subject: part way through porting the codebase off of twisted this provides a very basic eventlet-based service replacement for the twistd-based services, a replacement for task.LoopingCall also adds nova-combined with the goal of running a single service when doing local testing and dev --- nova/utils.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 7683fc9f4..a219e47bf 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -29,6 +29,9 @@ import subprocess import socket import sys +from eventlet import event +from eventlet import greenthread + from twisted.internet.threads import deferToThread from nova import exception @@ -212,3 +215,36 @@ def deferredToThread(f): def g(*args, **kwargs): return deferToThread(f, *args, **kwargs) return g + + +class LoopingCall(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + + def start(self, interval, now=True): + self._running = True + done = event.Event() + def _inner(): + if not now: + greenthread.sleep(interval) + try: + while self._running: + self.f(*self.args, **self.kw) + greenthread.sleep(interval) + except Exception: + logging.exception('hhmm') + done.send_exception(*sys.exc_info()) + return + + done.send(True) + + greenthread.spawn(_inner) + return done + + def stop(self): + self._running = False + + -- cgit From a7a24e2ac54b88ba09afe5966806f42181784e9d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 23 Nov 2010 21:48:32 +0000 Subject: add vpn ping and optimize vpn list --- nova/utils.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index e7892a212..b3297acd9 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -27,6 +27,7 @@ import os import random import subprocess import socket +import struct import sys from xml.sax import saxutils @@ -61,6 +62,51 @@ def import_object(import_str): return cls() +def vpn_ping(address, port, timeout=0.05, session_id=None): + """Sends a vpn negotiation packet and returns the server session. + + Returns False on a failure. Basic packet structure is below. + + Client packet (14 bytes):: + 0 1 8 9 13 + +-+--------+-----+ + |x| cli_id |?????| + +-+--------+-----+ + x = packet identifier 0x38 + cli_id = 64 bit identifier + ? = unknown, probably flags/padding + + Server packet (26 bytes):: + 0 1 8 9 13 14 21 2225 + +-+--------+-----+--------+----+ + |x| srv_id |?????| cli_id |????| + +-+--------+-----+--------+----+ + x = packet identifier 0x40 + cli_id = 64 bit identifier + ? = unknown, probably flags/padding + bit 9 was 1 and the rest were 0 in testing + """ + if session_id is None: + session_id = random.randint(0, 0xffffffffffffffff) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + data = struct.pack("!BQxxxxxx", 0x38, session_id) + sock.sendto(data, (address, port)) + sock.settimeout(timeout) + try: + received = sock.recv(2048) + except socket.timeout: + return False + finally: + sock.close() + fmt = "!BQxxxxxQxxxx" + if len(received) != struct.calcsize(fmt): + print struct.calcsize(fmt) + return False + (identifier, server_sess, client_sess) = struct.unpack(fmt, received) + if identifier == 0x40 and client_sess == session_id: + return server_sess + + def fetchfile(url, target): logging.debug("Fetching %s" % url) # c = pycurl.Curl() -- cgit From 3c85f1b7ed593a2d4d126a34241f217da5cf7ce6 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 8 Dec 2010 17:18:27 -0800 Subject: intermediate commit to checkpoint progress all relevant tests are passing except volume, next step is volume manager fixery --- nova/utils.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 66047ae8b..2c43203d8 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -34,8 +34,6 @@ from xml.sax import saxutils from eventlet import event from eventlet import greenthread -from twisted.internet.threads import deferToThread - from nova import exception from nova import flags from nova.exception import ProcessExecutionError @@ -78,7 +76,7 @@ def fetchfile(url, target): def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - logging.debug("Running cmd: %s", cmd) + logging.debug("Running cmd (subprocess): %s", cmd) env = os.environ.copy() if addl_env: env.update(addl_env) @@ -98,6 +96,10 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): stdout=stdout, stderr=stderr, cmd=cmd) + # NOTE(termie): this appears to be necessary to let the subprocess call + # clean something up in between calls, without it two + # execute calls in a row hangs the second one + greenthread.sleep(0) return result @@ -126,13 +128,14 @@ def debug(arg): def runthis(prompt, cmd, check_exit_code=True): logging.debug("Running %s" % (cmd)) - exit_code = subprocess.call(cmd.split(" ")) - logging.debug(prompt % (exit_code)) - if check_exit_code and exit_code != 0: - raise ProcessExecutionError(exit_code=exit_code, - stdout=None, - stderr=None, - cmd=cmd) + rv, err = execute(cmd, check_exit_code=check_exit_code) + #exit_code = subprocess.call(cmd.split(" ")) + #logging.debug(prompt % (exit_code)) + #if check_exit_code and exit_code != 0: + # raise ProcessExecutionError(exit_code=exit_code, + # stdout=None, + # stderr=None, + # cmd=cmd) def generate_uid(topic, size=8): -- cgit From 3b376b8ad167e91119e21180bbff41eceef22e26 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 15:19:56 -0800 Subject: get service unittests runnning again --- nova/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 2c43203d8..ddbe0479e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -253,13 +253,18 @@ class LoopingCall(object): return done.send(True) + + self.done = done greenthread.spawn(_inner) - return done + return self.done def stop(self): self._running = False + def wait(self): + return self.done.wait() + def xhtml_escape(value): """Escapes a string so it is valid within XML or XHTML. -- cgit From a1640f352806ee12f6b485a8d69a65bd42b51411 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 16:05:13 -0800 Subject: formatting and naming cleanup --- nova/utils.py | 7 ------- 1 file changed, 7 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index ddbe0479e..22bf5d8cf 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -129,13 +129,6 @@ def debug(arg): def runthis(prompt, cmd, check_exit_code=True): logging.debug("Running %s" % (cmd)) rv, err = execute(cmd, check_exit_code=check_exit_code) - #exit_code = subprocess.call(cmd.split(" ")) - #logging.debug(prompt % (exit_code)) - #if check_exit_code and exit_code != 0: - # raise ProcessExecutionError(exit_code=exit_code, - # stdout=None, - # stderr=None, - # cmd=cmd) def generate_uid(topic, size=8): -- cgit From 12802a76c775a35e9d5a651bf896cfa25bec547f Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Sat, 11 Dec 2010 15:23:40 -0500 Subject: First round of i18n-ifying strings in Nova --- nova/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 142584df8..f6f03b555 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -49,7 +49,7 @@ def import_class(import_str): __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError): - raise exception.NotFound('Class %s cannot be found' % class_str) + raise exception.NotFound(_('Class %s cannot be found') % class_str) def import_object(import_str): @@ -63,7 +63,7 @@ def import_object(import_str): def fetchfile(url, target): - logging.debug("Fetching %s" % url) + logging.debug(_("Fetching %s") % url) # c = pycurl.Curl() # fp = open(target, "wb") # c.setopt(c.URL, url) @@ -75,7 +75,7 @@ def fetchfile(url, target): def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - logging.debug("Running cmd: %s", cmd) + logging.debug(_("Running cmd: %s"), cmd) env = os.environ.copy() if addl_env: env.update(addl_env) @@ -88,7 +88,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): result = obj.communicate() obj.stdin.close() if obj.returncode: - logging.debug("Result was %s" % (obj.returncode)) + logging.debug(_("Result was %s") % (obj.returncode)) if check_exit_code and obj.returncode != 0: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, @@ -122,7 +122,7 @@ def debug(arg): def runthis(prompt, cmd, check_exit_code=True): - logging.debug("Running %s" % (cmd)) + logging.debug(_("Running %s") % (cmd)) exit_code = subprocess.call(cmd.split(" ")) logging.debug(prompt % (exit_code)) if check_exit_code and exit_code != 0: @@ -161,7 +161,7 @@ def get_my_ip(): csock.close() return addr except socket.gaierror as ex: - logging.warn("Couldn't get IP, using 127.0.0.1 %s", ex) + logging.warn(_("Couldn't get IP, using 127.0.0.1 %s"), ex) return "127.0.0.1" @@ -205,7 +205,7 @@ class LazyPluggable(object): if not self.__backend: backend_name = self.__pivot.value if backend_name not in self.__backends: - raise exception.Error('Invalid backend: %s' % backend_name) + raise exception.Error(_('Invalid backend: %s') % backend_name) backend = self.__backends[backend_name] if type(backend) == type(tuple()): -- cgit From a2a8406b5d793545c8ecb359e18b80bba618c509 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 14 Dec 2010 16:05:39 -0800 Subject: updates per review --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 22bf5d8cf..5f2d47202 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -241,7 +241,7 @@ class LoopingCall(object): self.f(*self.args, **self.kw) greenthread.sleep(interval) except Exception: - logging.exception('hhmm') + logging.exception('in looping call') done.send_exception(*sys.exc_info()) return -- cgit From dd4ee43cc2042299ed7a56b4690999fa1df120a1 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 15 Dec 2010 11:23:33 -0800 Subject: clean up tests and add overriden time method to utils --- nova/utils.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 142584df8..048a9d974 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -21,7 +21,6 @@ System-level utilities and helper functions. """ import datetime -import functools import inspect import logging import os @@ -29,6 +28,7 @@ import random import subprocess import socket import sys +import time from xml.sax import saxutils from twisted.internet.threads import deferToThread @@ -165,13 +165,51 @@ def get_my_ip(): return "127.0.0.1" +def utcnow(): + """Overridable version of datetime.datetime.utcnow.""" + if utcnow.override_time: + return utcnow.override_time + return datetime.datetime.utcnow() + + +utcnow.override_time = None + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return time.mktime(utcnow().timetuple()) + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """Override utils.utcnow to return a constant time.""" + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overriden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overriden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + def isotime(at=None): + """Returns iso formatted utcnow.""" if not at: - at = datetime.datetime.utcnow() + at = utcnow() return at.strftime(TIME_FORMAT) def parse_isotime(timestr): + """Turn an iso formatted time back into a datetime""" return datetime.datetime.strptime(timestr, TIME_FORMAT) -- cgit From 9b049acc27d477a1ab9e13c9e064e59d8bd0a3ae Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 16 Dec 2010 10:52:30 -0800 Subject: pep8 fixes --- nova/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 5f2d47202..ea1f04ca7 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -233,6 +233,7 @@ class LoopingCall(object): def start(self, interval, now=True): self._running = True done = event.Event() + def _inner(): if not now: greenthread.sleep(interval) @@ -244,14 +245,14 @@ class LoopingCall(object): logging.exception('in looping call') done.send_exception(*sys.exc_info()) return - + done.send(True) self.done = done - + greenthread.spawn(_inner) return self.done - + def stop(self): self._running = False -- cgit From 86f71493fa5a02762bc7c56308c85b9182913efb Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 17 Dec 2010 00:43:18 +0000 Subject: move some flags around --- nova/utils.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 11160c118..082a42acf 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -21,7 +21,6 @@ System-level utilities and helper functions. """ import datetime -import functools import inspect import logging import os @@ -36,11 +35,9 @@ from eventlet import event from eventlet import greenthread from nova import exception -from nova import flags from nova.exception import ProcessExecutionError -FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" @@ -197,8 +194,6 @@ def last_octet(address): def get_my_ip(): """Returns the actual ip of the local machine.""" - if getattr(FLAGS, 'fake_tests', None): - return '127.0.0.1' try: csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) csock.connect(('8.8.8.8', 80)) -- cgit