merge master into the export_methods branch

12 files changed, 108 insertions, 60 deletions

diff --git a/func/jobthing.py b/func/jobthing.py
index cc6808b..d5fc2a8 100644
--- a/func/jobthing.py
+++ b/func/jobthing.py
@@ -170,8 +170,9 @@ def batch_run(pool, callback, nforks,**extra_args):
         # kick off the job
         results = forkbomb.batch_run(pool, callback, nforks)
         
-        # we now have a list of job id's for each minion, kill the task
+        # write job IDs to the state file on overlord 
         __update_status(job_id, JOB_ID_PARTIAL, results)
+        # we now have a list of job id's for each minion, kill the task
         os._exit(0)
 
 def minion_async_run(retriever, method, args):
@@ -247,11 +248,11 @@ def job_status(jobid, client_class=None):
                 some_missing = True
 
         if some_missing or not interim_results:
-            if partial_results:
-                __update_status(jobid, JOB_ID_PARTIAL, partial_results)
             return (JOB_ID_PARTIAL, partial_results)
         
         else:
+            # Save partial results in state file so next time we don't
+            # call minions to get status.
             if partial_results:
                 __update_status(jobid,JOB_ID_FINISHED, partial_results)
             return (JOB_ID_FINISHED, partial_results)
diff --git a/func/minion/modules/func_module.py b/func/minion/modules/func_module.py
index 56f21f9..a405198 100644
--- a/func/minion/modules/func_module.py
+++ b/func/minion/modules/func_module.py
@@ -15,6 +15,7 @@ import inspect
 from func import logger
 from certmaster.config import read_config, BaseConfig
 from func.commonconfig import FuncdConfig
+from func.utils import is_public_valid_method
 from func.minion.func_arg import * #the arg getter stuff
 
 class FuncModule(object):
@@ -85,10 +86,7 @@ class FuncModule(object):
         return self.description
 
     def __is_public_valid_method(self,attr):
-        if inspect.ismethod(getattr(self, attr)) and attr[0] != '_' and\
-                attr != 'register_rpc' and attr!='register_method_args':
-                    return True
-        return False
+        return is_public_valid_method(self, attr, blacklist=['register_rpc', 'register_method_args'])
 
     def __get_method_args(self):
         """
diff --git a/func/minion/server.py b/func/minion/server.py
index ede0640..9bd77b0 100755
--- a/func/minion/server.py
+++ b/func/minion/server.py
@@ -33,7 +33,7 @@ import func.jobthing as jobthing
 # our modules
 import AuthedXMLRPCServer
 import codes
-import module_loader
+import func.module_loader as module_loader
 import func.minion.acls as acls_mod
 
 from certmaster import utils
diff --git a/func/minion/module_loader.py b/func/module_loader.py
index 3068ea8..2aa1378 100755
--- a/func/minion/module_loader.py
+++ b/func/module_loader.py
@@ -16,6 +16,7 @@
 import distutils.sysconfig
 import os
 import sys
+import inspect
 from gettext import gettext
 _ = gettext
 
@@ -24,6 +25,7 @@ logger = logger.Logger().logger
 
 from inspect import isclass
 from func.minion.modules import func_module
+from func.utils import is_public_valid_method
 
 def module_walker(topdir):
     module_files = []
@@ -36,13 +38,22 @@ def module_walker(topdir):
                 # in the module name, and foo..bar doesnt work -akl
                 module_files.append(os.path.normpath("%s/%s" % (root, filename)))
 
-
     return module_files
 
-def load_modules(blacklist=None):
-
-    module_file_path="%s/func/minion/modules/" % distutils.sysconfig.get_python_lib()
-    mod_path="%s/func/minion/"  % distutils.sysconfig.get_python_lib()
+def load_methods(path, main_class, parent_class=None):
+    methods = {}
+    modules = load_modules(path, main_class, parent_class=parent_class)
+    for x in modules.keys():
+        for method in dir(modules[x]):
+            if is_public_valid_method(modules[x], method):
+                methods["%s.%s" % (x,method)]=getattr(modules[x], method)
+    return methods
+
+def load_modules(path='func/minion/modules/', main_class=func_module.FuncModule, blacklist=None, parent_class=None):
+    python_path = distutils.sysconfig.get_python_lib()
+    module_file_path = "%s/%s" % (python_path, path)
+    (mod_path, mod_dir) = os.path.split(os.path.normpath(module_file_path))
+    mod_dir = "func."+module_file_path[len(python_path+'/func/'):].replace("/",".")
 
     sys.path.insert(0, mod_path)
     mods = {}
@@ -80,12 +91,15 @@ def load_modules(blacklist=None):
 
         try:
             # Auto-detect and load all FuncModules
-            blip =  __import__("modules.%s" % ( mod_imp_name), globals(), locals(), [mod_imp_name])
+            blip =  __import__("%s%s" % ( mod_dir,mod_imp_name), globals(), locals(), [mod_imp_name])
             for obj in dir(blip):
                 attr = getattr(blip, obj)
-                if isclass(attr) and issubclass(attr, func_module.FuncModule):
+                if isclass(attr) and issubclass(attr, main_class):
                     logger.debug("Loading %s module" % attr)
-                    mods[mod_imp_name] = attr()
+                    if parent_class:
+                        mods[mod_imp_name] = attr(parent_class)
+                    else:
+                        mods[mod_imp_name] = attr()
 
         except ImportError, e:
             # A module that raises an ImportError is (for now) simply not loaded.
@@ -101,7 +115,6 @@ def load_modules(blacklist=None):
 
     return mods
 
-
 if __name__ == "__main__":
 
     module_file_path = "/usr/lib/python2.5/site-packages/func/minion/modules/"
diff --git a/func/overlord/client.py b/func/overlord/client.py
index b02438d..54777e3 100755
--- a/func/overlord/client.py
+++ b/func/overlord/client.py
@@ -31,6 +31,8 @@ import delegation_tools as dtools
 import func.forkbomb as forkbomb
 import func.jobthing as jobthing
 from func.CommonErrors import *
+import func.module_loader as module_loader
+from func.overlord import overlord_module
 
 # ===================================
 # defaults
@@ -192,6 +194,8 @@ class Overlord(object):
     
         if init_ssl:
             self.setup_ssl()
+
+        self.methods = module_loader.load_methods('func/overlord/modules/', overlord_module.BaseModule, self)
             
     def setup_ssl(self, client_key=None, client_cert=None, ca=None):
         # defaults go:
@@ -285,7 +289,13 @@ class Overlord(object):
         If Overlord() was constructed with noglobs=True, the return is instead
         just a single value, not a hash.
         """
-        
+
+        if module == "local":
+            if method in self.methods.keys():
+                return self.methods[method](*args)
+            else:
+                raise AttributeError("no such method")
+
         if not self.delegate: #delegation is turned off, so run normally
             return self.run_direct(module, method, args, nforks)
         
diff --git a/func/overlord/cmd_modules/call.py b/func/overlord/cmd_modules/call.py
index c3c93b5..e1674fe 100644
--- a/func/overlord/cmd_modules/call.py
+++ b/func/overlord/cmd_modules/call.py
@@ -16,7 +16,6 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 import optparse
 import pprint
 import xmlrpclib
-import time
 import sys
 
 from func.overlord import client
@@ -156,27 +155,13 @@ class Call(base_command.BaseCommand):
             return async_results
 
         if self.options.async:
-            partial = {}
+            self.partial = {}
             if self.options.nopoll:
                 print "JOB_ID:", pprint.pformat(results)
                 return results
             else:
-                async_done = False
-                while not async_done:
-                    time.sleep(3)
-                    (return_code, async_results) = self.overlord_obj.job_status(results)
-                    if return_code == jobthing.JOB_ID_RUNNING:
-                        time.sleep(0.1)
-                    elif return_code == jobthing.JOB_ID_FINISHED:
-                        async_done = True
-                        partial = self.print_partial_results(partial, async_results, self.options.sort)
-                        return partial
-                    elif return_code == jobthing.JOB_ID_PARTIAL:
-                        if not self.options.sort:
-                            partial = self.print_partial_results(partial, async_results)
-                    else:
-                        sys.stderr.write("Async error")
-                        return 0
+                
+                return self.overlord_obj.local.utils.async_poll(results, self.print_results)
 
         # dump the return code stuff atm till we figure out the right place for it
         foo =  self.format_return(results)
@@ -185,13 +170,6 @@ class Call(base_command.BaseCommand):
         # nothing really makes use of this atm -akl
         return results
 
-    def print_partial_results(self, old, new, sort=0):
-        diff = dict([(k, v) for k, v in new.iteritems() if k not in old])
-        if len(diff) > 0:
-            iter=diff.iteritems()
-            if sort:
-                iter=sorted(iter)
-            for res in iter:
-                print self.format_return(res)
-            return new
-        return old
+    def print_results(self, res):
+        for i in res.iteritems():
+            print self.format_return(i)
diff --git a/func/overlord/func_command.py b/func/overlord/func_command.py
index 90de0fc..794b0ba 100644
--- a/func/overlord/func_command.py
+++ b/func/overlord/func_command.py
@@ -16,29 +16,21 @@ import sys
 
 
 import command
+import func.module_loader as module_loader
 
-#FIXME: need a plug-in runtime module loader here
-from cmd_modules import call
-from cmd_modules import show
-from cmd_modules import copyfile
-from cmd_modules import listminions
-from cmd_modules import ping
-from cmd_modules import check
-
-from func.overlord import client
+from func.overlord import client,base_command
 
 class FuncCommandLine(command.Command):
 
     name = "func"
     usage = "func [--options] \"hostname glob\" module method [arg1] [arg2] ... "
 
-    subCommandClasses = [
-        call.Call, show.Show, copyfile.CopyFile, 
-        listminions.ListMinions, ping.Ping, check.CheckAction
-    ]
+    subCommandClasses = []
 
     def __init__(self):
-
+        modules = module_loader.load_modules('func/overlord/cmd_modules/', base_command.BaseCommand)
+        for x in modules.keys():
+           self.subCommandClasses.append(modules[x].__class__)
         command.Command.__init__(self)
 
     def do(self, args):
diff --git a/func/overlord/modules/__init__.py b/func/overlord/modules/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/func/overlord/modules/__init__.py
diff --git a/func/overlord/modules/test.py b/func/overlord/modules/test.py
new file mode 100644
index 0000000..09f7a33
--- /dev/null
+++ b/func/overlord/modules/test.py
@@ -0,0 +1,5 @@
+from func.overlord import overlord_module
+
+class test(overlord_module.BaseModule):
+    def return1(self):
+        return 1
diff --git a/func/overlord/modules/utils.py b/func/overlord/modules/utils.py
new file mode 100644
index 0000000..f18fd50
--- /dev/null
+++ b/func/overlord/modules/utils.py
@@ -0,0 +1,40 @@
+import time
+
+from func.overlord import overlord_module
+import func.jobthing as jobthing
+
+class utils(overlord_module.BaseModule):
+    def __diff_dicts(self, a, b):
+        return dict([(k, v) for k, v in a.iteritems() if k not in b])
+        
+
+    def async_poll(self, job_id, partial_func=None, interval=0.5):
+        async_done = False
+        partial = {}
+        while not async_done:
+            (return_code, async_results) = self.parent.job_status(job_id)
+            if return_code == jobthing.JOB_ID_RUNNING:
+                pass
+            elif return_code == jobthing.JOB_ID_FINISHED:
+                async_done = True
+                if partial_func:
+                    diff = self.__diff_dicts(async_results, partial)
+                    if len(diff) > 0:
+                        partial_func(diff)
+                return async_results
+            elif return_code == jobthing.JOB_ID_PARTIAL:
+                pass
+                if partial_func:
+                    diff = self.__diff_dicts(async_results, partial)
+                    if len(diff) > 0:
+                        partial_func(diff)
+                        partial=async_results
+            else:
+                #FIXME -- raise exception instead of printing
+                print "Async error", return_code, async_results
+                return 0
+
+            time.sleep(interval)
+
+    def list_minions(self):
+       return self.parent.minions
diff --git a/func/overlord/overlord_module.py b/func/overlord/overlord_module.py
new file mode 100644
index 0000000..9d01728
--- /dev/null
+++ b/func/overlord/overlord_module.py
@@ -0,0 +1,3 @@
+class BaseModule:
+    def __init__(self, parent=None):
+        self.parent=parent
diff --git a/func/utils.py b/func/utils.py
index 628694d..891f459 100755
--- a/func/utils.py
+++ b/func/utils.py
@@ -15,6 +15,7 @@ import string
 import sys
 import traceback
 import socket
+import inspect
 
 REMOTE_ERROR = "REMOTE_ERROR"
 
@@ -52,3 +53,10 @@ def get_formated_jobid(**id_pack):
     job_id = "".join([glob,"-",module,"-",method,"-",pprint.pformat(time.time())])
     return job_id
 
+def is_public_valid_method(obj, attr, blacklist=[]):
+    if inspect.ismethod(getattr(obj, attr)) and attr[0] != '_':
+        for b in blacklist:
+            if attr==b:
+                return False
+        return True
+    return False