diff options
Diffstat (limited to 'func/overlord')
28 files changed, 1637 insertions, 0 deletions
diff --git a/func/overlord/.forkbomb.py.swp b/func/overlord/.forkbomb.py.swp Binary files differnew file mode 100644 index 0000000..242b6f4 --- /dev/null +++ b/func/overlord/.forkbomb.py.swp diff --git a/func/overlord/Makefile b/func/overlord/Makefile new file mode 100755 index 0000000..f2bc6c4 --- /dev/null +++ b/func/overlord/Makefile @@ -0,0 +1,18 @@ + + +PYFILES = $(wildcard *.py) + +PYCHECKER = /usr/bin/pychecker +PYFLAKES = /usr/bin/pyflakes + +clean:: + @rm -fv *.pyc *~ .*~ *.pyo + @find . -name .\#\* -exec rm -fv {} \; + @rm -fv *.rpm + + +pychecker:: + @$(PYCHECKER) $(PYFILES) || exit 0 + +pyflakes:: + @$(PYFLAKES) $(PYFILES) || exit 0 diff --git a/func/overlord/__init__.py b/func/overlord/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/func/overlord/__init__.py diff --git a/func/overlord/__init__.pyc b/func/overlord/__init__.pyc Binary files differnew file mode 100644 index 0000000..f74bc59 --- /dev/null +++ b/func/overlord/__init__.pyc diff --git a/func/overlord/client.py b/func/overlord/client.py new file mode 100755 index 0000000..cf1009c --- /dev/null +++ b/func/overlord/client.py @@ -0,0 +1,336 @@ +## +## func command line interface & client lib +## +## Copyright 2007, Red Hat, Inc +## Michael DeHaan <mdehaan@redhat.com> +## +AUTHORS +## +## This software may be freely redistributed under the terms of the GNU +## general public license. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +## + +import sys +import glob +import os + +from func.commonconfig import CMConfig +from func.config import read_config, CONFIG_FILE + +import sslclient + +import command +import groups +import func.forkbomb as forkbomb +import func.jobthing as jobthing +import func.utils as utils +from func.CommonErrors import * + +# =================================== +# defaults +# TO DO: some of this may want to come from config later + +DEFAULT_PORT = 51234 +FUNC_USAGE = "Usage: %s [ --help ] [ --verbose ] target.example.org module method arg1 [...]" + +# =================================== + +class CommandAutomagic(object): + """ + This allows a client object to act as if it were one machine, when in + reality it represents many. + """ + + def __init__(self, clientref, base, nforks=1): + self.base = base + self.clientref = clientref + self.nforks = nforks + + def __getattr__(self,name): + base2 = self.base[:] + base2.append(name) + return CommandAutomagic(self.clientref, base2, self.nforks) + + def __call__(self, *args): + if not self.base: + raise AttributeError("something wrong here") + if len(self.base) < 2: + raise AttributeError("no method called: %s" % ".".join(self.base)) + module = self.base[0] + method = ".".join(self.base[1:]) + return self.clientref.run(module,method,args,nforks=self.nforks) + + +def get_groups(): + group_class = groups.Groups() + return group_class.get_groups() + + +def get_hosts_by_groupgoo(groups, groupgoo): + group_gloobs = groupgoo.split(':') + hosts = [] + for group_gloob in group_gloobs: + if not group_gloob[0] == "@": + continue + if groups.has_key(group_gloob[1:]): + hosts = hosts + groups[group_gloob[1:]] + else: + print "group %s not defined" % group_gloob + return hosts + +# =================================== +# this is a module level def so we can use it and isServer() from +# other modules with a Client class +def expand_servers(spec, port=51234, noglobs=None, verbose=None, just_fqdns=False): + """ + Given a regex/blob of servers, expand to a list + of server ids. + """ + + + # FIXME: we need to refactor expand_servers, it seems to do + # weird things, reload the config and groups config everytime it's + # called for one, which may or may not be bad... -akl + config = read_config(CONFIG_FILE, CMConfig) + + if noglobs: + if not just_fqdns: + return [ "https://%s:%s" % (spec, port) ] + else: + return spec + + group_dict = get_groups() + + all_hosts = [] + all_certs = [] + seperate_gloobs = spec.split(";") + + new_hosts = get_hosts_by_groupgoo(group_dict, spec) + + seperate_gloobs = spec.split(";") + seperate_gloobs = seperate_gloobs + new_hosts + for each_gloob in seperate_gloobs: + actual_gloob = "%s/%s.cert" % (config.certroot, each_gloob) + certs = glob.glob(actual_gloob) + for cert in certs: + all_certs.append(cert) + host = cert.replace(config.certroot,"")[1:-5] + all_hosts.append(host) + + all_urls = [] + for x in all_hosts: + if not just_fqdns: + all_urls.append("https://%s:%s" % (x, port)) + else: + all_urls.append(x) + + if verbose and len(all_urls) == 0: + sys.stderr.write("no hosts matched\n") + + return all_urls + + +# does the hostnamegoo actually expand to anything? +def isServer(server_string): + servers = expand_servers(server_string) + if len(servers) > 0: + return True + return False + + +class Client(object): + + def __init__(self, server_spec, port=DEFAULT_PORT, interactive=False, + verbose=False, noglobs=False, nforks=1, config=None, async=False, init_ssl=True): + """ + Constructor. + @server_spec -- something like "*.example.org" or "foosball" + @port -- is the port where all funcd processes should be contacted + @verbose -- whether to print unneccessary things + @noglobs -- specifies server_spec is not a glob, and run should return single values + @config -- optional config object + """ + self.config = config + if config is None: + self.config = read_config(CONFIG_FILE, CMConfig) + + + self.server_spec = server_spec + self.port = port + self.verbose = verbose + self.interactive = interactive + self.noglobs = noglobs + self.nforks = nforks + self.async = async + + self.servers = expand_servers(self.server_spec, port=self.port, noglobs=self.noglobs,verbose=self.verbose) + + if init_ssl: + self.setup_ssl() + + def setup_ssl(self, client_key=None, client_cert=None, ca=None): + # defaults go: + # certmaster key, cert, ca + # funcd key, cert, ca + # raise FuncClientError + ol_key = '%s/funcmaster.key' % self.config.cadir + ol_crt = '%s/funcmaster.crt' % self.config.cadir + myname = utils.get_hostname() + # maybe /etc/pki/func is a variable somewhere? + fd_key = '/etc/pki/func/%s.pem' % myname + fd_crt = '/etc/pki/func/%s.cert' % myname + self.ca = '%s/funcmaster.crt' % self.config.cadir + if client_key and client_cert and ca: + if (os.access(client_key, os.R_OK) and os.access(client_cert, os.R_OK) + and os.access(ca, os.R_OK)): + self.key = client_key + self.cert = client_cert + self.ca = ca + # otherwise fall through our defaults + elif os.access(ol_key, os.R_OK) and os.access(ol_crt, os.R_OK): + self.key = ol_key + self.cert = ol_crt + elif os.access(fd_key, os.R_OK) and os.access(fd_crt, os.R_OK): + self.key = fd_key + self.cert = fd_crt + else: + raise Func_Client_Exception, 'Cannot read ssl credentials: ssl, cert, ca' + + + + + def __getattr__(self, name): + """ + This getattr allows manipulation of the object as if it were + a XMLRPC handle to a single machine, when in reality it is a handle + to an unspecified number of machines. + + So, it enables stuff like this: + + Client("*.example.org").yum.install("foo") + + # WARNING: any missing values in Client's source will yield + # strange errors with this engaged. Be aware of that. + """ + + return CommandAutomagic(self, [name], self.nforks) + + # ----------------------------------------------- + + def job_status(self, jobid): + """ + Use this to acquire status from jobs when using run with async client handles + """ + return jobthing.job_status(jobid, client_class=Client) + + # ----------------------------------------------- + + def run(self, module, method, args, nforks=1): + """ + Invoke a remote method on one or more servers. + Run returns a hash, the keys are server names, the values are the + returns. + + The returns may include exception objects. + If Client() was constructed with noglobs=True, the return is instead + just a single value, not a hash. + """ + + results = {} + + def process_server(bucketnumber, buckets, server): + + conn = sslclient.FuncServer(server, self.key, self.cert, self.ca ) + # conn = xmlrpclib.ServerProxy(server) + + if self.interactive: + sys.stderr.write("on %s running %s %s (%s)\n" % (server, + module, method, ",".join(args))) + + # FIXME: support userland command subclassing only if a module + # is present, otherwise run as follows. -- MPD + + try: + # thats some pretty code right there aint it? -akl + # we can't call "call" on s, since thats a rpc, so + # we call gettatr around it. + meth = "%s.%s" % (module, method) + + # async calling signature has an "imaginary" prefix + # so async.abc.def does abc.def as a background task. + # see Wiki docs for details + if self.async: + meth = "async.%s" % meth + + # this is the point at which we make the remote call. + retval = getattr(conn, meth)(*args[:]) + + if self.interactive: + print retval + except Exception, e: + (t, v, tb) = sys.exc_info() + retval = utils.nice_exception(t,v,tb) + if self.interactive: + sys.stderr.write("remote exception on %s: %s\n" % + (server, str(e))) + + if self.noglobs: + return retval + else: + left = server.rfind("/")+1 + right = server.rfind(":") + server_name = server[left:right] + return (server_name, retval) + + if not self.noglobs: + if self.nforks > 1 or self.async: + # using forkbomb module to distribute job over multiple threads + if not self.async: + results = forkbomb.batch_run(self.servers, process_server, nforks) + else: + results = jobthing.batch_run(self.servers, process_server, nforks) + else: + # no need to go through the fork code, we can do this directly + results = {} + for x in self.servers: + (nkey,nvalue) = process_server(0, 0, x) + results[nkey] = nvalue + else: + # globbing is not being used, but still need to make sure + # URI is well formed. + expanded = expand_servers(self.server_spec, port=self.port, noglobs=True, verbose=self.verbose)[0] + results = process_server(0, 0, expanded) + + return results + + # ----------------------------------------------- + + def cli_return(self,results): + """ + As the return code list could return strings and exceptions + and all sorts of crazy stuff, reduce it down to a simple + integer return. It may not be useful but we need one. + """ + numbers = [] + for x in results.keys(): + # faults are the most important + if type(x) == Exception: + return -911 + # then pay attention to numbers + if type(x) == int: + numbers.append(x) + + # if there were no numbers, assume 0 + if len(numbers) == 0: + return 0 + + # if there were numbers, return the highest + # (presumably the worst error code + max = -9999 + for x in numbers: + if x > max: + max = x + return max diff --git a/func/overlord/client.pyc b/func/overlord/client.pyc Binary files differnew file mode 100644 index 0000000..5c7874e --- /dev/null +++ b/func/overlord/client.pyc diff --git a/func/overlord/cmd_modules/__init__.py b/func/overlord/cmd_modules/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/func/overlord/cmd_modules/__init__.py diff --git a/func/overlord/cmd_modules/__init__.pyc b/func/overlord/cmd_modules/__init__.pyc Binary files differnew file mode 100644 index 0000000..287b354 --- /dev/null +++ b/func/overlord/cmd_modules/__init__.pyc diff --git a/func/overlord/cmd_modules/call.py b/func/overlord/cmd_modules/call.py new file mode 100644 index 0000000..7add5bf --- /dev/null +++ b/func/overlord/cmd_modules/call.py @@ -0,0 +1,114 @@ +""" +call func method invoker + +Copyright 2007, Red Hat, Inc +see AUTHORS + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + + +import optparse +import pprint +import xmlrpclib + +from func.overlord import command +from func.overlord import client + +DEFAULT_PORT = 51234 +DEFAULT_FORKS = 1 + +class Call(client.command.Command): + name = "call" + usage = "call module method name arg1 arg2..." + def addOptions(self): + self.parser.add_option("-v", "--verbose", dest="verbose", + action="store_true") + self.parser.add_option("-x", "--xmlrpc", dest="xmlrpc", + help="output return data in XMLRPC format", + action="store_true") + self.parser.add_option("", "--raw", dest="rawprint", + help="output return data using Python print", + action="store_true") + self.parser.add_option("-j", "--json", dest="json", + help="output return data using JSON", + action="store_true") + self.parser.add_option("-p", "--port", dest="port", + default=DEFAULT_PORT) + self.parser.add_option("-f", "--forks", dest="forks", + help="how many parallel processes? (default 1)", + default=DEFAULT_FORKS) + + def handleOptions(self, options): + self.options = options + + self.verbose = options.verbose + self.port = options.port + + # I'm not really a fan of the "module methodname" approach + # but we'll keep it for now -akl + + def parse(self, argv): + self.argv = argv + + return command.Command.parse(self, argv) + + + def format_return(self, data): + """ + The call module supports multiple output return types, the default is pprint. + """ + + if self.options.xmlrpc: + return xmlrpclib.dumps((data,"")) + + if self.options.json: + try: + import simplejson + return simplejson.dumps(data) + except ImportError: + print "WARNING: json support not found, install python-simplejson" + return data + + if self.options.rawprint: + return data + + return pprint.pformat(data) + + def do(self, args): + + # I'm not really a fan of the "module methodname" approach + # but we'll keep it for now -akl + + # I kind of feel like we shouldn't be parsing args here, but I'm + # not sure what the write place is -al; + self.module = args[0] + if len(args) > 1: + self.method = args[1] + else: + self.method = None + if len(args) > 2: + self.method_args = args[2:] + else: + self.method_args = [] + + # this could get weird, sub sub classes might be calling this + # this with multiple.parentCommand.parentCommands... + # maybe command.py needs a way to set attrs on subCommands? + # or some sort of shared datastruct? + self.server_spec = self.parentCommand.server_spec + + client_obj = client.Client(self.server_spec,port=self.port,interactive=True, + verbose=self.verbose, config=self.config, nforks=self.options.forks) + results = client_obj.run(self.module, self.method, self.method_args) + + # TO DO: add multiplexer support + # probably as a higher level module. + + # dump the return code stuff atm till we figure out the right place for it + return self.format_return(results) diff --git a/func/overlord/cmd_modules/call.pyc b/func/overlord/cmd_modules/call.pyc Binary files differnew file mode 100644 index 0000000..f6c588d --- /dev/null +++ b/func/overlord/cmd_modules/call.pyc diff --git a/func/overlord/cmd_modules/copyfile.py b/func/overlord/cmd_modules/copyfile.py new file mode 100644 index 0000000..295aeab --- /dev/null +++ b/func/overlord/cmd_modules/copyfile.py @@ -0,0 +1,73 @@ +""" +copyfile command line + +Copyright 2007, Red Hat, Inc +see AUTHORS + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + + +import optparse +import os +import pprint +import stat +import xmlrpclib + +from func.overlord import command +from func.overlord import client + +DEFAULT_PORT = 51234 + +class CopyFile(client.command.Command): + name = "copyfile" + usage = "copy a file to a client" + + + def addOptions(self): + self.parser.add_option("-f", "--file", dest="filename", + action="store") + self.parser.add_option("", "--remotepath", dest="remotepath", + action="store") + self.parser.add_option("", "--force", dest="force", + action="store_true") + self.parser.add_option("-v", "--verbose", dest="verbose", + action="store_true") + self.parser.add_option("-p", "--port", dest="port") + + def handleOptions(self, options): + self.port = DEFAULT_PORT + if self.options.port: + self.port = self.options.port + + + def do(self, args): + self.server_spec = self.parentCommand.server_spec + + client_obj = client.Client(self.server_spec, + port=self.port, + interactive=False, + verbose=self.options.verbose, + config=self.config) + + + try: + fb = open(self.options.filename, "r").read() + except IOError, e: + print "Unable to open file: %s: %s" % (self.options.filename, e) + return + + st = os.stat(self.options.filename) + mode = stat.S_IMODE(st.st_mode) + uid = st.st_uid + gid = st.st_gid + + + data = xmlrpclib.Binary(fb) + results = client_obj.run("copyfile", "copyfile", [self.options.remotepath, data, + mode, uid, gid]) diff --git a/func/overlord/cmd_modules/listminions.py b/func/overlord/cmd_modules/listminions.py new file mode 100644 index 0000000..50c7e24 --- /dev/null +++ b/func/overlord/cmd_modules/listminions.py @@ -0,0 +1,51 @@ +""" +copyfile command line + +Copyright 2007, Red Hat, Inc +see AUTHORS + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + + +import optparse +import os + +from func.overlord import command +from func.overlord import client +DEFAULT_PORT = 51234 + +class ListMinions(client.command.Command): + name = "list_minions" + usage = "show known minions" + + def addOptions(self): + self.parser.add_option("-v", "--verbose", dest="verbose", + action="store_true") + + def handleOptions(self, options): + self.port = DEFAULT_PORT + if options.verbose: + self.verbose = self.options.verbose + + def do(self, args): + self.server_spec = self.parentCommand.server_spec + + client_obj = client.Client(self.server_spec, + port=self.port, + interactive=False, + verbose=self.options.verbose, + config=self.config) + + servers = client_obj.servers + print servers + for server in servers: + # just cause I hate regex'es -akl + host = server.split(':')[-2] + host = host.split('/')[-1] + print host diff --git a/func/overlord/cmd_modules/ping.py b/func/overlord/cmd_modules/ping.py new file mode 100644 index 0000000..f756fd9 --- /dev/null +++ b/func/overlord/cmd_modules/ping.py @@ -0,0 +1,69 @@ +""" +copyfile command line + +Copyright 2007, Red Hat, Inc +Michael DeHaan <mdehaan@redhat.com> +also see AUTHORS + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + +import optparse +import os +import pprint +import stat +import xmlrpclib + +from func.overlord import command +from func.overlord import client + +# FIXME: this really should not be in each sub module. +DEFAULT_PORT = 51234 + + +class Ping(client.command.Command): + name = "ping" + usage = "see what func minions are up/accessible" + + def addOptions(self): + """ + Not too many options for you! (Seriously, it's a simple command ... func "*" ping) + """ + # FIXME: verbose and port should be added globally to all sub modules + self.parser.add_option("-v", "--verbose", dest="verbose", + action="store_true") + self.parser.add_option("-p", "--port", dest="port", + default=DEFAULT_PORT) + + def handleOptions(self, options): + """ + Nothing to do here... + """ + pass + + def do(self, args): + self.server_spec = self.parentCommand.server_spec + + # because this is mainly an interactive command, expand the server list and make seperate connections. + # to make things look more speedy. + + servers = client.expand_servers(self.server_spec, port=self.options.port, noglobs=None, + verbose=self.options.verbose, just_fqdns=True) + + for server in servers: + + client_obj = client.Client(server,port=self.options.port,interactive=False, + verbose=self.options.verbose,config=self.config, noglobs=True) + + results = client_obj.run("test", "ping", []) + if results == 1: + print "[ ok ... ] %s" % server + else: + print "[ FAILED ] %s" % server + + return 1 diff --git a/func/overlord/cmd_modules/show.py b/func/overlord/cmd_modules/show.py new file mode 100644 index 0000000..e1df554 --- /dev/null +++ b/func/overlord/cmd_modules/show.py @@ -0,0 +1,99 @@ +""" +show introspection commandline + +Copyright 2007, Red Hat, Inc +see AUTHORS + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + + +import optparse +import pprint +import xmlrpclib + +from func.overlord import command +from func.overlord import client + +DEFAULT_PORT = 51234 + + +class ShowHardware(client.command.Command): + name = "hardware" + usage = "show hardware details" + + # FIXME: we might as well make verbose be in the subclass + # and probably an inc variable while we are at it + def addOptions(self): + self.parser.add_option("-v", "--verbose", dest="verbose", + action="store_true") + self.parser.add_option("-p", "--port", dest="port") + + + def handleOptions(self, options): + self.port = DEFAULT_PORT + if self.options.port: + self.port = self.options.port + + def parse(self, argv): + self.argv = argv + return command.Command.parse(self,argv) + + def do(self,args): + + self.server_spec = self.parentCommand.parentCommand.server_spec + + client_obj = client.Client(self.server_spec, + port=self.port, + interactive=False, + verbose=self.options.verbose, + config=self.config) + + results = client_obj.run("hardware", "info", []) + + # if the user + top_options = ["port","verbose"] + + for minion in results: + print "%s:" % minion + minion_data = results[minion] + # if user set no args + if not args: + pprint.pprint(minion_data) + continue + + for arg in args: + if arg in minion_data: + print minion_data[arg] + + +class Show(client.command.Command): + name = "show" + usage = "various simple report stuff" + subCommandClasses = [ShowHardware] + def addOptions(self): + self.parser.add_option("-v", "--verbose", dest="verbose", + action="store_true") + self.parser.add_option("-p", "--port", dest="port", + default=DEFAULT_PORT) + + def handleOptions(self, options): + self.options = options + + self.verbose = options.verbose + self.port = options.port + + + def parse(self, argv): + self.argv = argv + + return command.Command.parse(self, argv) + + + def do(self, args): + pass diff --git a/func/overlord/command.py b/func/overlord/command.py new file mode 100644 index 0000000..7fb7de4 --- /dev/null +++ b/func/overlord/command.py @@ -0,0 +1,287 @@ +# -*- Mode: Python; test-case-name: test_command -*- +# vi:si:et:sw=4:sts=4:ts=4 + +# This file is released under the standard PSF license. +# +# from MOAP - https://thomas.apestaart.org/moap/trac +# written by Thomas Vander Stichele (thomas at apestaart dot org) +# + +""" +Command class. +""" + +import optparse +import sys + +from func.config import read_config, CONFIG_FILE +from func.commonconfig import CMConfig + +class CommandHelpFormatter(optparse.IndentedHelpFormatter): + """ + I format the description as usual, but add an overview of commands + after it if there are any, formatted like the options. + """ + _commands = None + + def addCommand(self, name, description): + if self._commands is None: + self._commands = {} + self._commands[name] = description + + ### override parent method + def format_description(self, description): + # textwrap doesn't allow for a way to preserve double newlines + # to separate paragraphs, so we do it here. + blocks = description.split('\n\n') + rets = [] + + for block in blocks: + rets.append(optparse.IndentedHelpFormatter.format_description(self, + block)) + ret = "\n".join(rets) + if self._commands: + commandDesc = [] + commandDesc.append("commands:") + keys = self._commands.keys() + keys.sort() + length = 0 + for key in keys: + if len(key) > length: + length = len(key) + for name in keys: + format = " %-" + "%d" % length + "s %s" + commandDesc.append(format % (name, self._commands[name])) + ret += "\n" + "\n".join(commandDesc) + "\n" + return ret + +class CommandOptionParser(optparse.OptionParser): + """ + I parse options as usual, but I explicitly allow setting stdout + so that our print_help() method (invoked by default with -h/--help) + defaults to writing there. + """ + _stdout = sys.stdout + + def set_stdout(self, stdout): + self._stdout = stdout + + # we're overriding the built-in file, but we need to since this is + # the signature from the base class + __pychecker__ = 'no-shadowbuiltin' + def print_help(self, file=None): + # we are overriding a parent method so we can't do anything about file + __pychecker__ = 'no-shadowbuiltin' + if file is None: + file = self._stdout + file.write(self.format_help()) + +class Command: + """ + I am a class that handles a command for a program. + Commands can be nested underneath a command for further processing. + + @cvar name: name of the command, lowercase + @cvar aliases: list of alternative lowercase names recognized + @type aliases: list of str + @cvar usage: short one-line usage string; + %command gets expanded to a sub-command or [commands] + as appropriate + @cvar summary: short one-line summary of the command + @cvar description: longer paragraph explaining the command + @cvar subCommands: dict of name -> commands below this command + @type subCommands: dict of str -> L{Command} + """ + name = None + aliases = None + usage = None + summary = None + description = None + parentCommand = None + subCommands = None + subCommandClasses = None + aliasedSubCommands = None + + def __init__(self, parentCommand=None, stdout=sys.stdout, + stderr=sys.stderr): + """ + Create a new command instance, with the given parent. + Allows for redirecting stdout and stderr if needed. + This redirection will be passed on to child commands. + """ + if not self.name: + self.name = str(self.__class__).split('.')[-1].lower() + self.stdout = stdout + self.stderr = stderr + self.parentCommand = parentCommand + + self.config = read_config(CONFIG_FILE, CMConfig) + + # create subcommands if we have them + self.subCommands = {} + self.aliasedSubCommands = {} + if self.subCommandClasses: + for C in self.subCommandClasses: + c = C(self, stdout=stdout, stderr=stderr) + self.subCommands[c.name] = c + if c.aliases: + for alias in c.aliases: + self.aliasedSubCommands[alias] = c + + # create our formatter and add subcommands if we have them + formatter = CommandHelpFormatter() + if self.subCommands: + for name, command in self.subCommands.items(): + formatter.addCommand(name, command.summary or + command.description) + + # expand %command for the bottom usage + usage = self.usage or self.name + if usage.find("%command") > -1: + usage = usage.split("%command")[0] + '[command]' + usages = [usage, ] + + # FIXME: abstract this into getUsage that takes an optional + # parentCommand on where to stop recursing up + # useful for implementing subshells + + # walk the tree up for our usage + c = self.parentCommand + while c: + usage = c.usage or c.name + if usage.find(" %command") > -1: + usage = usage.split(" %command")[0] + usages.append(usage) + c = c.parentCommand + usages.reverse() + usage = " ".join(usages) + + # create our parser + description = self.description or self.summary + self.parser = CommandOptionParser( + usage=usage, description=description, + formatter=formatter) + self.parser.set_stdout(self.stdout) + self.parser.disable_interspersed_args() + + # allow subclasses to add options + self.addOptions() + + def addOptions(self): + """ + Override me to add options to the parser. + """ + pass + + def do(self, args): + """ + Override me to implement the functionality of the command. + """ + pass + + def parse(self, argv): + """ + Parse the given arguments and act on them. + + @rtype: int + @returns: an exit code + """ + self.options, args = self.parser.parse_args(argv) + + # FIXME: make handleOptions not take options, since we store it + # in self.options now + ret = self.handleOptions(self.options) + if ret: + return ret + + # handle pleas for help + if args and args[0] == 'help': + self.debug('Asked for help, args %r' % args) + + # give help on current command if only 'help' is passed + if len(args) == 1: + self.outputHelp() + return 0 + + # complain if we were asked for help on a subcommand, but we don't + # have any + if not self.subCommands: + self.stderr.write('No subcommands defined.') + self.parser.print_usage(file=self.stderr) + self.stderr.write( + "Use --help to get more information about this command.\n") + return 1 + + # rewrite the args the other way around; + # help doap becomes doap help so it gets deferred to the doap + # command + args = [args[1], args[0]] + + + # if we have args that we need to deal with, do it now + # before we start looking for subcommands + self.handleArguments(args) + + # if we don't have subcommands, defer to our do() method + if not self.subCommands: + ret = self.do(args) + + # if everything's fine, we return 0 + if not ret: + ret = 0 + + return ret + + + # if we do have subcommands, defer to them + try: + command = args[0] + except IndexError: + self.parser.print_usage(file=self.stderr) + self.stderr.write( + "Use --help to get a list of commands.\n") + return 1 + + if command in self.subCommands.keys(): + return self.subCommands[command].parse(args[1:]) + + if self.aliasedSubCommands: + if command in self.aliasedSubCommands.keys(): + return self.aliasedSubCommands[command].parse(args[1:]) + + self.stderr.write("Unknown command '%s'.\n" % command) + return 1 + + def outputHelp(self): + """ + Output help information. + """ + self.parser.print_help(file=self.stderr) + + def outputUsage(self): + """ + Output usage information. + Used when the options or arguments were missing or wrong. + """ + self.parser.print_usage(file=self.stderr) + + def handleOptions(self, options): + """ + Handle the parsed options. + """ + pass + + def handleArguments(self, arguments): + """ + Handle the parsed arguments. + """ + pass + + def getRootCommand(self): + """ + Return the top-level command, which is typically the program. + """ + c = self + while c.parentCommand: + c = c.parentCommand + return c diff --git a/func/overlord/command.pyc b/func/overlord/command.pyc Binary files differnew file mode 100644 index 0000000..6fa44e8 --- /dev/null +++ b/func/overlord/command.pyc diff --git a/func/overlord/forkbomb.pyc b/func/overlord/forkbomb.pyc Binary files differnew file mode 100644 index 0000000..308557d --- /dev/null +++ b/func/overlord/forkbomb.pyc diff --git a/func/overlord/func_command.py b/func/overlord/func_command.py new file mode 100644 index 0000000..4cec8a0 --- /dev/null +++ b/func/overlord/func_command.py @@ -0,0 +1,71 @@ +#!/usr/bin/python + +## func command line interface & client lib +## +## Copyright 2007,2008 Red Hat, Inc +## +AUTHORS +## +## This software may be freely redistributed under the terms of the GNU +## general public license. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +import sys + + +import command + +#FIXME: need a plug-in runtime module loader here +from cmd_modules import call +from cmd_modules import show +from cmd_modules import copyfile +from cmd_modules import listminions +from cmd_modules import ping + +from func.overlord import client + +class FuncCommandLine(command.Command): + name = "func" + usage = "func is the commandline interface to a func minion" + + subCommandClasses = [call.Call, show.Show, + copyfile.CopyFile, listminions.ListMinions, ping.Ping] + + def __init__(self): + + command.Command.__init__(self) + + def do(self, args): + pass + + def addOptions(self): + self.parser.add_option('', '--version', action="store_true", + help="show version information") + + # just some ugly goo to try to guess if arg[1] is hostnamegoo or + # a command name + def _isGlob(self, str): + if str.find("*") or str.find("?") or str.find("[") or str.find("]"): + return True + return False + + def handleArguments(self, args): + if len(args) < 2: + print "see the func manpage for usage" + sys.exit(411) + server_string = args[0] + # try to be clever about this for now + if client.isServer(server_string) or self._isGlob(server_string): + self.server_spec = server_string + args.pop(0) + # if it doesn't look like server, assume it + # is a sub command? that seems wrong, what about + # typo's and such? How to catch that? -akl + # maybe a class variable self.data on Command? + + def handleOptions(self, options): + if options.version: + #FIXME + print "version is NOT IMPLEMENTED YET" diff --git a/func/overlord/func_command.pyc b/func/overlord/func_command.pyc Binary files differnew file mode 100644 index 0000000..1834e0e --- /dev/null +++ b/func/overlord/func_command.pyc diff --git a/func/overlord/groups.py b/func/overlord/groups.py new file mode 100644 index 0000000..8eaf28e --- /dev/null +++ b/func/overlord/groups.py @@ -0,0 +1,95 @@ +#!/usr/bin/python + +## func command line interface & client lib +## +## Copyright 2007,2008 Red Hat, Inc +## Adrian Likins <alikins@redhat.com> +## +AUTHORS +## +## This software may be freely redistributed under the terms of the GNU +## general public license. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +## + + +# this module lets you define groups of systems to work with from the +# commandline. It uses an "ini" style config parser like: + +#[groupname] +#host = foobar, baz, blip +#subgroup = blippy + + +import ConfigParser +import os + + +class Groups(object): + + def __init__(self, filename="/etc/func/groups"): + self.filename = filename + self.group_names = {} + self.groups = {} + self.__parse() + + def __parse(self): + + self.cp = ConfigParser.SafeConfigParser() + self.cp.read(self.filename) + + for section in self.cp.sections(): + self.add_group(section) + options = self.cp.options(section) + for option in options: + if option == "host": + self.add_hosts_to_group(section, self.cp.get(section, option)) + if option == "subgroup": + pass + + + def show(self): + print self.cp.sections() + print self.groups + + def add_group(self, group): + pass + + def __parse_hoststrings(self, hoststring): + hosts = [] + bits = hoststring.split(';') + for bit in bits: + blip = bit.strip().split(' ') + for host in blip: + if host not in hosts: + hosts.append(host.strip()) + + return hosts + + def add_hosts_to_group(self, group, hoststring): + hosts = self.__parse_hoststrings(hoststring) + for host in hosts: + self.add_host_to_group(group, host) + + + + def add_host_to_group(self, group, host): + if not self.groups.has_key(group): + self.groups[group] = [] + self.groups[group].append(host) + + def get_groups(self): + return self.groups + + + +def main(): + g = Groups("/tmp/testgroups") + print g.show() + + + +if __name__ == "__main__": + main() diff --git a/func/overlord/groups.pyc b/func/overlord/groups.pyc Binary files differnew file mode 100644 index 0000000..9ed2a92 --- /dev/null +++ b/func/overlord/groups.pyc diff --git a/func/overlord/highlevel.py b/func/overlord/highlevel.py new file mode 100644 index 0000000..977dcb4 --- /dev/null +++ b/func/overlord/highlevel.py @@ -0,0 +1,40 @@ +## +## func higher level API interface for overlord side operations +## +## Copyright 2007, Red Hat, Inc +## Michael DeHaan <mdehaan@redhat.com> +## +AUTHORS +## +## This software may be freely redistributed under the terms of the GNU +## general public license. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +## + +import exceptions + +class HigherLevelObject: + + def __init__(self, client): + self.client = client_handle + + def modify(self, key, properties): + """ + Modify or create an entity named key. + properties should contain all neccessary fields. + """ + raise exceptions.NotImplementedError + + def remove(self, key): + """ + Remove an entity named key. + """ + raise exceptions.NotImplementedError + + def list(self): + """ + List all objects + """ + raise exceptions.NotImplementedError diff --git a/func/overlord/inventory.py b/func/overlord/inventory.py new file mode 100755 index 0000000..8302a1c --- /dev/null +++ b/func/overlord/inventory.py @@ -0,0 +1,191 @@ +## +## func inventory app. +## use func to collect inventory data on anything, yes, anything +## +## Copyright 2007, Red Hat, Inc +## Michael DeHaan <mdehaan@redhat.com> +## +AUTHORS +## +## This software may be freely redistributed under the terms of the GNU +## general public license. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +## + +import os.path +import time +import optparse +import sys +import pprint +import xmlrpclib +from func.minion import sub_process +import func.overlord.client as func_client +import func.utils as utils + +DEFAULT_TREE = "/var/lib/func/inventory/" + + +class FuncInventory(object): + + def __init__(self): + pass + + def run(self,args): + + p = optparse.OptionParser() + p.add_option("-v", "--verbose", + dest="verbose", + action="store_true", + help="provide extra output") + p.add_option("-s", "--server-spec", + dest="server_spec", + default="*", + help="run against specific servers, default: '*'") + p.add_option("-m", "--methods", + dest="methods", + default="inventory", + help="run inventory only on certain function names, default: 'inventory'") + p.add_option("-M", "--modules", + dest="modules", + default="all", + help="run inventory only on certain module names, default: 'all'") + p.add_option("-t", "--tree", + dest="tree", + default=DEFAULT_TREE, + help="output results tree here, default: %s" % DEFAULT_TREE) + p.add_option("-n", "--no-git", + dest="nogit", + action="store_true", + help="disable useful change tracking features") + p.add_option("-x", "--xmlrpc", dest="xmlrpc", + help="output data using XMLRPC format", + action="store_true") + p.add_option("-j", "--json", dest="json", + help="output data using JSON", + action="store_true") + + + (options, args) = p.parse_args(args) + self.options = options + + filtered_module_list = options.modules.split(",") + filtered_function_list = options.methods.split(",") + + self.git_setup(options) + + # see what modules each host provides (as well as what hosts we have) + host_methods = func_client.Client(options.server_spec).system.list_methods() + + # call all remote info methods and handle them + if options.verbose: + print "- scanning ..." + # for (host, modules) in host_modules.iteritems(): + + for (host, methods) in host_methods.iteritems(): + + if utils.is_error(methods): + print "-- connection refused: %s" % host + break + + for each_method in methods: + + #if type(each_method) == int: + # if self.options.verbose: + # print "-- connection refused: %s" % host + # break + + tokens = each_method.split(".") + module_name = ".".join(tokens[:-1]) + method_name = tokens[-1] + + if not "all" in filtered_module_list and not module_name in filtered_module_list: + continue + + if not "all" in filtered_function_list and not method_name in filtered_function_list: + continue + + client = func_client.Client(host,noglobs=True) # ,noglobs=True) + results = getattr(getattr(client,module_name),method_name)() + if self.options.verbose: + print "-- %s: running: %s %s" % (host, module_name, method_name) + self.save_results(options, host, module_name, method_name, results) + self.git_update(options) + return 1 + + def format_return(self, data): + """ + The call module supports multiple output return types, the default is pprint. + """ + + # special case... if the return is a string, just print it straight + if type(data) == str: + return data + + if self.options.xmlrpc: + return xmlrpclib.dumps((data,"")) + + if self.options.json: + try: + import simplejson + return simplejson.dumps(data) + except ImportError: + print "ERROR: json support not found, install python-simplejson" + sys.exit(1) + + return pprint.pformat(data) + + # FUTURE: skvidal points out that guest symlinking would be an interesting feature + + def save_results(self, options, host_name, module_name, method_name, results): + dirname = os.path.join(options.tree, host_name, module_name) + if not os.path.exists(dirname): + os.makedirs(dirname) + filename = os.path.join(dirname, method_name) + results_file = open(filename,"w+") + data = self.format_return(results) + results_file.write(data) + results_file.close() + + def git_setup(self,options): + if options.nogit: + return + if not os.path.exists("/usr/bin/git"): + print "git-core is not installed, so no change tracking is available." + print "use --no-git or, better, just install it." + sys.exit(411) + + if not os.path.exists(options.tree): + os.makedirs(options.tree) + dirname = os.path.join(options.tree, ".git") + if not os.path.exists(dirname): + if options.verbose: + print "- initializing git repo: %s" % options.tree + cwd = os.getcwd() + os.chdir(options.tree) + rc1 = sub_process.call(["/usr/bin/git", "init"], shell=False) + # FIXME: check rc's + os.chdir(cwd) + else: + if options.verbose: + print "- git already initialized: %s" % options.tree + + def git_update(self,options): + if options.nogit: + return + else: + if options.verbose: + print "- updating git" + mytime = time.asctime() + cwd = os.getcwd() + os.chdir(options.tree) + rc1 = sub_process.call(["/usr/bin/git", "add", "*" ], shell=False) + rc2 = sub_process.call(["/usr/bin/git", "commit", "-a", "-m", "Func-inventory update: %s" % mytime], shell=False) + # FIXME: check rc's + os.chdir(cwd) + + +if __name__ == "__main__": + inv = FuncInventory() + inv.run(sys.argv) diff --git a/func/overlord/jobthing.pyc b/func/overlord/jobthing.pyc Binary files differnew file mode 100644 index 0000000..cba36cb --- /dev/null +++ b/func/overlord/jobthing.pyc diff --git a/func/overlord/modules/netapp.py b/func/overlord/modules/netapp.py new file mode 100644 index 0000000..987901e --- /dev/null +++ b/func/overlord/modules/netapp.py @@ -0,0 +1,82 @@ +## +## Overlord library to interface with minion-side netapp operations +## +## Most of this is just wrappers to create some cleaner, earier to use +## interfaces. Also allows users to get function signatures and use +## nice things like kwargs client side, for those of us who can't live +## without ipython introspection. +## +## Copyright 2008, Red Hat, Inc +## John Eckersberg <jeckersb@redhat.com> +## +## This software may be freely redistributed under the terms of the GNU +## general public license. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +## + +from func.overlord.client import Client + +class RemoteError(Exception): pass + +def _(res): + if type(res) == type([]) and res[0] == 'REMOTE_ERROR': + raise RemoteError, res[2] + else: + return res + +class Filer(Client): + def __init__(self, filer, admin_host): + Client.__init__(self, admin_host) + self.filer = filer + self.admin_host = admin_host + + def create_volume(self, vol, aggr, size): + return _(self.netapp.vol.create(self.filer, vol, aggr, size)[self.admin_host]) + + def destroy_volume(self, vol): + # offline it first + try: + self.netapp.vol.offline(self.filer, vol) + except: + pass + return _(self.netapp.vol.destroy(self.filer, vol)[self.admin_host]) + + def offline_volume(self, vol): + return _(self.netapp.vol.offline(self.filer, vol)[self.admin_host]) + + def online_volume(self, vol): + return _(self.netapp.vol.online(self.filer, vol)[self.admin_host]) + + def get_volume_size(self, vol): + return _(self.netapp.vol.size(self.filer, vol)[self.admin_host]) + + def resize_volume(self, vol, delta): + return _(self.netapp.vol.size(self.filer, vol, delta)[self.admin_host]) + + def create_snapshot(self, vol, snap): + return _(self.netapp.snap.create(self.filer, vol, snap)[self.admin_host]) + + def delete_snapshot(self, vol, snap): + return _(self.netapp.snap.delete(self.filer, vol, snap)[self.admin_host]) + + def create_clone_volume(self, vol, parent, snap): + return _(self.netapp.vol.clone.create(self.filer, vol, parent, snap)[self.admin_host]) + + def split_clone_volume(self, vol): + return _(self.netapp.vol.clone.split(self.filer, vol)[self.admin_host]) + + def list_volumes(self): + vols = _(self.netapp.vol.status(self.filer)) + return_list = [] + for vol in vols: + return_list.append(vol['name']) + return return_list + + def volume_details(self, vol=None): + if vol: + return _(self.netapp.vol.status(self.filer, vol)[self.admin_host]) + else: + return _(self.netapp.vol.status(self.filer)[self.admin_host]) diff --git a/func/overlord/sslclient.py b/func/overlord/sslclient.py new file mode 100755 index 0000000..3861bb8 --- /dev/null +++ b/func/overlord/sslclient.py @@ -0,0 +1,50 @@ +import sys +import xmlrpclib +import urllib + +from func import SSLCommon + + +class SSL_Transport(xmlrpclib.Transport): + + user_agent = "pyOpenSSL_XMLRPC/%s - %s" % ('0.1', xmlrpclib.Transport.user_agent) + + def __init__(self, ssl_context, timeout=None, use_datetime=0): + if sys.version_info[:3] >= (2, 5, 0): + xmlrpclib.Transport.__init__(self, use_datetime) + self.ssl_ctx=ssl_context + self._timeout = timeout + + def make_connection(self, host): + # Handle username and password. + try: + host, extra_headers, x509 = self.get_host_info(host) + except AttributeError: + # Yay for Python 2.2 + pass + _host, _port = urllib.splitport(host) + return SSLCommon.HTTPS(_host, int(_port), ssl_context=self.ssl_ctx, timeout=self._timeout) + + +class SSLXMLRPCServerProxy(xmlrpclib.ServerProxy): + def __init__(self, uri, pkey_file, cert_file, ca_cert_file, timeout=None): + self.ctx = SSLCommon.CreateSSLContext(pkey_file, cert_file, ca_cert_file) + xmlrpclib.ServerProxy.__init__(self, uri, SSL_Transport(ssl_context=self.ctx, timeout=timeout)) + + +class FuncServer(SSLXMLRPCServerProxy): + def __init__(self, uri, pem=None, crt=None, ca=None): + self.pem = pem + self.crt = crt + self.ca = ca + + SSLXMLRPCServerProxy.__init__(self, uri, + self.pem, + self.crt, + self.ca) + + +if __name__ == "__main__": + s = SSLXMLRPCServerProxy('https://localhost:51234/', '/etc/pki/func/slave.pem', '/etc/pki/func/slave.cert', '/etc/pki/func/ca/funcmaster.crt') + f = s.ping(1, 2) + print f diff --git a/func/overlord/sslclient.pyc b/func/overlord/sslclient.pyc Binary files differnew file mode 100644 index 0000000..fdc21f2 --- /dev/null +++ b/func/overlord/sslclient.pyc diff --git a/func/overlord/test_func.py b/func/overlord/test_func.py new file mode 100755 index 0000000..2b3f041 --- /dev/null +++ b/func/overlord/test_func.py @@ -0,0 +1,61 @@ +#!/usr/bin/python + + +# FIXME: should import the client lib, not XMLRPC lib, when we are done + +import xmlrpclib +import sys + +TEST_GETATTR = True +TEST_PROCESS = False +TEST_VIRT = False +TEST_SERVICES = False +TEST_HARDWARE = False +TEST_SMART = True + +if TEST_GETATTR: + import func.overlord.client as func_client + print func_client.Client("*").hardware.pci_info() + #print func_client.Client("*").test.add(1,2) + #print func_client.Client("*").hardware.info() + #print func_client.Client("*").run("hardware","info",[]) + #print func_client.Client(socket.gethostname(),noglobs=True).test.add("1","2") + sys.exit(1) + +# get a connecton (to be replaced by client lib logic) +s = xmlrpclib.ServerProxy("http://127.0.0.1:51234") + +# here's the basic test... +print s.test.add(1, 2) + +if TEST_SMART: + print s.smart.info() + +if TEST_PROCESS: + print s.process.info() + # print s.process.pkill("thunderbird") + +# here's the service module testing +if TEST_SERVICES: + print s.service.restart("httpd") + +if TEST_HARDWARE: + print s.hardware.info() + +# this is so I can remember how the virt module works +if TEST_VIRT: + + # example of using koan to install a virtual machine + #s.virt_install("mdehaan.rdu.redhat.com","profileX") + + # wait ... + vms = s.virt.list_vms() + # example of stopping all stopped virtual machines + print "list of virtual instances = %s" % vms + for vm in vms: + status = s.virt.status(vm) + print status + if status == "shutdown": + s.virt.start(vm) + +# add more tests here |