diff options
author | Adrian Likins <alikins@grimlock.devel.redhat.com> | 2008-02-25 13:28:32 -0500 |
---|---|---|
committer | Adrian Likins <alikins@grimlock.devel.redhat.com> | 2008-02-25 13:28:32 -0500 |
commit | 71ca4184404df52dabfea318e3e0a1ca8c1b4c61 (patch) | |
tree | 5a4b69c9b9e38057256624f1c173f59d928487e5 /func | |
parent | e648895b13205a4669ba9d3ea8756209b6a6d9eb (diff) | |
parent | e27f9d8383848d523d301706046c54e99b5f9676 (diff) | |
download | third_party-func-71ca4184404df52dabfea318e3e0a1ca8c1b4c61.tar.gz third_party-func-71ca4184404df52dabfea318e3e0a1ca8c1b4c61.tar.xz third_party-func-71ca4184404df52dabfea318e3e0a1ca8c1b4c61.zip |
Merge branch 'master' of ssh://git.fedoraproject.org/git/hosted/func
Diffstat (limited to 'func')
-rw-r--r-- | func/commonconfig.py | 2 | ||||
-rw-r--r-- | func/forkbomb.py | 2 | ||||
-rw-r--r-- | func/jobthing.py | 2 | ||||
-rw-r--r-- | func/minion/modules/certmaster.py | 4 | ||||
-rw-r--r-- | func/minion/modules/jobs.py | 1 | ||||
-rw-r--r-- | func/minion/modules/process.py | 66 | ||||
-rwxr-xr-x | func/minion/server.py | 1 | ||||
-rwxr-xr-x | func/minion/utils.py | 1 | ||||
-rwxr-xr-x | func/overlord/client.py | 7 | ||||
-rw-r--r-- | func/overlord/cmd_modules/check.py | 143 | ||||
-rw-r--r-- | func/overlord/cmd_modules/listminions.py | 16 | ||||
-rw-r--r-- | func/overlord/cmd_modules/ping.py | 6 | ||||
-rw-r--r-- | func/overlord/func_command.py | 10 | ||||
-rw-r--r-- | func/overlord/groups.py | 1 | ||||
-rwxr-xr-x | func/utils.py | 1 |
15 files changed, 206 insertions, 57 deletions
diff --git a/func/commonconfig.py b/func/commonconfig.py index 9b4612e..292eb45 100644 --- a/func/commonconfig.py +++ b/func/commonconfig.py @@ -1,4 +1,4 @@ -from config import BaseConfig, BoolOption, IntOption, Option +from config import BaseConfig, BoolOption, Option class CMConfig(BaseConfig): listen_addr = Option('') diff --git a/func/forkbomb.py b/func/forkbomb.py index 3dfa6f2..73fa924 100644 --- a/func/forkbomb.py +++ b/func/forkbomb.py @@ -20,8 +20,6 @@ import bsddb import sys import tempfile import fcntl -import utils -import xmlrpclib DEFAULT_FORKS = 4 DEFAULT_CACHE_DIR = "/var/lib/func" diff --git a/func/jobthing.py b/func/jobthing.py index 67ad1a6..75a1d1a 100644 --- a/func/jobthing.py +++ b/func/jobthing.py @@ -20,11 +20,9 @@ import time # for testing only import shelve import bsddb import sys -import tempfile import fcntl import forkbomb import utils -import traceback JOB_ID_RUNNING = 0 JOB_ID_FINISHED = 1 diff --git a/func/minion/modules/certmaster.py b/func/minion/modules/certmaster.py index 9ca484f..c30a39c 100644 --- a/func/minion/modules/certmaster.py +++ b/func/minion/modules/certmaster.py @@ -13,10 +13,6 @@ ## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ## -# other modules -import sub_process -import codes - # our modules import func_module from func import certmaster as certmaster diff --git a/func/minion/modules/jobs.py b/func/minion/modules/jobs.py index 69fb75f..90c7421 100644 --- a/func/minion/modules/jobs.py +++ b/func/minion/modules/jobs.py @@ -14,7 +14,6 @@ ## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ## -import codes from func import jobthing import func_module diff --git a/func/minion/modules/process.py b/func/minion/modules/process.py index 848e847..22141c3 100644 --- a/func/minion/modules/process.py +++ b/func/minion/modules/process.py @@ -72,7 +72,7 @@ class ProcessModule(func_module.FuncModule): import os our_pid=os.getpid() results = [] - have_smaps=0 + global have_pss have_pss=0 def kernel_ver(): @@ -85,79 +85,83 @@ class ProcessModule(func_module.FuncModule): kv=kernel_ver() def getMemStats(pid): - """ return Rss,Pss,Shared (note Private = Rss-Shared) """ + """ return Private,Shared """ + global have_pss + Private_lines=[] Shared_lines=[] Pss_lines=[] pagesize=os.sysconf("SC_PAGE_SIZE")/1024 #KiB Rss=int(open("/proc/"+str(pid)+"/statm").readline().split()[1])*pagesize if os.path.exists("/proc/"+str(pid)+"/smaps"): #stat - global have_smaps - have_smaps=1 for line in open("/proc/"+str(pid)+"/smaps").readlines(): #open - #Note in smaps Shared+Private = Rss above - #The Rss in smaps includes video card mem etc. if line.startswith("Shared"): Shared_lines.append(line) + elif line.startswith("Private"): + Private_lines.append(line) elif line.startswith("Pss"): - global have_pss have_pss=1 Pss_lines.append(line) Shared=sum([int(line.split()[1]) for line in Shared_lines]) - Pss=sum([int(line.split()[1]) for line in Pss_lines]) + Private=sum([int(line.split()[1]) for line in Private_lines]) + #Note Shared + Private = Rss above + #The Rss in smaps includes video card mem etc. + if have_pss: + pss_adjust=0.5 #add 0.5KiB as this average error due to trunctation + Pss=sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) + Shared = Pss - Private elif (2,6,1) <= kv <= (2,6,9): - Pss=0 Shared=0 #lots of overestimation, but what can we do? + Private = Rss else: - Pss=0 Shared=int(open("/proc/"+str(pid)+"/statm").readline().split()[2])*pagesize - return (Rss, Pss, Shared) + Private = Rss - Shared + return (Private, Shared) + + def getCmdName(pid): + cmd = file("/proc/%d/status" % pid).readline()[6:-1] + exe = os.path.basename(os.path.realpath("/proc/%d/exe" % pid)) + if exe.startswith(cmd): + cmd=exe #show non truncated version + #Note because we show the non truncated name + #one can have separated programs as follows: + #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) + # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin + return cmd cmds={} shareds={} count={} for pid in os.listdir("/proc/"): try: - pid = int(pid) #note Thread IDs not listed in /proc/ - if pid ==our_pid: continue + pid = int(pid) #note Thread IDs not listed in /proc/ which is good + if pid == our_pid: continue except: continue - cmd = file("/proc/%d/status" % pid).readline()[6:-1] try: - exe = os.path.basename(os.path.realpath("/proc/%d/exe" % pid)) - if exe.startswith(cmd): - cmd=exe #show non truncated version - #Note because we show the non truncated name - #one can have separated programs as follows: - #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) - #56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin + cmd = getCmdName(pid) except: #permission denied or #kernel threads don't have exe links or #process gone continue try: - rss, pss, shared = getMemStats(pid) - private = rss-shared - #Note shared is always a subset of rss (trs is not always) + private, shared = getMemStats(pid) except: continue #process gone if shareds.get(cmd): - if pss: #add shared portion of PSS together - shareds[cmd]+=pss-private + if have_pss: #add shared portion of PSS together + shareds[cmd]+=shared elif shareds[cmd] < shared: #just take largest shared val shareds[cmd]=shared else: - if pss: - shareds[cmd]=pss-private - else: - shareds[cmd]=shared + shareds[cmd]=shared cmds[cmd]=cmds.setdefault(cmd,0)+private if count.has_key(cmd): count[cmd] += 1 else: count[cmd] = 1 - #Add max shared mem for each program + #Add shared mem for each program total=0 for cmd in cmds.keys(): cmds[cmd]=cmds[cmd]+shareds[cmd] diff --git a/func/minion/server.py b/func/minion/server.py index f1b827f..2fa175a 100755 --- a/func/minion/server.py +++ b/func/minion/server.py @@ -17,7 +17,6 @@ import SimpleXMLRPCServer import string import sys import traceback -import socket import fnmatch from gettext import textdomain diff --git a/func/minion/utils.py b/func/minion/utils.py index a7ea788..ea8854c 100755 --- a/func/minion/utils.py +++ b/func/minion/utils.py @@ -18,7 +18,6 @@ import time import traceback import xmlrpclib import glob -import traceback import codes from func import certs diff --git a/func/overlord/client.py b/func/overlord/client.py index f07e526..fdcf875 100755 --- a/func/overlord/client.py +++ b/func/overlord/client.py @@ -105,6 +105,7 @@ class Minions(object): def _get_new_hosts(self): self.new_hosts = self.group_class.get_hosts_by_groupgoo(self.spec) + return self.new_hosts def _get_all_hosts(self): seperate_gloobs = self.spec.split(";") @@ -116,6 +117,12 @@ class Minions(object): self.all_certs.append(cert) host = cert.replace(self.config.certroot,"")[1:-5] self.all_hosts.append(host) + return self.all_hosts + + def get_all_hosts(self): + self._get_new_hosts() + self._get_all_hosts() + return self.all_hosts def get_urls(self): self._get_new_hosts() diff --git a/func/overlord/cmd_modules/check.py b/func/overlord/cmd_modules/check.py new file mode 100644 index 0000000..cf1badb --- /dev/null +++ b/func/overlord/cmd_modules/check.py @@ -0,0 +1,143 @@ +""" +check checks to see how happy func is. +it provides sanity checks for basic user setup. + +Copyright 2008, Red Hat, Inc +Michael DeHaan <mdehaan@redhat.com> + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + + +import optparse +import os +import urllib2 + +from func.overlord import command +from func.overlord import client +from func import utils +from func.minion import sub_process +from func.config import read_config +from func.commonconfig import FuncdConfig + +# FIXME: don't hardcode this here +DEFAULT_PORT = 51234 + +class CheckAction(client.command.Command): + name = "check" + usage = "check func for possible setup problems" + + def addOptions(self): + self.parser.add_option("-c", "--certmaster", action="store_true", help="check the certmaster configuration on this box") + self.parser.add_option("-m", "--minion", action="store_true", help="check the minion configuration on this box") + + + def handleOptions(self, options): + # FIXME: all through the code we have this constant in each + # file, need to make this common. + self.port = DEFAULT_PORT + self.check_certmaster = options.certmaster + self.check_minion = options.minion + + def do(self, args): + + if not self.check_certmaster and not self.check_minion: + print "* specify --certmaster, --minion, or both" + return + else: + print "SCAN RESULTS:" + + hostname = utils.get_hostname() + print "* FQDN is detected as %s, verify that is correct" % hostname + self.check_iptables() + + if not os.getuid() == 0: + print "* root is required to run these setup tests" + return + + if self.check_minion: + + # check that funcd is running + self.check_service("funcd") + + # check that the configured certmaster is reachable + self.check_talk_to_certmaster() + + if self.check_certmaster: + + # check that certmasterd is running + self.check_service("certmasterd") + + # see if we have any waiting CSRs + # FIXME: TODO + + # see if we have signed any certs + # FIXME: TODO + + # construct a client handle and see if any hosts are reachable + self.server_spec = self.parentCommand.server_spec + + client_obj = client.Client( + self.server_spec, + port=self.port, + interactive=False, + verbose=False, + config=self.config + ) + results = client_obj.test.add(1,2) + hosts = results.keys() + if len(hosts) == 0: + print "* no systems have signed certs" + else: + failed = 0 + for x in hosts: + if results[x] != 3: + failed = failed+1 + if failed != 0: + print "* unable to connect to %s registered minions from overlord" % failed + print "* run func '*' ping to check status" + + # see if any of our certs have expired + + # warn about iptables if running + print "End of Report." + + def check_service(self, which): + if os.path.exists("/etc/rc.d/init.d/%s" % which): + rc = sub_process.call("/sbin/service %s status >/dev/null 2>/dev/null" % which, shell=True) + if rc != 0: + print "* service %s is not running" % which + + def check_iptables(self): + if os.path.exists("/etc/rc.d/init.d/iptables"): + rc = sub_process.call("/sbin/service iptables status >/dev/null 2>/dev/null", shell=True) + + if rc == 0: + # FIXME: don't hardcode port + print "* iptables may be running, ensure 51234 is unblocked" + + def check_talk_to_certmaster(self): + config_file = '/etc/func/minion.conf' + config = read_config(config_file, FuncdConfig) + cert_dir = config.cert_dir + # FIXME: don't hardcode port + master_uri = "http://%s:51235/" % config.certmaster + print "* this minion is configured in /etc/func/minion.conf to talk to host '%s' for certs, verify that is correct" % config.certmaster + # this will be a 501, unsupported GET, but we should be + # able to tell if we can make contact + connect_ok = True + try: + fd = urllib2.urlopen(master_uri) + data = fd.read() + fd.close() + except urllib2.HTTPError: + pass + except: + connect_ok = False + if not connect_ok: + print "cannot connect to certmaster at %s" % (master_uri) diff --git a/func/overlord/cmd_modules/listminions.py b/func/overlord/cmd_modules/listminions.py index 50c7e24..9421b8d 100644 --- a/func/overlord/cmd_modules/listminions.py +++ b/func/overlord/cmd_modules/listminions.py @@ -1,5 +1,6 @@ """ -copyfile command line +list minions provides a command line way to see what certs are +registered. Copyright 2007, Red Hat, Inc see AUTHORS @@ -42,10 +43,13 @@ class ListMinions(client.command.Command): verbose=self.options.verbose, config=self.config) - servers = client_obj.servers - print servers + results = client_obj.test.add(1,2) + servers = results.keys() + servers.sort() + + # print servers for server in servers: # just cause I hate regex'es -akl - host = server.split(':')[-2] - host = host.split('/')[-1] - print host + # host = server.split(':')[-2] + # host = host.split('/')[-1] + print server diff --git a/func/overlord/cmd_modules/ping.py b/func/overlord/cmd_modules/ping.py index f756fd9..438e2a9 100644 --- a/func/overlord/cmd_modules/ping.py +++ b/func/overlord/cmd_modules/ping.py @@ -1,5 +1,5 @@ """ -copyfile command line +ping minions to see whether they are up. Copyright 2007, Red Hat, Inc Michael DeHaan <mdehaan@redhat.com> @@ -52,8 +52,8 @@ class Ping(client.command.Command): # because this is mainly an interactive command, expand the server list and make seperate connections. # to make things look more speedy. - servers = client.expand_servers(self.server_spec, port=self.options.port, noglobs=None, - verbose=self.options.verbose, just_fqdns=True) + minion_set = client.Minions(self.server_spec, port=self.options.port) + servers = minion_set.get_all_hosts() for server in servers: diff --git a/func/overlord/func_command.py b/func/overlord/func_command.py index 8bc6b7c..bd718bb 100644 --- a/func/overlord/func_command.py +++ b/func/overlord/func_command.py @@ -23,15 +23,19 @@ from cmd_modules import show from cmd_modules import copyfile from cmd_modules import listminions from cmd_modules import ping +from cmd_modules import check from func.overlord import client class FuncCommandLine(command.Command): + name = "func" - usage = "func is the commandline interface to a func minion" + usage = "func is the command line interface for controlling func minions" - subCommandClasses = [call.Call, show.Show, - copyfile.CopyFile, listminions.ListMinions, ping.Ping] + subCommandClasses = [ + call.Call, show.Show, copyfile.CopyFile, + listminions.ListMinions, ping.Ping, check.CheckAction + ] def __init__(self): diff --git a/func/overlord/groups.py b/func/overlord/groups.py index a0a9d78..7097366 100644 --- a/func/overlord/groups.py +++ b/func/overlord/groups.py @@ -24,7 +24,6 @@ import ConfigParser -import os class Groups(object): diff --git a/func/utils.py b/func/utils.py index 4aca97e..9577bd9 100755 --- a/func/utils.py +++ b/func/utils.py @@ -14,7 +14,6 @@ import os import string import sys import traceback -import xmlrpclib import socket REMOTE_ERROR = "REMOTE_ERROR" |