Restructured the rteval source tree a bit

This breaks rteval's functionality until the code has been
adopted to this new layout.

Signed-off-by: David Sommerseth <davids@redhat.com>

3 files changed, 387 insertions, 0 deletions

diff --git a/rteval/modules/loads/__init__.py b/rteval/modules/loads/__init__.py
new file mode 100644
index 0000000..5d76999
--- /dev/null
+++ b/rteval/modules/loads/__init__.py
@@ -0,0 +1,96 @@
+#
+#   Copyright 2009,2010   Clark Williams <williams@redhat.com>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import sys
+import os
+import os.path
+import time
+import subprocess
+import threading
+
+class Load(threading.Thread):
+    def __init__(self, name="<unnamed>", params={}):
+        threading.Thread.__init__(self)
+        self.name = name
+        self.builddir = params.setdefault('builddir', os.path.abspath("../build"))	# abs path to top dir
+        self.srcdir = params.setdefault('srcdir', os.path.abspath("../loadsource"))	# abs path to src dir
+        self.num_cpus = params.setdefault('numcores', 1)
+        self.debugging = params.setdefault('debugging', False)
+        self.source = params.setdefault('source', None)
+        self.reportdir = params.setdefault('reportdir', os.getcwd())
+        self.logging = params.setdefault('logging', False)
+        self.memsize = params.setdefault('memsize', (0, 'GB'))
+        self.params = params
+        self.ready = False
+        self.mydir = None
+        self.startevent = threading.Event()
+        self.stopevent = threading.Event()
+
+        if not os.path.exists(self.builddir):
+            os.makedirs(self.builddir)
+
+    def debug(self, str):
+        if self.debugging: print "%s: %s" % (self.name, str)
+
+    def isReady(self):
+        return self.ready
+
+    def shouldStop(self):
+        return self.stopevent.isSet()
+
+    def shouldStart(self):
+        return self.startevent.isSet()
+
+    def setup(self, builddir, tarball):
+        pass
+
+    def build(self, builddir):
+        pass
+
+    def runload(self, rundir):
+        pass
+
+    def run(self):
+        if self.shouldStop():
+            return
+        self.setup()
+        if self.shouldStop():
+            return
+        self.build()
+        while True:
+            if self.shouldStop():
+                return
+            self.startevent.wait(1.0)
+            if self.shouldStart():
+                break
+        self.runload()
+
+    def report(self):
+        pass
+
+    def genxml(self, x):
+        pass
+
+    def open_logfile(self, name):
+        return os.open(os.path.join(self.reportdir, "logs", name), os.O_CREAT|os.O_WRONLY)
diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py
new file mode 100644
index 0000000..4a112dd
--- /dev/null
+++ b/rteval/modules/loads/hackbench.py
@@ -0,0 +1,132 @@
+#  
+#   hackbench.py - class to manage an instance of hackbench load
+#
+#   Copyright 2009,2010   Clark Williams <williams@redhat.com>
+#   Copyright 2009,2010   David Sommerseth <davids@redhat.com>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+
+import sys
+import os
+import time
+import glob
+import subprocess
+import errno
+from signal import SIGTERM
+from signal import SIGKILL
+sys.pathconf = "."
+import load
+
+class Hackbench(load.Load):
+    def __init__(self, params={}):
+        load.Load.__init__(self, "hackbench", params)
+
+    def __del__(self):
+        null = open("/dev/null", "w")
+        subprocess.call(['killall', '-9', 'hackbench'], 
+                        stdout=null, stderr=null)
+        os.close(null)
+
+    def setup(self):
+        'calculate arguments based on input parameters'
+        (mem, units) = self.memsize
+        if units == 'KB':
+            mem = mem / (1024.0 * 1024.0)
+        elif units == 'MB':
+            mem = mem / 1024.0
+        elif units == 'TB':
+            mem = mem * 1024
+        ratio = float(mem) / float(self.num_cpus)
+        if ratio >= 0.75:
+            mult = float(self.params.setdefault('jobspercore', 2))
+        else:
+            print "hackbench: low memory system (%f GB/core)! Not running\n" % ratio
+            mult = 0
+        self.jobs = self.num_cpus * mult
+
+        self.args = ['hackbench',  '-P',
+                     '-g', str(self.jobs), 
+                     '-l', str(self.params.setdefault('loops', '100')),
+                     '-s', str(self.params.setdefault('datasize', '100'))
+                     ]
+        self.err_sleep = 5.0
+
+    def build(self):
+        self.ready = True
+
+    def start_hackbench(self, inf, outf, errf):
+        self.debug("running: %s" % " ".join(self.args))
+        return subprocess.Popen(self.args, stdin=inf, stdout=outf, stderr=errf)
+
+    def runload(self):
+        # if we don't have any jobs just wait for the stop event and return
+        if self.jobs == 0:
+            self.stopevent.wait()
+            return
+        null = os.open("/dev/null", os.O_RDWR)
+        if self.logging:
+            out = self.open_logfile("hackbench.stdout")
+            err = self.open_logfile("hackbench.stderr")
+        else:
+            out = err = null
+        self.debug("starting loop (jobs: %d)" % self.jobs)
+
+        p = self.start_hackbench(null, out, err)
+        while not self.stopevent.isSet():
+            try:
+                # if poll() returns an exit status, restart
+                if p.poll() != None:
+                    p = self.start_hackbench(null, out, err)
+                time.sleep(1.0)
+            except OSError, e:
+                if e.errno != errno.ENOMEM:
+                    raise
+                # Catch out-of-memory errors and wait a bit to (hopefully) 
+                # ease memory pressure
+                print "hackbench: %s, sleeping for %f seconds" % (e.strerror, self.err_sleep)
+                time.sleep(self.err_sleep)
+                if self.err_sleep < 60.0:
+                    self.err_sleep *= 2.0
+                if self.err_sleep > 60.0:
+                    self.err_sleep = 60.0
+
+        self.debug("stopping")
+        if p.poll() == None:
+            os.kill(p.pid, SIGKILL)
+        p.wait()
+        self.debug("returning from runload()")
+        os.close(null)
+        if self.logging:
+            os.close(out)
+            os.close(err)
+
+    def genxml(self, x):
+        x.taggedvalue('command_line', self.jobs and ' '.join(self.args) or None, 
+                      {'name':'hackbench', 'run': self.jobs and '1' or '0'})
+
+def create(params = {}):
+    return Hackbench(params)
+
+
+if __name__ == '__main__':
+    h = Hackbench(params={'debugging':True, 'verbose':True})
+    h.run()
diff --git a/rteval/modules/loads/kcompile.py b/rteval/modules/loads/kcompile.py
new file mode 100644
index 0000000..e4c9034
--- /dev/null
+++ b/rteval/modules/loads/kcompile.py
@@ -0,0 +1,159 @@
+#
+#   Copyright 2009,2010   Clark Williams <williams@redhat.com>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+#   For the avoidance of doubt the "preferred form" of this code is one which
+#   is in an open unpatent encumbered format. Where cryptographic key signing
+#   forms part of the process of creating an executable the information
+#   including keys needed to generate an equivalently functional executable
+#   are deemed to be part of the source code.
+#
+import sys
+import os
+import time
+import glob
+import subprocess
+from signal import SIGTERM
+sys.pathconf = "."
+import load
+import xmlout
+
+kernel_prefix="linux-2.6"
+
+class Kcompile(load.Load):
+    def __init__(self, params={}):
+        load.Load.__init__(self, "kcompile", params)
+
+    def setup(self):
+        # find our source tarball
+        if self.params.has_key('tarball'):
+            tarfile = os.path.join(self.srcdir, self.params.tarfile)
+            if not os.path.exists(tarfile):
+                raise RuntimeError, " tarfile %s does not exist!" % tarfile
+            self.source = tarfile
+        else:
+            tarfiles = glob.glob(os.path.join(self.srcdir, "%s*" % kernel_prefix))
+            if len(tarfiles):
+                self.source = tarfiles[0]
+            else:
+                raise RuntimeError, " no kernel tarballs found in %s" % self.srcdir
+
+        # check for existing directory
+        kdir=None
+        names=os.listdir(self.builddir)
+        for d in names:
+            if d.startswith(kernel_prefix):
+                kdir=d
+                break
+        if kdir == None:
+            self.debug("unpacking kernel tarball")
+            tarargs = ['tar', '-C', self.builddir, '-x']
+            if self.source.endswith(".bz2"):
+                tarargs.append("-j")
+            elif self.source.endswith(".gz"):
+                tarargs.append("-z")
+            tarargs.append("-f")
+            tarargs.append(self.source)
+            try:
+                subprocess.call(tarargs)
+            except:
+                self.debug("untar'ing kernel self.source failed!")
+                sys.exit(-1)
+            names = os.listdir(self.builddir)
+            for d in names:
+                self.debug("checking %s" % d)
+                if d.startswith(kernel_prefix):
+                    kdir=d
+                    break
+        if kdir == None:
+            raise RuntimeError, "Can't find kernel directory!"
+        self.mydir = os.path.join(self.builddir, kdir)
+        self.debug("mydir = %s" % self.mydir)
+
+    def build(self):
+        self.debug("setting up all module config file in %s" % self.mydir)
+        null = os.open("/dev/null", os.O_RDWR)
+        out = self.open_logfile("kcompile-build.stdout")
+        err = self.open_logfile("kcompile-build.stderr")
+        # clean up from potential previous run
+        try:
+            ret = subprocess.call(["make", "-C", self.mydir, "mrproper", "allmodconfig"], 
+                                  stdin=null, stdout=out, stderr=err)
+            if ret:
+                raise RuntimeError, "kcompile setup failed: %d" % ret
+        except KeyboardInterrupt, m:
+            self.debug("keyboard interrupt, aborting")
+            return
+        self.debug("ready to run")
+        self.ready = True
+        os.close(null)
+        os.close(out)
+        os.close(err)
+
+    def calc_numjobs(self):
+        mult = int(self.params.setdefault('jobspercore', 1))
+        mem = self.memsize[0]
+        if self.memsize[1] == 'KB':
+            mem = mem / (1024.0 * 1024.0)
+        elif self.memsize[1] == 'MB':
+            mem = mem / 1024.0
+        elif self.memsize[1] == 'TB':
+            mem = mem * 1024
+        ratio = float(mem) / float(self.num_cpus)
+        if ratio > 1.0:
+            njobs = self.num_cpus * mult
+        else:
+            self.debug("low memory system (%f GB/core)! Dropping jobs to one per core\n" % ratio)
+            njobs = self.num_cpus
+        return njobs
+
+    def runload(self):
+        null = os.open("/dev/null", os.O_RDWR)
+        if self.logging:
+            out = self.open_logfile("kcompile.stdout")
+            err = self.open_logfile("kcompile.stderr")
+        else:
+            out = err = null
+
+        njobs = self.calc_numjobs()
+        self.debug("starting loop (jobs: %d)" % njobs)
+        self.args = ["make", "-C", self.mydir, 
+                     "-j%d" % njobs ] 
+        p = subprocess.Popen(self.args, 
+                             stdin=null,stdout=out,stderr=err)
+        while not self.stopevent.isSet():
+            time.sleep(1.0)
+            if p.poll() != None:
+                r = p.wait()
+                self.debug("restarting compile job (exit status: %s)" % r)
+                p = subprocess.Popen(self.args,
+                                     stdin=null,stdout=out,stderr=err)
+        self.debug("out of stopevent loop")
+        if p.poll() == None:
+            self.debug("killing compile job with SIGTERM")
+            os.kill(p.pid, SIGTERM)
+        p.wait()
+        os.close(null)
+        if self.logging:
+            os.close(out)
+            os.close(err)
+
+    def genxml(self, x):
+        x.taggedvalue('command_line', ' '.join(self.args), {'name':'kcompile', 'run':'1'})
+
+def create(params = {}):
+    return Kcompile(params)
+