Merge remote branch 'davids/master_ipv4' into work

16 files changed, 530 insertions, 117 deletions

diff --git a/doc/release-checklist.txt b/doc/release-checklist.txt
index a7139a9..2a14f7e 100644
--- a/doc/release-checklist.txt
+++ b/doc/release-checklist.txt
@@ -19,4 +19,4 @@
 10. Create signed tag of the form v<major>.<minor> for the release:
     e.g.:  git tag -u williams@redhat.com  v1.10
 11. Push master branch back to origin
-
+    e.g.:  git push --tags origin
diff --git a/rteval.spec b/rteval.spec
index 342edb1..66bcdf8 100644
--- a/rteval.spec
+++ b/rteval.spec
@@ -2,7 +2,7 @@
 %{!?python_ver: %define python_ver %(%{__python} -c "import sys ; print sys.version[:3]")}
 
 Name:		rteval
-Version:	1.19
+Version:	1.21
 Release:	1%{?dist}
 Summary:	Utility to evaluate system suitability for RT Linux
 
@@ -63,7 +63,7 @@ rm -rf $RPM_BUILD_ROOT
 
 %dir %{_datadir}/%{name}
 
-%doc COPYING
+%doc COPYING doc/rteval.txt
 %{_mandir}/man8/rteval.8*
 %{_datadir}/%{name}/rteval_*.xsl
 %config(noreplace) %{_sysconfdir}/rteval.conf
@@ -71,6 +71,28 @@ rm -rf $RPM_BUILD_ROOT
 /usr/bin/rteval
 
 %changelog
+* Thu Apr 13 2010 Clark Williams <williams@redhat.com> - 1.21-1
+- from Luis Claudio Goncalves <lgoncalv@redhat.com>:
+  - remove unecessary wait() call in cyclictest.py
+  - close /dev/null after using it
+  - call subprocess.wait() when needed
+  - remove delayloop code in hackbench.py
+- from David Sommerseth <davids@redhat.com>:
+  - add SIGINT handler
+  - handle non-root user case
+  - process DMI warnings before command line arguments
+  - added --annotate feature to rteval
+  - updates to xmlrpc code
+
+* Thu Apr  6 2010 Clark Williams <williams@redhat.com> - 1.20-1
+- code fixes from Luis Claudio Goncalves <lgoncalv@redhat.com>
+- from David Sommerseth <davids@redhat.com>:
+  - xmlrpc server updates 
+  - cputopology.py for recording topology in xml
+  - added NUMA node recording for run data
+  - rpmlint fixes
+- added start of rteval whitepaper in docs dir
+
 * Tue Mar 16 2010 Clark Williams <williams@redhat.com> - 1.19-1
 - add ability for --summarize to read tarfiles
 - from David Sommerseth <davids@redhat.com>
diff --git a/rteval/cputopology.py b/rteval/cputopology.py
index 61ef982..4b696f7 100644
--- a/rteval/cputopology.py
+++ b/rteval/cputopology.py
@@ -99,12 +99,24 @@ class CPUtopology:
     def getCPUsockets(self):
         return self.__cpu_sockets
 
-if __name__ == '__main__':
-    cputop = CPUtopology()
-    n = cputop.parse()
-
-    x = libxml2.newDoc('1.0')
-    x.setRootElement(n)
-    x.saveFormatFileEnc('-','UTF-8',1)
+def unit_test(rootdir):
+    try:
+        cputop = CPUtopology()
+        n = cputop.parse()
+
+        print " ---- XML Result ---- "
+        x = libxml2.newDoc('1.0')
+        x.setRootElement(n)
+        x.saveFormatFileEnc('-','UTF-8',1)
+
+        print " ---- getCPUcores() / getCPUscokets() ---- "
+        print "CPU cores: %i (online: %i) - CPU sockets: %i" % (cputop.getCPUcores(False),
+                                                                cputop.getCPUcores(True),
+                                                                cputop.getCPUsockets())
+        return 0
+    except Exception, e:
+        print "** EXCEPTION %s", str(e)
+        return 1
 
-    # print "CPU cores: %i (online: %i) - CPU sockets: %i" % (cputop.getCPUcores(False), cputop.getCPUcores(True), cputop.getCPUsockets())
+if __name__ == '__main__':
+    unit_test()
diff --git a/rteval/cyclictest.py b/rteval/cyclictest.py
index 5a3ffbf..7970f35 100644
--- a/rteval/cyclictest.py
+++ b/rteval/cyclictest.py
@@ -233,7 +233,8 @@ class Cyclictest(Thread):
                 break
             time.sleep(1.0)
         self.debug("stopping")
-        os.kill(c.pid, signal.SIGINT)
+        if c.poll() == None:
+            os.kill(c.pid, signal.SIGINT)
         # now parse the histogram output
         for line in c.stdout:
             if line.startswith('#'): continue
@@ -245,6 +246,7 @@ class Cyclictest(Thread):
         for n in self.data.keys():
             self.data[n].reduce()
         self.finished.set()
+        os.close(null)
 
     def genxml(self, x):
         x.openblock('cyclictest')
diff --git a/rteval/dmi.py b/rteval/dmi.py
index cd1b05e..7d1ab19 100644
--- a/rteval/dmi.py
+++ b/rteval/dmi.py
@@ -33,6 +33,28 @@ import libxml2
 import libxslt
 import dmidecode
 
+def ProcessWarnings():
+    if not hasattr(dmidecode, 'get_warnings'):
+        return
+
+    warnings = dmidecode.get_warnings()
+    if warnings == None:
+        return
+
+    for warnline in warnings.split('\n'):
+        # Ignore these warnings, as they are "valid" if not running as root
+        if warnline == '/dev/mem: Permission denied':
+            continue
+        if warnline == 'No SMBIOS nor DMI entry point found, sorry.':
+            continue
+
+        # All other warnings will be printed
+        if len(warnline) > 0:
+            print "** DMI WARNING ** %s" % warnline
+
+    dmidecode.clear_warnings()
+
+
 class DMIinfo(object):
     '''class used to obtain DMI info via python-dmidecode'''
 
@@ -64,12 +86,35 @@ class DMIinfo(object):
         xml.AppendXMLnodes(node)
 
 
-if __name__ == '__main__':
+def unit_test(rootdir):
     from pprint import pprint
-    
-    d = DMIinfo('.')
-    x = xmlout.XMLOut('dmi_test', "0.0")
-    x.NewReport()
-    d.genxml(x)
-    x.close()
-    x.Write('-')
+
+    class unittest_ConfigDummy(object):
+        def __init__(self, rootdir):
+            self.config = {'installdir': '%s/rteval'}
+            self.__update_vars()
+
+        def __update_vars(self):
+            for k in self.config.keys():
+                self.__dict__[k] = self.config[k]
+
+    try:
+        ProcessWarnings()
+        if os.getuid() != 0:
+            print "** ERROR **  Must be root to run this unit_test()"
+            return 1
+
+        cfg = unittest_ConfigDummy(rootdir)
+        d = DMIinfo(cfg)
+        x = xmlout.XMLOut('dmi_test', "0.0")
+        x.NewReport()
+        d.genxml(x)
+        x.close()
+        x.Write('-')
+        return 0
+    except Exception, e:
+        print "** EXCEPTION: %s" % str(e)
+        return 1
+
+if __name__ == '__main__':
+    sys.exit(unit_test('.'))
diff --git a/rteval/hackbench.py b/rteval/hackbench.py
index 65d0133..685b8d6 100644
--- a/rteval/hackbench.py
+++ b/rteval/hackbench.py
@@ -30,6 +30,7 @@ import time
 import glob
 import subprocess
 from signal import SIGTERM
+from signal import SIGKILL
 sys.pathconf = "."
 import load
 
@@ -41,6 +42,7 @@ class Hackbench(load.Load):
         null = open("/dev/null", "w")
         subprocess.call(['killall', '-9', 'hackbench'], 
                         stdout=null, stderr=null)
+        os.close(null)
 
     def setup(self):
         # find our tarball
@@ -93,6 +95,7 @@ class Hackbench(load.Load):
             raise RuntimeError, "Can't find hackbench executable: %s" % self.exe
         self.args = [self.exe, str(self.jobs)]
         self.ready = True
+        os.close(null)
 
     def runload(self):
         null = os.open("/dev/null", os.O_RDWR)
@@ -104,15 +107,11 @@ class Hackbench(load.Load):
                 p.wait()
                 p = subprocess.Popen(self.args,stdin=null,stdout=null,stderr=null)
         self.debug("stopping")
-        os.kill(p.pid, SIGTERM)
-        count = 30
-        while count > 0 and p.poll() == None:
-            time.sleep(1.0)
-            count -= 1
         if p.poll() == None:
             os.kill(p.pid, SIGKILL)
         p.wait()
         self.debug("returning from runload()")
+        os.close(null)
 
     def genxml(self, x):
         x.taggedvalue('command_line', ' '.join(self.args), {'name':'hackbench'})
diff --git a/rteval/kcompile.py b/rteval/kcompile.py
index 1e0863a..f57a18b 100644
--- a/rteval/kcompile.py
+++ b/rteval/kcompile.py
@@ -97,6 +97,7 @@ class Kcompile(load.Load):
             return
         self.debug("ready to run")
         self.ready = True
+        os.close(null)
 
     def runload(self):
         null = os.open("/dev/null", os.O_RDWR)
@@ -118,7 +119,10 @@ class Kcompile(load.Load):
                 p = subprocess.Popen(self.args,
                                      stdin=null,stdout=null,stderr=null)
         self.debug("stopping")
-        os.kill(p.pid, SIGTERM)
+        if p.poll() == None:
+            os.kill(p.pid, SIGTERM)
+        p.wait()
+        os.close(null)
 
     def genxml(self, x):
         x.taggedvalue('command_line', ' '.join(self.args), {'name':'kcompile'})
diff --git a/rteval/rteval.py b/rteval/rteval.py
index 8522dbc..8fcaf1f 100644
--- a/rteval/rteval.py
+++ b/rteval/rteval.py
@@ -44,6 +44,7 @@ import optparse
 import tempfile
 import statvfs
 import shutil
+import signal
 import rtevalclient
 import ethtool
 import xmlrpclib
@@ -59,11 +60,17 @@ import rtevalConfig
 import rtevalMailer
 from cputopology import CPUtopology
 
+
+sigint_received = False
+def sigint_handler(signum, frame):
+    global sigint_received
+    sigint_received = True
+    print "*** SIGINT received - stopping rteval run ***"
+
+
 class RtEval(object):
     def __init__(self, cmdargs):
-        if os.getuid() != 0:
-            raise RuntimeError, "must be root to run rteval"
-        self.version = "1.19"
+        self.version = "1.21"
         self.load_modules = []
         self.workdir = os.getcwd()
         self.inifile = None
@@ -134,6 +141,7 @@ class RtEval(object):
         self.kthreads = None
         self.xml = None
         self.baseos = "unknown"
+        self.annotate = self.cmd_options.annotate
 
         if not self.config.xslt_report.startswith(self.config.installdir):
             self.config.xslt_report = os.path.join(self.config.installdir, "rteval_text.xsl")
@@ -156,18 +164,45 @@ class RtEval(object):
         res = None
         if self.config.xmlrpc:
             self.debug("Checking if XML-RPC server '%s' is reachable" % self.config.xmlrpc)
-            try:
-                client = rtevalclient.rtevalclient("http://%s/rteval/API1/" % self.config.xmlrpc)
-                res = client.Hello()
-            except xmlrpclib.ProtocolError:
-                # Server do not support Hello(), but is reachable
-                self.info("Got XML-RPC connection with %s but it did not support Hello()"
-                          % self.config.xmlrpc)
-                res = None
-            except socket.error, err:
-                self.info("Could not establish XML-RPC contact with %s\n%s"
-                          % (self.config.xmlrpc, str(err)))
-                sys.exit(2)
+            attempt = 0
+            warning_sent = False
+            ping_failed = False
+            while attempt < 6:
+                try:
+                    client = rtevalclient.rtevalclient("http://%s/rteval/API1/" % self.config.xmlrpc)
+                    res = client.Hello()
+                    attempt = 10
+                    ping_failed = False
+                except xmlrpclib.ProtocolError:
+                    # Server do not support Hello(), but is reachable
+                    self.info("Got XML-RPC connection with %s but it did not support Hello()"
+                              % self.config.xmlrpc)
+                    res = None
+                except socket.error, err:
+                    self.info("Could not establish XML-RPC contact with %s\n%s"
+                              % (self.config.xmlrpc, str(err)))
+
+                    if (self.mailer is not None) and (not warning_sent):
+                        self.mailer.SendMessage("[RTEVAL:WARNING] Failed to ping XML-RPC server",
+                                                "Server %s did not respond.  Not giving up yet."
+                                                % self.config.xmlrpc)
+                        warning_sent = True
+
+                    # Do attempts handling
+                    attempt += 1
+                    if attempt > 5:
+                        break # To avoid sleeping before we abort
+
+                    print "Failed pinging XML-RPC server.  Doing another attempt(%i) " % attempt
+                    time.sleep(attempt*15) # Incremental sleep - sleep attempts*15 seconds
+                    ping_failed = True
+
+            if ping_failed:
+                if not self.cmd_options.xmlrpc_noabort:
+                    print "ERROR: Could not reach XML-RPC server '%s'.  Aborting." % self.config.xmlrpc
+                    sys.exit(2)
+                else:
+                    print "WARNING: Could not ping the XML-RPC server.  Will continue anyway."
 
             if res:
                 self.info("Verified XML-RPC connection with %s (XML-RPC API version: %i)"
@@ -320,6 +355,9 @@ class RtEval(object):
         parser.add_option("-X", '--xmlrpc-submit', dest='xmlrpc',
                           action='store', default=self.config.xmlrpc, metavar='HOST',
                           help='Hostname to XML-RPC server to submit reports')
+        parser.add_option("-P", "--xmlrpc-no-abort", dest="xmlrpc_noabort",
+                          action='store_true', default=False,
+                          help="Do not abort if XML-RPC server do not respond to ping request");
         parser.add_option("-Z", '--summarize', dest='summarize',
                           action='store_true', default=False,
                           help='summarize an already existing XML report')
@@ -329,6 +367,9 @@ class RtEval(object):
         parser.add_option("-f", "--inifile", dest="inifile",
                           type='string', default=None,
                           help="initialization file for configuring loads and behavior")
+        parser.add_option("-a", "--annotate", dest="annotate",
+                          type="string", default=None,
+                          help="Add a little annotation which is stored in the report")
 
         (self.cmd_options, self.cmd_arguments) = parser.parse_args(args = cmdargs)
         if self.cmd_options.duration:
@@ -398,6 +439,8 @@ class RtEval(object):
                                  'seconds': seconds})
         self.xmlreport.taggedvalue('date', self.start.strftime('%Y-%m-%d'))
         self.xmlreport.taggedvalue('time', self.start.strftime('%H:%M:%S'))
+        if self.annotate:
+            self.xmlreport.taggedvalue('annotate', self.annotate)
         self.xmlreport.closeblock()
         self.xmlreport.openblock('uname')
         self.xmlreport.taggedvalue('node', node)
@@ -598,6 +641,7 @@ class RtEval(object):
         if minutes: r = r - (minutes * 60)
         print "rteval time remaining: %d days, %d hours, %d minutes, %d seconds" % (days, hours, minutes, r)
 
+
     def measure(self):
         # Collect misc system info
         self.baseos = self.get_base_os()
@@ -669,11 +713,12 @@ class RtEval(object):
             report_interval = int(self.config.GetSection('rteval').report_interval)
 
             # wait for time to expire or thread to die
+            signal.signal(signal.SIGINT, sigint_handler)
             self.info("waiting for duration (%f)" % self.config.duration)
             stoptime = (time.time() + self.config.duration)
             currtime = time.time()
             rpttime = currtime + report_interval
-            while currtime <= stoptime:
+            while (currtime <= stoptime) and not sigint_received:
                 time.sleep(1.0)
                 if not self.cyclictest.isAlive():
                     raise RuntimeError, "cyclictest thread died!"
@@ -687,7 +732,7 @@ class RtEval(object):
                     self.show_remaining_time(left_to_run)
                     rpttime = currtime + report_interval
                 currtime = time.time()
-                
+            signal.signal(signal.SIGINT, signal.SIG_DFL)
                 
         finally:
             # stop cyclictest
@@ -801,6 +846,9 @@ class RtEval(object):
         ''' main function for rteval'''
         retval = 0;
 
+        # Parse initial DMI decoding errors
+        dmi.ProcessWarnings()
+
         # if --summarize was specified then just parse the XML, print it and exit
         if self.cmd_options.summarize or self.cmd_options.rawhistogram:
             if len(self.cmd_arguments) < 1:
@@ -815,7 +863,7 @@ class RtEval(object):
             sys.exit(0)
 
         if os.getuid() != 0:
-            print "Must be root to run evaluator!"
+            print "Must be root to run rteval!"
             sys.exit(-1)
 
         self.debug('''rteval options: 
@@ -854,6 +902,9 @@ if __name__ == '__main__':
     import pwd, grp
 
     try:
+        # Parse initial DMI decoding errors
+        dmi.ProcessWarnings()
+
         rteval = RtEval(sys.argv[1:])
         ec = rteval.rteval()
         sys.exit(ec)
diff --git a/rteval/rtevalConfig.py b/rteval/rtevalConfig.py
index 5fad260..bb77da2 100644
--- a/rteval/rtevalConfig.py
+++ b/rteval/rtevalConfig.py
@@ -185,3 +185,19 @@ class rtevalConfig(rtevalCfgSection):
             return rtevalCfgSection(self.__config_data[section])
         except KeyError, err:
             raise KeyError("The section '%s' does not exist in the config file" % section)
+
+
+def unit_test(rootdir):
+    try:
+        cfg = rtevalConfig()
+        cfg.Load(rootdir + '/rteval/rteval.conf')
+        print cfg
+        return 0
+    except Exception, e:
+        print "** EXCEPTION %s", str(e)
+        return 1
+
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(unit_test('..'))
diff --git a/rteval/rteval_text.xsl b/rteval/rteval_text.xsl
index d11cda8..2e4d385 100644
--- a/rteval/rteval_text.xsl
+++ b/rteval/rteval_text.xsl
@@ -18,6 +18,11 @@
     <xsl:value-of select="run_info/@hours"/><xsl:text>h </xsl:text>
     <xsl:value-of select="run_info/@minutes"/><xsl:text>m </xsl:text>
     <xsl:value-of select="run_info/@seconds"/><xsl:text>s</xsl:text>
+    <xsl:text>&#10;</xsl:text>
+    <xsl:if test="run_info/annotate">
+      <xsl:text>   Remarks:      </xsl:text>
+      <xsl:value-of select="run_info/annotate"/>
+    </xsl:if>
     <xsl:text>&#10;&#10;</xsl:text>
 
     <xsl:text>   Tested node:  </xsl:text>
diff --git a/rteval/xmlout.py b/rteval/xmlout.py
index b4fe371..648131b 100644
--- a/rteval/xmlout.py
+++ b/rteval/xmlout.py
@@ -257,58 +257,76 @@ class XMLOut(object):
         return self.currtag.addChild(nodes)
 
 
+def unit_test(rootdir):
+    try:
+        x = XMLOut('rteval', 'UNIT-TEST', None, 'UTF-8')
+        x.NewReport()
+        x.openblock('run_info', {'days': 0, 'hours': 0, 'minutes': 32, 'seconds': 18})
+        x.taggedvalue('time', '11:22:33')
+        x.taggedvalue('date', '2000-11-22')
+        x.closeblock()
+        x.openblock('uname')
+        x.taggedvalue('node', u'testing - \xe6\xf8')
+        x.taggedvalue('kernel', 'my_test_kernel', {'is_RT': 0})
+        x.taggedvalue('arch', 'mips')
+        x.closeblock()
+        x.openblock('hardware')
+        x.taggedvalue('cpu_cores', 2)
+        x.taggedvalue('memory_size', 1024*1024*2)
+        x.closeblock()
+        x.openblock('loads', {'load_average': 3.29})
+        x.taggedvalue('command_line','./load/loader --extreme --ultimate --threads 4096', 
+                      {'name': 'heavyloader'})
+        x.taggedvalue('command_line','dd if=/dev/zero of=/dev/null', {'name': 'lightloader'})
+        x.closeblock()
+        x.close()
+        print "------------- XML OUTPUT ----------------------------"
+        x.Write("-")
+        print "------------- XSLT PARSED OUTPUT --------------------"
+        x.Write("-", "rteval_text.xsl")
+        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        x.Write("/tmp/xmlout-test.xml")
+        del x
+
+        print "------------- LOAD XML FROM FILE -----------------------------"
+        x = XMLOut('rteval','UNIT-TEST', None, 'UTF-8')
+        x.LoadReport("/tmp/xmlout-test.xml", True)
+        print "------------- LOADED XML DATA --------------------------------"
+        x.Write("-")
+        print "------------- XSLT PARSED OUTPUT FROM LOADED XML--------------"
+        x.Write("-", "rteval_text.xsl")
+        x.close()
+
+        ##  Test new data parser ... it eats most data types
+        print "------------- TESTING XMLOut::ParseData() --------------"
+        x.NewReport()
+        x.ParseData("ParseTest", "test string", {"type": "simple_string"})
+        x.ParseData("ParseTest", 1234, {"type": "integer"})
+        x.ParseData("ParseTest", 39.3904, {"type": "float"})
+        x.ParseData("ParseTest", (11,22,33,44,55), {"type": "tuples"})
+        x.ParseData("ParseTest", (99,88,77), {"type": "tuples", "comment": "Changed default tuple tag name"},
+                    "int_values")
+        test = {"var1": "value 1",
+                "var2": { "varA1": 1,
+                          "pi": 3.1415926,
+                          "varA3": (1,
+                                    2,
+                                    {"test1": "val1"},
+                                    (4.1,4.2,4.3),
+                                    5),
+                          "varA4": {'another_level': True,
+                                    'another_value': "blabla"}
+                          },
+                "utf8 data": u'æøå',
+                u"løpe": True}
+        x.ParseData("ParseTest", test, {"type": "dict"}, prefix="test ")
+        x.close()
+        x.Write("-")
+        return 0
+    except Exception, e:
+        print "** EXCEPTION %s", str(e)
+        return 1
+
 if __name__ == '__main__':
-    x = XMLOut('rteval', '0.6', None, 'UTF-8')
-    x.NewReport()
-    x.openblock('run_info', {'days': 0, 'hours': 0, 'minutes': 32, 'seconds': 18})
-    x.taggedvalue('time', '11:22:33')
-    x.taggedvalue('date', '2000-11-22')
-    x.closeblock()
-    x.openblock('uname')
-    x.taggedvalue('node', u'testing - \xe6\xf8')
-    x.taggedvalue('kernel', 'my_test_kernel', {'is_RT': 0})
-    x.taggedvalue('arch', 'mips')
-    x.closeblock()
-    x.openblock('hardware')
-    x.taggedvalue('cpu_cores', 2)
-    x.taggedvalue('memory_size', 1024*1024*2)
-    x.closeblock()
-    x.openblock('loads', {'load_average': 3.29})
-    x.taggedvalue('command_line','./load/loader --extreme --ultimate --threads 4096', {'name': 'heavyloader'})
-    x.taggedvalue('command_line','dd if=/dev/zero of=/dev/null', {'name': 'lightloader'})
-    x.closeblock()
-    x.close()
-    print "------------- XML OUTPUT ----------------------------"
-    x.Write("-")
-    print "------------- XSLT PARSED OUTPUT --------------------"
-    x.Write("-", "rteval_text.xsl")
-    print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    x.LoadReport("latency.xml", True)
-    x.Write("-")
-    x.Write("-", "rteval_text.xsl")
-    x.close()
-
-    ##  Test new data parser ... it eats most data types
-    x.NewReport()
-    x.ParseData("ParseTest", "test string", {"type": "simple_string"})
-    x.ParseData("ParseTest", 1234, {"type": "integer"})
-    x.ParseData("ParseTest", 39.3904, {"type": "float"})
-    x.ParseData("ParseTest", (11,22,33,44,55), {"type": "tuples"})
-    x.ParseData("ParseTest", (99,88,77), {"type": "tuples", "comment": "Changed default tuple tag name"},
-                "int_values")
-    test = {"var1": "value 1",
-            "var2": { "varA1": 1,
-                      "pi": 3.1415926,
-                      "varA3": (1,
-                                2,
-                                {"test1": "val1"},
-                                (4.1,4.2,4.3),
-                                5),
-                      "varA4": {'another_level': True,
-                                'another_value': "blabla"}
-                      },
-            "utf8 data": u'æøå',
-            u"løpe": True}
-    x.ParseData("ParseTest", test, {"type": "dict"}, prefix="test ")
-    x.close()
-    x.Write("-")
+    sys.exit(unit_test('..'))
+
diff --git a/server/parser/xmlparser.xsl b/server/parser/xmlparser.xsl
index 0b0df19..4330fec 100644
--- a/server/parser/xmlparser.xsl
+++ b/server/parser/xmlparser.xsl
@@ -159,7 +159,7 @@
               </value>
               <value fid="4" type="xmlblob">
                 <rteval_details>
-                  <xsl:copy-of select="clocksource|services|kthreads|network_config|loads|cyclictest/command_line"/>
+                  <xsl:copy-of select="clocksource|services|kthreads|network_config|loads|cyclictest/command_line|run_info/annotate"/>
                   <hardware>
                     <xsl:copy-of select="hardware/memory_size|hardware/cpu_topology"/>
                   </hardware>
diff --git a/server/rteval_testserver.py b/server/rteval_testserver.py
index 927c70a..49bbfd7 100644
--- a/server/rteval_testserver.py
+++ b/server/rteval_testserver.py
@@ -44,11 +44,39 @@ class RequestHandler(SimpleXMLRPCRequestHandler):
     rpc_paths = ('/rteval/API1/',)
 
 
+class RTevald_config(object):
+    def __init__(self):
+        self.config = {'datadir': '/tmp/rteval-xmlrpc-testsrv',
+                       'db_server': 'localhost',
+                       'db_port': 5432,
+                       'database': 'dummy',
+                       'db_username': None,
+                       'db_password': None}
+        self.__update_vars()
+
+    def __update_vars(self):
+        for k in self.config.keys():
+            self.__dict__[k] = self.config[k]
+
+
 class RTevald():
     def __init__(self, options, log):
         self.options = options
         self.log = log
         self.server = None
+        self.config = RTevald_config()
+
+    def __prepare_datadir(self):
+        startdir = os.getcwd()
+        for dir in self.config.datadir.split("/"):
+            if dir is '':
+                continue
+            if not os.path.exists(dir):
+                os.mkdir(dir, 0700)
+            os.chdir(dir)
+        if not os.path.exists('queue'):
+            os.mkdir('queue', 0700)
+        os.chdir(startdir)
 
     def StartServer(self):
         # Create server
@@ -57,19 +85,21 @@ class RTevald():
         self.server.register_introspection_functions()
 
         # setup a class to handle requests
-        self.server.register_instance(xmlrpc_API1.XMLRPC_API1("./testsrv/", nodbaction=True, debug=True))
+        self.server.register_instance(xmlrpc_API1.XMLRPC_API1(self.config, nodbaction=True, debug=True))
 
         # Run the server's main loop
         self.log.Log("StartServer", "Listening on %s:%i" % (self.options.listen, self.options.port))
         try:
+            self.__prepare_datadir()
             self.server.serve_forever()
         except KeyboardInterrupt:
             self.log.Log("StartServer", "Server caught SIGINT")
-            pass
+            self.server.shutdown()
         finally:
             self.log.Log("StartServer", "Server stopped")
 
-
+    def StopServer(self):
+        self.server.shutdown()
 
 
 logger = None
diff --git a/server/testclient.py b/server/testclient.py
index 09dca6a..9632a7b 100644
--- a/server/testclient.py
+++ b/server/testclient.py
@@ -48,21 +48,9 @@ d.saveFormatFileEnc('-','UTF-8', 1)
 
 
 print "** Testing API"
-client = rtevalclient.rtevalclient("http://localhost:65432/rteval/API1")
-
-server_fname = client.SendDataAsFile('test.data', "this is just a simple test file, compressed\n")
-print "1:  File name on server: %s" % server_fname
-
-server_fname = client.SendDataAsFile('test.data',
-                                     "this is just a simple test file, uncompressed server side\n", True)
-print "2:  File name on server: %s" % server_fname
-
-server_fname = client.SendFile('test.log')
-print "3:  File name on server: %s" % server_fname
-
-server_fname = client.SendFile('test.log', True)
-print "4:  File name on server: %s" % server_fname
+client = rtevalclient.rtevalclient("http://localhost:65432/rteval/API1/")
 
+print "** 1: Hello(): %s" % str(client.Hello())
 status = client.SendReport(d)
-print "5:  SendReport(xmlDoc): %s" % status
+print "** 2: SendReport(xmlDoc): %s" % str(status)
 
diff --git a/server/unittest.py b/server/unittest.py
new file mode 100644
index 0000000..b22e583
--- /dev/null
+++ b/server/unittest.py
@@ -0,0 +1,90 @@
+import sys, threading, time, signal, libxml2
+from optparse import OptionParser
+from rteval_testserver import RTevald
+from Logger import Logger
+
+sys.path.insert(0,'..')
+sys.path.insert(0,'../rteval')
+sys.path.insert(0,'rteval')
+import rtevalclient
+
+class ServerThread(threading.Thread):
+    def __init__(self, port):
+        threading.Thread.__init__(self)
+        self.port = port
+        self.log = Logger('unit-test-server.log','rteval-xmlrpc-testsrv')
+
+        parser = OptionParser()
+        parser.add_option("-L", "--listen", action="store", dest="listen", default="127.0.0.1",
+                          help="Which interface to listen to [default: %default]", metavar="IPADDR")
+        parser.add_option("-P", "--port", action="store", type="int", dest="port", default=self.port,
+                          help="Which port to listen to [default: %default]",  metavar="PORT")
+
+        (options, args) = parser.parse_args()
+
+        self.child = RTevald(options, self.log)
+
+    def run(self):
+        self.child.StartServer()
+
+    def stop(self):
+        self.child.StopServer()
+
+    def sigcatch(self, signum, frame):
+        print "Shutting down"
+        self.stop()
+
+
+class ClientTest(object):
+    def __init__(self, port):
+        self.client = rtevalclient.rtevalclient("http://localhost:%s/rteval/API1/" % port)
+
+    def __prepare_data(self):
+        d = libxml2.newDoc("1.0")
+        n = libxml2.newNode('TestNode1')
+        d.setRootElement(n)
+        n2 = n.newTextChild(None, 'TestNode2','Just a little test')
+        n2.newProp('test','true')
+        for i in range(1,5):
+            n2 = n.newTextChild(None, 'TestNode3', 'Test line %i' %i)
+        self.testdoc = d
+
+    def RunTest(self):
+        try:
+            print "** Creating XML test document"
+            self.__prepare_data()
+            self.testdoc.saveFormatFileEnc('-','UTF-8', 1)
+
+            print "** Client test [1]: Hello(): %s" % str(self.client.Hello())
+            status = self.client.SendReport(self.testdoc)
+            print "** Client test [2]; SendReport(xmlDoc): %s" % str(status)
+        except Exception, e:
+            raise Exception("XML-RPC client test failed: %s" % str(e))
+
+
+def unit_test(rootdir):
+    ret = 1
+    try:
+        # Prepare server and client objects
+        srvthread = ServerThread('65432')
+        clienttest = ClientTest('65432')
+        signal.signal(signal.SIGINT, srvthread.sigcatch)
+
+        # Start a local XML-RPC test server
+        srvthread.start()
+        print "** Waiting 2 seconds for server to settle"
+        time.sleep(2)
+
+        # Start the client test
+        print "** Starting client tests"
+        clienttest.RunTest()
+        ret = 0
+    except Exception, e:
+        print "** EXCEPTION: %s" % str(e)
+    finally:
+        # Stop the local XML-RPC test server
+        srvthread.stop()
+    return ret
+
+if __name__ == "__main__":
+    sys.exit(unit_test('..'))
diff --git a/unit-tests/unittest.py b/unit-tests/unittest.py
new file mode 100755
index 0000000..65a3616
--- /dev/null
+++ b/unit-tests/unittest.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# Copyright (c) 2010 Red Hat, Inc. All rights reserved. This copyrighted material
+# is made available to anyone wishing to use, modify, copy, or
+# redistribute it subject to the terms and conditions of the GNU General
+# Public License v.2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# Author: David Sommerseth <davids@redhat.com>
+#
+
+import os, sys
+
+class UnitTest(object):
+    "Unified unit test class"
+
+    def __init__(self, srcrootdir):
+        "UnitTest constructor.  srcrootdir argument must point at the source root directory"
+        self.imported_mods = []
+        self.mod_impcount = 0
+        self.mod_impfail = 0
+        self.mod_testpass = 0
+        self.mod_testfail = 0
+        self.mod_testmiss = 0
+        self.rootdir = srcrootdir
+        sys.path.insert(0, self.rootdir)
+
+
+    def LoadModules(self, modules):
+        """Loads all the defined modules.  The modules argument takes a tuple list
+ consisting of ('subdir','module name')"""
+
+        for (directory, mod) in modules:
+            # Check if the directory is in the "include" path
+            try:
+                sys.path.index('%s/%s' % (self.rootdir,directory))
+            except ValueError:
+                # Not found, insert it
+                sys.path.insert(0, '%s/%s' % (self.rootdir, directory))
+
+            try:
+                impmod = __import__(mod)
+                print "** Imported %s/%s" % (directory, mod)
+                self.imported_mods.append({'name': '%s/%s' %(directory, mod),
+                                           'module':  impmod})
+                self.mod_impcount += 1
+            except ImportError, e:
+                print "!! ** ERROR ** Failed to import %s/%s (Exception: %s)" % (directory, mod, str(e))
+                self.mod_impfail += 1
+
+        return True
+
+
+    def RunTests(self):
+        "Runs the unit_test() function in all successfully imported modules"
+
+        for m in self.imported_mods:
+            try:
+                # Check if the unit_test() function exists and is callable before trying
+                # to run the unit test
+                if callable(m['module'].unit_test):
+                    print
+                    print 78 * '-'
+                    print "** Running unit test for: %s" % m['name']
+                    print 78 * '.'
+                    res = m['module'].unit_test(self.rootdir)
+                    print 78 * '.'
+                    if res == 0:
+                        print "** Result of %s: PASS" % m['name']
+                        self.mod_testpass += 1
+                    else:
+                        print "** Result of %s: FAILED (return code: %s)" % (m['name'], str(res))
+                        self.mod_testfail += 1
+                    print 78 * '='
+                else:
+                    self.mod_testmiss += 1
+                    print "!!! ** ERROR **  Could not run %s::unit_test()" % m['name']
+            except AttributeError:
+                self.mod_testmiss += 1
+                print "!!! ** ERROR **  No %s::unit_test() method found" % m['name']
+
+
+    def PrintTestSummary(self):
+        "Prints a result summary of all the tests"
+        print
+        print " --------------------"
+        print "  ** TEST SUMMARY ** "
+        print " --------------------"
+        print
+        print "  - Modules:"
+        print "      Declared for test:      %i" % (self.mod_impcount + self.mod_impfail)
+        print "      Successfully imported:  %i" % self.mod_impcount
+        print "      Failed import:          %i" % self.mod_impfail
+        print
+        print "  - Tests:"
+        print "      Tests scheduled:        %i" % (self.mod_testpass + self.mod_testfail + self.mod_testmiss)
+        print "      Sucessfully tests:      %i" % self.mod_testpass
+        print "      Failed tests:           %i" % self.mod_testfail
+        print "      Missing unit_test()     %i" % self.mod_testmiss
+        print
+
+
+if __name__ == '__main__':
+
+    # Retrieve the root directory if the source dir
+    # - use the first occurence of the 'v7' subdir as the root dirq
+    srcrootdir_ar = os.getcwd().split('/')
+    rootdir = '/'.join(srcrootdir_ar[0:srcrootdir_ar.index('rteval')+1])
+    print "** Source root dir: %s" % rootdir
+
+    # Prepare the unit tester
+    tests = UnitTest(rootdir)
+
+    # Load defined modules  ('subdir','import name')
+    tests.LoadModules((
+            ('rteval','cputopology'),
+            ('rteval','dmi'),
+            ('rteval','rtevalConfig'),
+            ('rteval','xmlout'),
+            ('server','unittest')
+            ))
+    # Run all tests
+    tests.RunTests()
+    tests.PrintTestSummary()
+