Add in some support for test coverage.

test/unittest/plugins/*:
	add a funccover nosetest plugin that can write out code coverage
	attribution information

cover_to_html.py:
	script to convert coverage information to html output.

This isn't integrated into the test scripts yet, but should work
for manually ran tests. See plugins/README for more info

4 files changed, 585 insertions, 0 deletions

diff --git a/test/unittest/cover_to_html.py b/test/unittest/cover_to_html.py
new file mode 100755
index 0000000..2d69242
--- /dev/null
+++ b/test/unittest/cover_to_html.py
@@ -0,0 +1,355 @@
+#!/usr/bin/python
+
+#Copyright (c) 2005 Drew Smathers
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy 
+# of this software and associated documentation files (the "Software"), to deal 
+# in the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 
+# of the Software, and to permit persons to whom the Software is furnished to do 
+# so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all 
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
+# SOFTWARE.
+
+
+
+#From http://svn.xix.python-hosting.com/trunk/xix/utils/cover.py
+
+"""More things for working with coverage.py
+"""
+
+from UserList import UserList
+import sys, os
+import glob
+from StringIO import StringIO
+import lxml.etree as ET
+
+__author__ = 'Drew Smathers'
+__version__ = '$Revision$'[11:-2]
+
+class AnnotationLine:
+    def __init__(self, text, isexec=False, covered=False):
+        self.text = text
+        self.isexec = isexec
+        self.covered = covered
+        # hack
+        if self.text and self.text[-1] == '\n':
+            self.text = self.text[:-1]
+
+    def __repr__(self):
+        buf = ""
+        buf = buf + self.text + "\n"
+        return buf
+
+
+class Annotation(UserList):
+    pass
+
+class AnnotationParser:
+    """Parser for annotation files generated by coverage.
+    """
+    def parse(self, input):
+        fd = input
+        if not hasattr(input, 'read'):
+            fd = open(input)
+        annotation = Annotation()
+        for line in input:
+#            print line
+            if not line or line[0] not in ('>', '!'):
+                annotation.append(AnnotationLine(line[2:], isexec=False))
+            elif line[0] == '>':
+                annotation.append(AnnotationLine(line[2:], isexec=True, covered=True))
+            elif line[0] == '!':
+                annotation.append(AnnotationLine(line[2:], isexec=True, covered=False))
+            else:
+                print >> sys.stderr, 'invalid annotation line: ' + line
+
+        return annotation
+
+def annotationToXML(annotation, tree=False):
+    """Transform annotation object to XML string or ElementTree instance
+    if tree is True.
+
+    @param annotation: Annotation instance
+    @param tree: set to true to return ElementTree instance.
+    """
+    etree = _annotationToXML(annotation)
+    if tree:
+        return etree
+    buffer = StringIO()
+    etree.write(buffer)
+    return buffer.getvalue()
+
+def _annotationToXML(annotation):
+    root = ET.Element('coverageAnnotation')
+    for line in annotation:
+        aline = ET.SubElement(root, 'line')
+        aline.attrib['executable'] = str(line.isexec).lower()
+        aline.attrib['covered'] = str(line.covered).lower()
+        aline.text = line.text
+    return ET.ElementTree(root)
+    
+
+def annotationToHTML(annotation, xml=None, xslt=None, tree=False):
+    """Transform annotation object to HTML string or ElementTree instance
+    if tree is True.
+
+    @param annotation: Annotation instance
+    @param xslt: xslt source for transformation
+    @param tree: set to true to return ElementTree instance.
+    """
+    style = xslt or _XSLT
+#    style = _XSLT
+    if not hasattr(style, 'read'):
+        style = StringIO(style)
+    style = ET.parse(style)
+    etree = _annotationToXML(annotation)
+    html = etree.xslt(style)
+    if tree:
+        return html
+    buffer = StringIO()
+    html.write(buffer)
+    return buffer.getvalue()
+
+###########################################
+# Coverage reports
+###########################################
+
+class CoverageReport(UserList):
+    package_name = None
+    summary = None
+
+class CoverageReportEntry:
+
+    def __init__(self, modname, statements, executed, coverage, missing):
+        self.modname = modname
+        self.statements = statements
+        self.executed = executed
+        self.coverage = coverage
+        self.missing = missing
+
+class CoverageReportParser:
+
+    def parse(self, input):
+        fd = input
+        if not hasattr(input, 'read'):
+            fd = open(input)
+        report = CoverageReport()
+        for line in fd.readlines()[2:]:
+            tokens = line.split()
+            try:
+                modname, stmts, execd, coverage = tokens[:4]
+                perc = coverage[:-1]
+                _stmts, _execd = int(stmts), int(execd)
+            except Exception, e:
+                continue
+            if modname == 'TOTAL':
+                report.summary = CoverageReportEntry(modname, stmts, execd, coverage, None)
+                break
+            modname = modname.replace(os.path.sep, '.')
+            missed = [ tk.replace(',','') for tk in tokens[4:] ]
+            report.append(CoverageReportEntry(modname, stmts, execd, coverage, missed))
+        return report
+
+
+def reportToXML(report, tree=False):
+    """Transform report object to XML representation as string or ElementTree
+    instance if tree arg is set to True.
+
+    @param report: report instance
+    @param tree: set to true to return ElementTree instance
+    """
+    etree = _reportToXML(report)
+    if tree:
+        return etree
+    buffer = StringIO()
+    etree.write(buffer)
+    return buffer.getvalue()
+
+def _reportToXML(report):
+    root = ET.Element('coverage-report')
+    elm = ET.SubElement(root, 'summary')
+    attr = elm.attrib
+    attr['statements'] = report.summary.statements
+    attr['executed'] = report.summary.executed
+    attr['coverage'] = report.summary.coverage
+    for entry in report:
+        elm = ET.SubElement(root, 'module')
+        attr = elm.attrib
+        attr['name'] = entry.modname
+        attr['statements'] = entry.statements
+        attr['coverage'] = entry.coverage
+        attr['executed'] = entry.executed
+        melm = ET.SubElement(elm, 'missing-ranges')
+        for missed in entry.missing:
+            start_end = missed.split('-')
+            if len(start_end) == 2:
+                start, end = start_end
+            else:
+                start = end = start_end[0]
+            relm = ET.SubElement(melm, 'range')
+            relm.attrib['start'] = start
+            relm.attrib['end'] = end
+    return ET.ElementTree(root)
+    
+
+def gen_html(path_to_cover, path_for_html):
+	cover_files = glob.glob("%s/*,cover" % path_to_cover)
+
+        # write out the css file
+        f = open("coverage.css", "w")
+        f.write(_CSS)
+        f.close()
+
+	for cf in cover_files:
+                fd = open(cf, "r")
+		ann = AnnotationParser().parse(fd) 
+		html = annotationToHTML(ann, xslt=_XSLT) 
+		base_name = os.path.basename(cf)
+		source_name = base_name.split(',')[0]
+		html_name = "%s/%s.html" % (path_for_html, source_name)
+		f = open(html_name, "w")
+		f.write(html)
+		f.close()
+
+#        fd = open("%s/cover.report" % path_to_cover)
+#        crp = CoverageReportParser().parse(fd)
+        
+        
+_CSS = """
+body {
+    margin: 0px;
+}
+
+h2 {
+   padding: 3px;
+   margin: 0px;
+   background-color: #ddd;
+   border-bottom: 2px solid;
+}
+
+th {
+    text-align: left;
+    padding-right: 28px;
+}
+
+.report-column {
+   border-bottom: 1px dashed;
+   padding-right: 28px;
+}
+
+.summary {
+   background-color: #eed;
+   border-bottom: 0px;
+}
+
+a {
+   text-decoration: none;
+}
+
+.annotation-line {
+   font-style: italic;
+   font-weight: 700;
+   font-family: mono,arial;
+   border-left: 8px solid #aaa;
+   padding-left: 5px;
+}
+
+.covered {
+   border-left: 8px solid #3e3;
+   background-color: #cfc;
+}
+
+.uncovered {
+    border-left: 8px solid red;
+}
+
+pre {
+   margin: 1px;
+   display: inline;
+}
+
+.uncovered {
+   background: #ebb;
+}
+
+.non-exec {
+    border-left: 8px solid #aaa;
+    background-color: #eee;
+}
+
+.lineno {
+    margin-right: 5px;
+    background-color: #ef5;
+}
+
+#colophon {
+    border-top: 1px solid;
+    background-color: #eee;
+    display: block;
+    position: relative;
+    bottom: 0px;
+    padding: 8px;
+    float: bottom;
+    width: 100%;
+    margin-top: 20px;
+    font-size: 75%; 
+    text-align: center;
+}
+
+"""
+
+
+_XSLT = '''<?xml version="1.0"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:param name="modname"/>
+<xsl:template match="/">
+    <html><head><title>Coverage Results for <xsl:value-of select="$modname"/></title>
+    <link rel="stylesheet" type="text/css" href="coverage.css"/></head>
+    <body>
+    <h2>Coverage Results for <xsl:value-of select="$modname"/></h2>
+    <div class="annotated-source-code">
+    <table width="100%">
+    <xsl:for-each select="/coverageAnnotation/line">
+        <xsl:variable name="lineno" select="position()"/>
+        <xsl:choose>
+            <xsl:when test="@executable='false'">
+                <!--<div class="annotation-line non-exec">-->
+                <tr class="annotation-line non-exec">
+                <td class="non-exec"><xsl:value-of select="$lineno"/></td>
+                <td><pre><xsl:value-of select="."/></pre></td></tr>
+            </xsl:when>
+            <xsl:when test="@executable='true' and @covered='true'">
+                <tr class="annotation-line executable covered">
+                <td class="covered"><xsl:value-of select="$lineno"/></td>
+                <td><pre><xsl:value-of select="."/></pre></td></tr>
+            </xsl:when>
+            <xsl:otherwise>
+                <tr class="annotation-line executable uncovered">
+                <td class="uncovered"><xsl:value-of select="$lineno"/></td>
+                <td><pre><xsl:value-of select="."/></pre></td></tr>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:for-each>
+    </table>
+    </div></body></html>
+</xsl:template>
+</xsl:stylesheet>'''
+
+
+
+
+if __name__ == "__main__":
+	gen_html(sys.argv[1], sys.argv[2])
+
+
diff --git a/test/unittest/plugins/README b/test/unittest/plugins/README
new file mode 100644
index 0000000..fd0c005
--- /dev/null
+++ b/test/unittest/plugins/README
@@ -0,0 +1,47 @@
+This is a modified version of the "cover" module that is include in the
+python nosetest module. 
+
+This version adds support for writing out attribution files
+
+To install, run "easy_install .", and it should install
+as a nosetest plugin egg. To use, 
+
+
+
+
+  --with-funccoverage   Enable plugin FuncCoverage:  If you have Ned
+                        Batchelder's coverage module installed, you may
+                        activate a coverage report. The coverage report will
+                        cover any python source module imported after the
+                        start of the test run, excluding modules that match
+                        testMatch. If you want to include those modules too,
+                        use the --cover-tests switch, or set the
+                        NOSE_COVER_TESTS environment variable to a true value.
+                        To restrict the coverage report to modules from a
+                        particular package or packages, use the --cover-
+                        packages switch or the NOSE_COVER_PACKAGES environment
+                        variable.  [NOSE_WITH_FUNCCOVERAGE]
+  --func-cover-package=COVER_PACKAGES
+                        Restrict coverage output to selected packages
+                        [FUNC_NOSE_COVER_PACKAGE]
+  --func-cover-erase    Erase previously collected coverage statistics before
+                        run
+  --func-cover-tests    Include test modules in coverage report
+                        [FUNC_NOSE_COVER_TESTS]
+  --func-cover-annotate
+                        write out annotated files[FUNC_NOSE_COVER_ANNOTATE]
+  --func-cover-dir=COVER_DIR
+                        directory to write data to[FUNC_NOSE_COVER_DIR]
+  --func-cover-inclusive
+                        Include all python files under working directory in
+                        coverage report.  Useful for discovering holes in test
+                        coverage if not all files are imported by the test
+                        suite. [FUNC_NOSE_COVER_INCLUSIVE]
+
+Example:
+
+	
+ nosetests --with-funccoverage --func-cover-dir data/ --func-cover-annotate --func-cover-erase --func-cover-package func -v -d -s test_client.py:TestTest
+
+To generate html'ified coverage reports, run "cover_to_html.py data/ html" from the unittest dir
+
diff --git a/test/unittest/plugins/funccover.py b/test/unittest/plugins/funccover.py
new file mode 100644
index 0000000..a45d9e1
--- /dev/null
+++ b/test/unittest/plugins/funccover.py
@@ -0,0 +1,159 @@
+"""If you have Ned Batchelder's coverage_ module installed, you may activate a
+coverage report with the --with-coverage switch or NOSE_WITH_COVERAGE
+environment variable. The coverage report will cover any python source module
+imported after the start of the test run, excluding modules that match
+testMatch. If you want to include those modules too, use the --cover-tests
+switch, or set the NOSE_COVER_TESTS environment variable to a true value. To
+restrict the coverage report to modules from a particular package or packages,
+use the --cover-packages switch or the NOSE_COVER_PACKAGES environment
+variable.
+
+.. _coverage: http://www.nedbatchelder.com/code/modules/coverage.html
+"""
+import logging
+import os
+import sys
+from nose.plugins.base import Plugin
+from nose.util import tolist
+
+log =  logging.getLogger(__name__)
+
+class FuncCoverage(Plugin):
+    """
+    If you have Ned Batchelder's coverage module installed, you may
+    activate a coverage report. The coverage report will cover any
+    python source module imported after the start of the test run, excluding
+    modules that match testMatch. If you want to include those modules too,
+    use the --cover-tests switch, or set the NOSE_COVER_TESTS environment
+    variable to a true value. To restrict the coverage report to modules from
+    a particular package or packages, use the --cover-packages switch or the
+    NOSE_COVER_PACKAGES environment variable.
+    """
+    coverTests = False
+    coverPackages = None
+    
+    def options(self, parser, env=os.environ):
+        Plugin.options(self, parser, env)
+        parser.add_option("--func-cover-package", action="append",
+                          default=env.get('FUNC_NOSE_COVER_PACKAGE'),
+                          dest="cover_packages",
+                          help="Restrict coverage output to selected packages "
+                          "[FUNC_NOSE_COVER_PACKAGE]")
+        parser.add_option("--func-cover-erase", action="store_true",
+                          default=env.get('FUNC_NOSE_COVER_ERASE'),
+                          dest="cover_erase",
+                          help="Erase previously collected coverage "
+                          "statistics before run")
+        parser.add_option("--func-cover-tests", action="store_true",
+                          dest="cover_tests",
+                          default=env.get('FUNC_NOSE_COVER_TESTS'),
+                          help="Include test modules in coverage report "
+                          "[FUNC_NOSE_COVER_TESTS]")
+        parser.add_option("--func-cover-annotate", action="store_true",
+                          dest="cover_annotate",
+                          help="write out annotated files"
+                          "[FUNC_NOSE_COVER_ANNOTATE]"),
+        parser.add_option("--func-cover-dir", action="store",
+                          dest="cover_dir",
+                          help="directory to write data to"
+                          "[FUNC_NOSE_COVER_DIR]"),
+
+        parser.add_option("--func-cover-inclusive", action="store_true",
+                          dest="cover_inclusive",
+                          default=env.get('FUNC_NOSE_COVER_INCLUSIVE'),
+                          help="Include all python files under working "
+                          "directory in coverage report.  Useful for "
+                          "discovering holes in test coverage if not all "
+                          "files are imported by the test suite. "
+                          "[FUNC_NOSE_COVER_INCLUSIVE]")
+        
+
+    def configure(self, options, config):
+        Plugin.configure(self, options, config)
+        if self.enabled:
+            try:
+                import coverage
+            except ImportError:
+                log.error("Coverage not available: "
+                          "unable to import coverage module")
+                self.enabled = False
+                return
+        self.conf = config
+        self.coverErase = options.cover_erase
+        self.coverTests = options.cover_tests
+        self.coverPackages = []
+        self.coverDir = options.cover_dir
+        self.coverAnnotate = options.cover_annotate
+        if options.cover_packages:
+            for pkgs in [tolist(x) for x in options.cover_packages]:
+                self.coverPackages.extend(pkgs)
+        self.coverInclusive = options.cover_inclusive
+        if self.coverPackages:
+            log.info("Coverage report will include only packages: %s",
+                     self.coverPackages)
+            
+    def begin(self):
+        log.debug("Coverage begin")
+        import coverage
+        self.skipModules = sys.modules.keys()[:]
+        if self.coverErase:
+            log.debug("Clearing previously collected coverage statistics")
+            coverage.erase()
+        coverage.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]')
+        coverage.start()
+        
+    def report(self, stream):
+        log.debug("Coverage report")
+        import coverage
+        coverage.stop()
+        modules = [ module
+                    for name, module in sys.modules.items()
+                    if self.wantModuleCoverage(name, module) ]
+        log.debug("Coverage report will cover modules: %s", modules)
+        if self.coverDir and self.coverAnnotate:
+            coverage.annotate(modules, self.coverDir)
+        fd = open("%s/cover.report" % self.coverDir, "w")
+        coverage.report(modules, file=fd)
+        fd.close()
+
+    def wantModuleCoverage(self, name, module):
+        if not hasattr(module, '__file__'):
+            log.debug("no coverage of %s: no __file__", name)
+            return False
+        root, ext = os.path.splitext(module.__file__)
+        if not ext in ('.py', '.pyc', '.pyo'):
+            log.debug("no coverage of %s: not a python file", name)
+            return False
+        if self.coverPackages:
+            for package in self.coverPackages:
+                if (name.startswith(package)
+                    and (self.coverTests
+                         or not self.conf.testMatch.search(name))):
+                    log.debug("coverage for %s", name)
+                    return True                
+        if name in self.skipModules:
+            log.debug("no coverage for %s: loaded before coverage start",
+                      name)
+            return False
+        if self.conf.testMatch.search(name) and not self.coverTests:
+            log.debug("no coverage for %s: is a test", name)
+            return False
+        # accept any package that passed the previous tests, unless
+        # coverPackages is on -- in that case, if we wanted this
+        # module, we would have already returned True
+        return not self.coverPackages
+
+    def wantFile(self, file, package=None):    
+        """If inclusive coverage enabled, return true for all source files 
+        in wanted packages.
+        """
+        if self.coverInclusive:
+            if file.endswith(".py"): 
+                if package and self.coverPackages:
+                    for want in self.coverPackages:
+                        if package.startswith(want):
+                            return True
+                else:
+                    return True
+        return None
+    
diff --git a/test/unittest/plugins/setup.py b/test/unittest/plugins/setup.py
new file mode 100644
index 0000000..b3a6464
--- /dev/null
+++ b/test/unittest/plugins/setup.py
@@ -0,0 +1,24 @@
+import sys
+try:
+    import ez_setup
+    ez_setup.use_setuptools()
+except ImportError:
+    pass
+
+from setuptools import setup
+
+setup(
+    name='func coverage output pluin',
+    version='0.1',
+    author='Adrian Likins',
+    author_email = 'alikins@redhat.com',
+    description = 'extended coverage output',
+    license = 'public domain',
+    py_modules = ['funccover'],
+    entry_points = {
+        'nose.plugins': [
+            'funccoverplug = funccover:FuncCoverage'
+            ]
+        }
+
+    )