Report generators for unit tests and memory checks. Configure with

--enable-testing.

21 files changed, 555 insertions, 0 deletions

diff --git a/report-generators/lib/log.rb b/report-generators/lib/log.rb
new file mode 100644
index 00000000..c98b1839
--- /dev/null
+++ b/report-generators/lib/log.rb
@@ -0,0 +1,30 @@
+# Merely wraps the logger library with a bit of standard policy.
+require 'logger'
+
+module Log
+  $log = Logger.new(STDERR)
+
+  def init(io_)
+    $log = Logger.new(io_)
+  end
+end
+
+def fatal(*args)
+  $log.fatal(*args)
+end
+
+def error(*args)
+  $log.error(*args)
+end
+
+def info(*args)
+  $log.info(*args)
+end
+
+def warning(*args)
+  $log.warn(*args)
+end
+
+def debug(*args)
+  $log.debug(*args)
+end
diff --git a/report-generators/lib/report_templates.rb b/report-generators/lib/report_templates.rb
new file mode 100644
index 00000000..83e3acbf
--- /dev/null
+++ b/report-generators/lib/report_templates.rb
@@ -0,0 +1,28 @@
+# Policy for the location of report templates
+require 'string-store'
+
+class TemplateStringStore < StringStore
+  def initialize()
+    super(['report-generators/templates'])
+  end
+end
+
+module ReportTemplates
+  def generate_report(report, bs, dest_path = nil)
+    include Reports
+    reports = ReportRegister.new
+    template_store = TemplateStringStore.new
+    report = reports.get_report(report)
+    erb = ERB.new(template_store.lookup(report.template))
+    body = erb.result(bs)
+    title = report.short_desc
+
+    erb = ERB.new(template_store.lookup("boiler_plate.rhtml"))
+    txt = erb.result(binding)
+
+    dest_path = dest_path.nil? ? report.path : dest_path
+    dest_path.open("w") do |out|
+      out.puts txt
+    end
+  end
+end
diff --git a/report-generators/lib/reports.rb b/report-generators/lib/reports.rb
new file mode 100644
index 00000000..57333324
--- /dev/null
+++ b/report-generators/lib/reports.rb
@@ -0,0 +1,48 @@
+# Data about the various reports we support
+require 'log'
+require 'pathname'
+
+module Reports
+  Report = Struct.new(:short_desc, :desc, :path, :template)
+
+  class ReportRegister
+    attr_reader :reports
+
+    private
+    def add_report(sym, *args)
+      @reports[sym] = Report.new(*args)
+    end
+
+    public
+    def initialize()
+      @reports = Hash.new
+
+      add_report(:unit_test,
+                 "Unit Tests",
+                 "unit tests",
+                 Pathname.new("reports/unit.html"),
+                 Pathname.new("unit_test.rhtml"))
+
+      add_report(:memcheck,
+                 "Memory Tests",
+                 "unit tests with valgrind memory checking",
+                 Pathname.new("reports/memcheck.html"),
+                 Pathname.new("memcheck.rhtml"))
+
+      add_report(:unit_detail,
+                 "Unit Test Detail",
+                 "unit test detail",
+                 Pathname.new("reports/unit_detail.html"), # FIXME replace this with a lambda
+                 Pathname.new("unit_detail.rhtml"))
+    end
+
+    def get_report(sym)
+      raise RuntimeError, "unknown report '#{sym}'" unless @reports.member?(sym)
+      @reports[sym]
+    end
+
+    def each(&block)
+      @reports.each(&block)
+    end
+  end
+end
diff --git a/report-generators/lib/schedule_file.rb b/report-generators/lib/schedule_file.rb
new file mode 100644
index 00000000..1e164f45
--- /dev/null
+++ b/report-generators/lib/schedule_file.rb
@@ -0,0 +1,46 @@
+# Parses the simple colon delimited test schedule files.
+
+ScheduledTest = Struct.new(:desc, :command_line, :status, :output)
+
+class Schedule
+  attr_reader :dir, :schedules
+
+  def initialize(dir, ss)
+    @dir = dir
+    @schedules = ss
+  end
+
+  def run
+    Dir::chdir(@dir.to_s) do
+      @schedules.each do |s|
+        reader, writer = IO.pipe
+        print "#{s.desc} ... "
+        pid = spawn(s.command_line, [ STDERR, STDOUT ] => writer)
+        writer.close
+        _, s.status = Process::waitpid2(pid)
+        puts (s.status.success? ? "pass" : "fail")
+        s.output = reader.read
+      end
+    end
+  end
+
+  def self.read(dir, io)
+    ss = Array.new
+
+    io.readlines.each do |line|
+      case line.strip
+      when /^\#.*/
+        next
+
+      when /([^:]+):(.*)/
+        ss << ScheduledTest.new($1.strip, $2.strip)
+
+      else
+        raise RuntimeError, "badly formatted schedule line"
+      end  
+    end
+
+    Schedule.new(dir, ss)
+  end
+end
+
diff --git a/report-generators/lib/string-store.rb b/report-generators/lib/string-store.rb
new file mode 100644
index 00000000..36d819a9
--- /dev/null
+++ b/report-generators/lib/string-store.rb
@@ -0,0 +1,32 @@
+# Provides a simple way of accessing the contents of files by a symbol
+# name.  Useful for erb templates.
+
+require 'pathname'
+
+class StringStore
+  attr_accessor :path
+
+  def initialize(p)
+    @paths = p.nil? ? Array.new : p # FIXME: do we need to copy p ?
+  end
+
+  def lookup(sym)
+    files = expansions(sym)
+
+    @paths.each do |p|
+      files.each do |f|
+        pn = Pathname.new("#{p}/#{f}")
+        if pn.file?
+          return pn.read
+        end
+      end
+    end
+
+    raise RuntimeError, "unknown string entry: #{sym}"
+  end
+
+  private
+  def expansions(sym)
+    ["#{sym}", "#{sym}.txt"]
+  end
+end
diff --git a/report-generators/memcheck.rb b/report-generators/memcheck.rb
new file mode 100644
index 00000000..fcc8927e
--- /dev/null
+++ b/report-generators/memcheck.rb
@@ -0,0 +1,76 @@
+# Reads the schedule files given on the command line.  Runs them and
+# generates the reports.
+
+# FIXME: a lot of duplication with unit_test.rb
+
+require 'schedule_file'
+require 'pathname'
+require 'reports'
+require 'erb'
+require 'report_templates'
+
+include ReportTemplates
+
+schedules = ARGV.map do |f|
+  p = Pathname.new(f)
+  Schedule.read(p.dirname, p)
+end
+
+total_passed = 0
+total_failed = 0
+
+# We need to make sure the lvm shared libs are in the LD_LIBRARY_PATH
+ENV['LD_LIBRARY_PATH'] = `pwd`.strip + "/libdm:" + (ENV['LD_LIBRARY_PATH'] || '')
+
+ENV['TEST_TOOL'] = "valgrind --leak-check=full --show-reachable=yes"
+
+schedules.each do |s|
+  s.run
+
+  s.schedules.each do |t|
+    if t.status.success?
+      total_passed += 1
+    else
+      total_failed += 1
+    end
+  end
+end
+
+def mangle(txt)
+  txt.gsub(/\s+/, '_')
+end
+
+MemcheckStats = Struct.new(:definitely_lost, :indirectly_lost, :possibly_lost, :reachable)
+
+def format(bytes, blocks)
+  "#{bytes} bytes, #{blocks} blocks"
+end
+
+# Examines the output for details of leaks
+def extract_stats(t)
+  d = i = p = r = '-'
+
+  t.output.split("\n").each do |l|
+    case l
+    when /==\d+==    definitely lost: ([0-9,]+) bytes in ([0-9,]+) blocks/
+        d = format($1, $2)
+    when /==\d+==    indirectly lost: ([0-9,]+) bytes in ([0-9,]+) blocks/
+        i = format($1, $2)
+    when /==\d+==    possibly lost: ([0-9,]+) bytes in ([0-9,]+) blocks/
+        p = format($1, $2)
+    when /==\d+==    still reachable: ([0-9,]+) bytes in ([0-9,]+) blocks/
+        r = format($1, $2)
+    end
+  end
+
+  MemcheckStats.new(d, i, p, r)
+end
+
+generate_report(:memcheck, binding)
+
+# now we generate a detail report for each schedule
+schedules.each do |s|
+  s.schedules.each do |t|
+    generate_report(:unit_detail, binding, Pathname.new("reports/memcheck_#{mangle(t.desc)}.html"))
+  end
+end
diff --git a/report-generators/templates/boiler_plate.rhtml b/report-generators/templates/boiler_plate.rhtml
new file mode 100644
index 00000000..23f01cbd
--- /dev/null
+++ b/report-generators/templates/boiler_plate.rhtml
@@ -0,0 +1,25 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=US-ASCII">
+<title><%= title %></title>
+<link title="Style" type="text/css" rel="stylesheet" href="stylesheet.css">
+</head>
+
+<body>
+<div id="banner">
+<h2><%= title %></h2>
+</div>
+<div id="main">
+  <div id="controls">
+    <table>
+      <tr><td><a href="index.html">Generation times</a></td></tr>
+      <tr><td><a href="unit.html">Unit tests</a></td></tr>
+      <tr><td><a href="memcheck.html">Memory tests</a></td></tr>
+    </table>
+  </div>
+
+  <div id="body">
+    <%= body %>
+  </div>
+</div>
+</body>
diff --git a/report-generators/templates/index.rhtml b/report-generators/templates/index.rhtml
new file mode 100644
index 00000000..6d72081f
--- /dev/null
+++ b/report-generators/templates/index.rhtml
@@ -0,0 +1,17 @@
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+<tr><th>Report</th><th>Generation time</th></tr>
+<% [:unit_test, :memcheck].each do |sym| %>
+<% r = reports.get_report(sym) %>
+<tr>
+  <td>
+    <% if r.path.file? %>
+    <a href="<%= r.path.to_s.gsub(/^reports\//, '') %>"><%= r.short_desc %></a>
+    <% else %>
+    <%= r.short_desc %>
+    <% end %>
+  </td>
+  <td><%= safe_mtime(r) %></td>
+</tr>
+<% end %>
+</table>
+
diff --git a/report-generators/templates/memcheck.rhtml b/report-generators/templates/memcheck.rhtml
new file mode 100644
index 00000000..75872ed4
--- /dev/null
+++ b/report-generators/templates/memcheck.rhtml
@@ -0,0 +1,30 @@
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+  <tr><th>Tests passed</th><th>Tests failed</th></tr>
+  <tr><td class="pass"><%= total_passed %></td><td <%= total_failed == 0 ? "" : "class=\"fail\""%>><%= total_failed %></td></tr>
+</table>
+
+<% schedules.each do |s| %>
+<h3><%= s.dir.sub('./unit-tests/', '') %></h3>
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+<tr><th>Test</th><th>Result</th><th>Definitely lost</th><th>indirectly lost</th><th>possibly lost</th><th>still reachable</th><tr>
+
+<% s.schedules.each do |t| %>
+<tr>
+  <td>
+  <a href="memcheck_<%= mangle(t.desc) %>.html"><%= t.desc %></a>
+  </td>
+  <% if t.status.success? %>
+  <td class="pass">pass</td>
+  <% else %>
+  <td class="fail">fail</td>
+  <% end %>
+ 
+  <% stats = extract_stats(t) %>
+  <td><%= stats.definitely_lost %></td>
+  <td><%= stats.indirectly_lost %></td>
+  <td><%= stats.possibly_lost %></td>
+  <td><%= stats.reachable %></td>
+</tr>
+<% end %>
+</table>
+<% end %>
diff --git a/report-generators/templates/unit_detail.rhtml b/report-generators/templates/unit_detail.rhtml
new file mode 100644
index 00000000..5324f07c
--- /dev/null
+++ b/report-generators/templates/unit_detail.rhtml
@@ -0,0 +1,37 @@
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+<tr><th>Test</th><th>Result</th></tr>
+<tr>
+  <td>
+  <%= t.desc %>
+  </td>
+  <% if t.status.success? %>
+  <td class="pass">pass</td>
+  <% else %>
+  <td class="fail">fail</td>
+  <% end %>
+</tr>
+</table>
+
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+<tr><th>Command line</th></tr>
+<tr>
+  <td>
+  <pre>
+<%= t.command_line %>
+  </pre>
+  </td>
+</tr>
+</table>
+
+
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+<tr><th>Output</th></tr>
+<tr>
+  <td>
+  <pre>
+<%= t.output %>
+  </pre>
+  </td>
+</tr>
+</table>
+
diff --git a/report-generators/templates/unit_test.rhtml b/report-generators/templates/unit_test.rhtml
new file mode 100644
index 00000000..3137abdd
--- /dev/null
+++ b/report-generators/templates/unit_test.rhtml
@@ -0,0 +1,23 @@
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+  <tr><th>Tests passed</th><th>Tests failed</th></tr>
+  <tr><td class="pass"><%= total_passed %></td><td <%= total_failed == 0 ? "" : "class=\"fail\""%>><%= total_failed %></td></tr>
+</table>
+
+<% schedules.each do |s| %>
+<h3><%= s.dir.sub('./unit-tests/', '') %></h3>
+<table width="95%" cellspacing="2" cellpadding="5" border="0" class="stripes">
+<tr><th>Test</th><th>Result</th></tr>
+<% s.schedules.each do |t| %>
+<tr>
+  <td>
+  <a href="detail_<%= mangle(t.desc) %>.html"><%= t.desc %></a>
+  </td>
+  <% if t.status.success? %>
+  <td class="pass">pass</td>
+  <% else %>
+  <td class="fail">fail</td>
+  <% end %>
+</tr>
+<% end %>
+</table>
+<% end %>
diff --git a/report-generators/test/example.schedule b/report-generators/test/example.schedule
new file mode 100644
index 00000000..f617187a
--- /dev/null
+++ b/report-generators/test/example.schedule
@@ -0,0 +1,4 @@
+# This is a comment
+description number 1:$TEST_TOOL ls
+foo bar:   $TEST_TOOL du -hs .
+   	this comment is prefixed with whitespace: $TEST_TOOL date
\ No newline at end of file
diff --git a/report-generators/test/strings/more_strings/test3.txt b/report-generators/test/strings/more_strings/test3.txt
new file mode 100644
index 00000000..3e9ffe06
--- /dev/null
+++ b/report-generators/test/strings/more_strings/test3.txt
@@ -0,0 +1 @@
+lorem
diff --git a/report-generators/test/strings/test1.txt b/report-generators/test/strings/test1.txt
new file mode 100644
index 00000000..af5626b4
--- /dev/null
+++ b/report-generators/test/strings/test1.txt
@@ -0,0 +1 @@
+Hello, world!
diff --git a/report-generators/test/strings/test2 b/report-generators/test/strings/test2
new file mode 100644
index 00000000..54d55bf0
--- /dev/null
+++ b/report-generators/test/strings/test2
@@ -0,0 +1,3 @@
+one
+two
+three
\ No newline at end of file
diff --git a/report-generators/test/tc_log.rb b/report-generators/test/tc_log.rb
new file mode 100644
index 00000000..a7e9023c
--- /dev/null
+++ b/report-generators/test/tc_log.rb
@@ -0,0 +1,26 @@
+require 'test/unit'
+require 'stringio'
+require 'log'
+
+class TestLog < Test::Unit::TestCase
+  include Log
+
+  private
+  def remove_timestamps(l)
+    l.gsub(/\[[^\]]*\]/, '')
+  end
+
+  public
+  def test_log
+    StringIO.open do |out|
+      init(out)
+
+      info("msg1")
+      warning("msg2")
+      debug("msg3")
+
+      assert_equal("I,   INFO -- : msg1\nW,   WARN -- : msg2\nD,  DEBUG -- : msg3\n",
+                   remove_timestamps(out.string))
+    end
+  end
+end
diff --git a/report-generators/test/tc_schedule_file.rb b/report-generators/test/tc_schedule_file.rb
new file mode 100644
index 00000000..ec4bc3ea
--- /dev/null
+++ b/report-generators/test/tc_schedule_file.rb
@@ -0,0 +1,28 @@
+require 'test/unit'
+require 'pathname'
+require 'schedule_file'
+
+class TestScheduleFile < Test::Unit::TestCase
+  def test_reading
+    p = Pathname.new("report-generators/test/example.schedule")
+    p.open do |f|
+      s = Schedule.read(p.dirname, f)
+
+      assert_equal(3, s.schedules.size)
+      assert_equal(s.schedules[2].desc, "this comment is prefixed with whitespace")
+      assert_equal(s.schedules[0].command_line, "$TEST_TOOL ls")
+    end
+  end
+
+  def test_running
+    p = Pathname.new("report-generators/test/example.schedule")
+    p.open do |f|
+      s = Schedule.read(p.dirname, f)
+      s.run
+
+      s.schedules.each do |t|
+        assert(t.status.success?)
+      end
+    end
+  end
+end
diff --git a/report-generators/test/tc_string_store.rb b/report-generators/test/tc_string_store.rb
new file mode 100644
index 00000000..5ff89dd8
--- /dev/null
+++ b/report-generators/test/tc_string_store.rb
@@ -0,0 +1,19 @@
+require 'string-store'
+require 'test/unit'
+
+class TestStringStore < Test::Unit::TestCase
+  def setup
+    @ss = StringStore.new(['report-generators/test/strings',
+                           'report-generators/test/strings/more_strings'])
+  end
+
+  def test_lookup
+    assert_equal("Hello, world!\n", @ss.lookup(:test1))
+    assert_equal("one\ntwo\nthree", @ss.lookup(:test2))
+    assert_equal("lorem\n", @ss.lookup(:test3))
+
+    assert_raises(RuntimeError) do
+      @ss.lookup(:unlikely_name)
+    end
+  end
+end
diff --git a/report-generators/test/ts.rb b/report-generators/test/ts.rb
new file mode 100644
index 00000000..0a8cc910
--- /dev/null
+++ b/report-generators/test/ts.rb
@@ -0,0 +1,3 @@
+require 'tc_log'
+require 'tc_string_store'
+require 'tc_schedule_file'
diff --git a/report-generators/title_page.rb b/report-generators/title_page.rb
new file mode 100644
index 00000000..f6873eb1
--- /dev/null
+++ b/report-generators/title_page.rb
@@ -0,0 +1,32 @@
+# This generates the index for the reports, including generation
+# times.
+
+require 'log'
+require 'string-store'
+require 'reports'
+require 'erb'
+require 'report_templates'
+
+include Reports
+
+reports = ReportRegister.new
+
+def safe_mtime(r)
+  r.path.file? ? r.path.mtime.to_s : "not generated"
+end
+
+template_store = TemplateStringStore.new
+
+# FIXME: use generate_report() method
+erb = ERB.new(template_store.lookup("index.rhtml"))
+body = erb.result(binding)
+title = "Generation times"
+
+erb = ERB.new(template_store.lookup("boiler_plate.rhtml"))
+txt = erb.result(binding)
+
+Pathname.new("reports/index.html").open("w") do |f|
+  f.puts txt
+end
+
+
diff --git a/report-generators/unit_test.rb b/report-generators/unit_test.rb
new file mode 100644
index 00000000..ebc37855
--- /dev/null
+++ b/report-generators/unit_test.rb
@@ -0,0 +1,46 @@
+# Reads the schedule files given on the command line.  Runs them and
+# generates the reports.
+
+require 'schedule_file'
+require 'pathname'
+require 'reports'
+require 'erb'
+require 'report_templates'
+
+include ReportTemplates
+
+schedules = ARGV.map do |f|
+  p = Pathname.new(f)
+  Schedule.read(p.dirname, p)
+end
+
+total_passed = 0
+total_failed = 0
+
+# We need to make sure the lvm shared libs are in the LD_LIBRARY_PATH
+ENV['LD_LIBRARY_PATH'] = `pwd`.strip + "/libdm:" + (ENV['LD_LIBRARY_PATH'] || '')
+
+schedules.each do |s|
+  s.run
+
+  s.schedules.each do |t|
+    if t.status.success?
+      total_passed += 1
+    else
+      total_failed += 1
+    end
+  end
+end
+
+def mangle(txt)
+  txt.gsub(/\s+/, '_')
+end
+
+generate_report(:unit_test, binding)
+
+# now we generate a detail report for each schedule
+schedules.each do |s|
+  s.schedules.each do |t|
+    generate_report(:unit_detail, binding, Pathname.new("reports/detail_#{mangle(t.desc)}.html"))
+  end
+end