Fix #4339 - Save a last run report summary to $statedir/last_run_summary.yaml

Once a configuration run is done, puppetd will save on the node a
yaml summary report roughly akin to:

---
  time:
    notify: 0.001025
    last_run: 1289561427
    schedule: 0.00071
    config_retrieval: 0.039518
    filebucket: 0.000126
  resources:
    changed: 1
    total: 8
    out_of_sync: 1
  events:
    total: 1
    success: 1
  changes:
    total: 1

This is almost an hash version of the current --summarize output, with
the notable exception that the time section includes the last run unix
timestamp.

The whole idea is to be able to monitor locally if a puppetd does its job.
For instance this could be used in a nagios check or to send an SNMP trap.
The last_run information might help detect staleness, and this summary
can also be used for performance monitoring (ie time section).
The resource section can also show the number of failed resources.

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

7 files changed, 145 insertions, 25 deletions

diff --git a/lib/puppet/configurer.rb b/lib/puppet/configurer.rb
index 31d31c2d2..e46e9a6e7 100644
--- a/lib/puppet/configurer.rb
+++ b/lib/puppet/configurer.rb
@@ -168,19 +168,30 @@ class Puppet::Configurer
     execute_postrun_command
 
     Puppet::Util::Log.close(report)
-
     send_report(report, transaction)
   end
 
   def send_report(report, trans = nil)
     trans.generate_report if trans
     puts report.summary if Puppet[:summarize]
-    report.save if Puppet[:report]
+    save_last_run_summary(report)
+    if Puppet[:report]
+      report.save
+    end
   rescue => detail
     puts detail.backtrace if Puppet[:trace]
     Puppet.err "Could not send report: #{detail}"
   end
 
+  def save_last_run_summary(report)
+    Puppet::Util::FileLocking.writelock(Puppet[:lastrunfile], 0660) do |file|
+      file.print YAML.dump(report.raw_summary)
+    end
+  rescue => detail
+    puts detail.backtrace if Puppet[:trace]
+    Puppet.err "Could not save last run local report: #{detail}"
+  end
+
   private
 
   def self.timeout
diff --git a/lib/puppet/defaults.rb b/lib/puppet/defaults.rb
index 7ae553827..9b80c9262 100644
--- a/lib/puppet/defaults.rb
+++ b/lib/puppet/defaults.rb
@@ -602,6 +602,10 @@ module Puppet
     :report => [false,
       "Whether to send reports after every transaction."
     ],
+    :lastrunfile =>  { :default => "$statedir/last_run_summary.yaml",
+      :mode => 0660,
+      :desc => "Where puppet agent stores the last run report summary in yaml format."
+    },
     :graph => [false, "Whether to create dot graph files for the different
       configuration graphs.  These dot files can be interpreted by tools
       like OmniGraffle or dot (which is part of ImageMagick)."],
diff --git a/lib/puppet/transaction/report.rb b/lib/puppet/transaction/report.rb
index e6d1e0528..1d3091428 100644
--- a/lib/puppet/transaction/report.rb
+++ b/lib/puppet/transaction/report.rb
@@ -62,30 +62,49 @@ class Puppet::Transaction::Report
     host
   end
 
-  # Provide a summary of this report.
+  # Provide a human readable textual summary of this report.
   def summary
+    report = raw_summary
+
     ret = ""
+    report.keys.sort { |a,b| a.to_s <=> b.to_s }.each do |key|
+      ret += "#{Puppet::Util::Metric.labelize(key)}:\n"
 
-    @metrics.sort { |a,b| a[1].label <=> b[1].label }.each do |name, metric|
-      ret += "#{metric.label}:\n"
-      metric.values.sort { |a,b|
+      report[key].keys.sort { |a,b|
         # sort by label
-        if a[0] == :total
+        if a == :total
           1
-        elsif b[0] == :total
+        elsif b == :total
           -1
         else
-          a[1] <=> b[1]
+          report[key][a].to_s <=> report[key][b].to_s
         end
-      }.each do |name, label, value|
+      }.each do |label|
+        value = report[key][label]
         next if value == 0
         value = "%0.2f" % value if value.is_a?(Float)
-        ret += "   %15s %s\n" % [label + ":", value]
+        ret += "   %15s %s\n" % [Puppet::Util::Metric.labelize(label) + ":", value]
       end
     end
     ret
   end
 
+  # Provide a raw hash summary of this report.
+  def raw_summary
+    report = {}
+
+    @metrics.each do |name, metric|
+      key = metric.name.to_s
+      report[key] = {}
+      metric.values.each do |name, label, value|
+        report[key][name.to_s] = value
+      end
+      report[key]["total"] = 0 unless key == "time" or report[key].include?("total")
+    end
+    (report["time"] ||= {})["last_run"] = Time.now.tv_sec
+    report
+  end
+
   # Based on the contents of this report's metrics, compute a single number
   # that represents the report. The resulting number is a bitmask where
   # individual bits represent the presence of different metrics.
@@ -103,7 +122,6 @@ class Puppet::Transaction::Report
     resource_statuses.each do |name, status|
       metrics[:total] += status.change_count if status.change_count
     end
-
     add_metric(:changes, metrics)
   end
 
@@ -124,7 +142,6 @@ class Puppet::Transaction::Report
     metrics[:total] = resource_statuses.length
 
     resource_statuses.each do |name, status|
-
       Puppet::Resource::Status::STATES.each do |state|
         metrics[state] += 1 if status.send(state)
       end
diff --git a/lib/puppet/util/metric.rb b/lib/puppet/util/metric.rb
index 90a244836..d61fb3df6 100644
--- a/lib/puppet/util/metric.rb
+++ b/lib/puppet/util/metric.rb
@@ -122,7 +122,7 @@ class Puppet::Util::Metric
   def initialize(name,label = nil)
     @name = name.to_s
 
-    @label = label || labelize(name)
+    @label = label || self.class.labelize(name)
 
     @values = []
   end
@@ -132,7 +132,7 @@ class Puppet::Util::Metric
   end
 
   def newvalue(name,value,label = nil)
-    label ||= labelize(name)
+    label ||= self.class.labelize(name)
     @values.push [name,label,value]
   end
 
@@ -173,10 +173,8 @@ class Puppet::Util::Metric
     @values.sort { |a, b| a[1] <=> b[1] }
   end
 
-  private
-
   # Convert a name into a label.
-  def labelize(name)
+  def self.labelize(name)
     name.to_s.capitalize.gsub("_", " ")
   end
 end
diff --git a/spec/integration/configurer_spec.rb b/spec/integration/configurer_spec.rb
index 9a8b66fe4..cb7d3d779 100755
--- a/spec/integration/configurer_spec.rb
+++ b/spec/integration/configurer_spec.rb
@@ -5,6 +5,8 @@ require File.dirname(__FILE__) + '/../spec_helper'
 require 'puppet/configurer'
 
 describe Puppet::Configurer do
+  include PuppetSpec::Files
+
   describe "when downloading plugins" do
     it "should use the :pluginsignore setting, split on whitespace, for ignoring remote files" do
       resource = Puppet::Type.type(:notify).new :name => "yay"
@@ -17,19 +19,50 @@ describe Puppet::Configurer do
   end
 
   describe "when running" do
-    it "should send a transaction report with valid data" do
-      catalog = Puppet::Resource::Catalog.new
-      catalog.add_resource(Puppet::Type.type(:notify).new(:title => "testing"))
+    before(:each) do
+      @catalog = Puppet::Resource::Catalog.new
+      @catalog.add_resource(Puppet::Type.type(:notify).new(:title => "testing"))
 
-      configurer = Puppet::Configurer.new
+      # Make sure we don't try to persist the local state after the transaction ran, 
+      # because it will fail during test (the state file is in an not existing directory)
+      # and we need the transaction to be successful to be able to produce a summary report
+      @catalog.host_config = false
+
+      @configurer = Puppet::Configurer.new
+    end
+
+    it "should send a transaction report with valid data" do
 
+      @configurer.stubs(:save_last_run_summary)
       Puppet::Transaction::Report.indirection.expects(:save).with do |x, report|
         report.time.class == Time and report.logs.length > 0
       end
 
       Puppet[:report] = true
 
-      configurer.run :catalog => catalog
+      @configurer.run :catalog => @catalog
+    end
+
+    it "should save a correct last run summary" do
+      report = Puppet::Transaction::Report.new
+      report.stubs(:save)
+
+      Puppet[:lastrunfile] = tmpfile("lastrunfile")
+      Puppet[:report] = true
+
+      @configurer.run :catalog => @catalog, :report => report
+
+      summary = nil
+      File.open(Puppet[:lastrunfile], "r") do |fd|
+        summary = YAML.load(fd.read)
+      end
+
+      summary.should be_a(Hash)
+      %w{time changes events resources}.each do |key|
+        summary.should be_key(key)
+      end
+      summary["time"].should be_key("notify")
+      summary["time"]["last_run"].should >= Time.now.tv_sec
     end
   end
 end
diff --git a/spec/unit/configurer_spec.rb b/spec/unit/configurer_spec.rb
index 0c9d06362..5a2c49406 100755
--- a/spec/unit/configurer_spec.rb
+++ b/spec/unit/configurer_spec.rb
@@ -89,6 +89,7 @@ describe Puppet::Configurer, "when executing a catalog run" do
     @catalog = Puppet::Resource::Catalog.new
     @catalog.stubs(:apply)
     @agent.stubs(:retrieve_catalog).returns @catalog
+    @agent.stubs(:save_last_run_summary)
 
     Puppet::Util::Log.stubs(:newdestination)
     Puppet::Util::Log.stubs(:close)
@@ -236,6 +237,7 @@ describe Puppet::Configurer, "when sending a report" do
   before do
     Puppet.settings.stubs(:use).returns(true)
     @configurer = Puppet::Configurer.new
+    @configurer.stubs(:save_last_run_summary)
 
     @report = stub 'report'
     @trans = stub 'transaction'
@@ -284,6 +286,20 @@ describe Puppet::Configurer, "when sending a report" do
     @configurer.send_report(@report)
   end
 
+  it "should save the last run summary if reporting is enabled" do
+    Puppet.settings[:report] = true
+
+    @configurer.expects(:save_last_run_summary).with(@report)
+    @configurer.send_report(@report)
+  end
+
+  it "should not save the last run summary if reporting is disabled" do
+    Puppet.settings[:report] = false
+
+    @configurer.expects(:save_last_run_summary).never
+    @configurer.send_report(@report)
+  end
+
   it "should log but not fail if saving the report fails" do
     Puppet.settings[:report] = true
 
@@ -294,6 +310,36 @@ describe Puppet::Configurer, "when sending a report" do
   end
 end
 
+describe Puppet::Configurer, "when saving the summary report file" do
+  before do
+    Puppet.settings.stubs(:use).returns(true)
+    @configurer = Puppet::Configurer.new
+
+    @report = stub 'report'
+    @trans = stub 'transaction'
+    @lastrunfd = stub 'lastrunfd'
+    Puppet::Util::FileLocking.stubs(:writelock).yields(@lastrunfd)
+  end
+
+  it "should write the raw summary to the lastrunfile setting value" do
+    Puppet::Util::FileLocking.expects(:writelock).with(Puppet[:lastrunfile], 0660)
+    @configurer.save_last_run_summary(@report)
+  end
+
+  it "should write the raw summary as yaml" do
+    @report.expects(:raw_summary).returns("summary")
+    @lastrunfd.expects(:print).with(YAML.dump("summary"))
+    @configurer.save_last_run_summary(@report)
+  end
+
+  it "should log but not fail if saving the last run summary fails" do
+    Puppet::Util::FileLocking.expects(:writelock).raises "exception"
+    Puppet.expects(:err)
+    lambda { @configurer.save_last_run_summary(@report) }.should_not raise_error
+  end
+
+end
+
 describe Puppet::Configurer, "when retrieving a catalog" do
   before do
     Puppet.settings.stubs(:use).returns(true)
diff --git a/spec/unit/transaction/report_spec.rb b/spec/unit/transaction/report_spec.rb
index 7e0b0554b..b310713d2 100755
--- a/spec/unit/transaction/report_spec.rb
+++ b/spec/unit/transaction/report_spec.rb
@@ -225,8 +225,19 @@ describe Puppet::Transaction::Report do
       @report.calculate_metrics
     end
 
-    %w{Changes Total Resources}.each do |main|
-      it "should include information on #{main} in the summary" do
+    %w{changes time resources events}.each do |main|
+      it "should include the key #{main} in the raw summary hash" do
+        @report.raw_summary.should be_key main
+      end
+    end
+
+    it "should include the last run time in the raw summary hash" do
+      Time.stubs(:now).returns(Time.utc(2010,11,10,12,0,24))
+      @report.raw_summary["time"]["last_run"].should == 1289390424
+    end
+
+    %w{Changes Total Resources Time Events}.each do |main|
+      it "should include information on #{main} in the textual summary" do
         @report.summary.should be_include(main)
       end
     end