Merge branch 'next'

This marks the end of the agile iteration from 11/3-11/10.

3 files changed, 282 insertions, 26 deletions

diff --git a/ext/cert_inspector b/ext/cert_inspector
new file mode 100755
index 000000000..1effcaa04
--- /dev/null
+++ b/ext/cert_inspector
@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+require 'openssl'
+
+class X509Collector
+  include Enumerable
+
+  def initialize
+    @collected_data = {}
+  end
+
+  def interpret_contents(contents)
+    cls = case contents.split("\n")[0]
+          when /BEGIN X509 CRL/ then OpenSSL::X509::CRL
+          when /BEGIN CERTIFICATE REQUEST/ then OpenSSL::X509::Request
+          when /BEGIN CERTIFICATE/ then OpenSSL::X509::Certificate
+          when /BEGIN RSA (PRIVATE|PUBLIC) KEY/ then OpenSSL::PKey::RSA
+          else return nil
+          end
+    cls.new(contents)
+  rescue
+    nil
+  end
+
+  def expected_non_x509_files
+    ['inventory.txt', 'ca.pass', 'serial']
+  end
+
+  def investigate_path(path)
+    if File.directory?(path)
+      Dir.foreach(path) do |x|
+        next if ['.', '..'].include? x
+        investigate_path File.join(path, x)
+      end
+    else
+      contents = File.read path
+      meaning = interpret_contents contents
+      unless meaning || expected_non_x509_files.include?(File.basename(path))
+        puts "WARNING: file #{path.inspect} could not be interpreted"
+      end
+      @collected_data[path] = meaning if meaning
+    end
+  end
+
+  def each(&block)
+    @collected_data.each(&block)
+  end
+
+  def extract_public_key_info(path, meaning)
+    case meaning
+    when OpenSSL::PKey::RSA
+      if meaning.private?
+        [meaning.public_key, 2, path]
+      else
+        [meaning, 3, path]
+      end
+    when OpenSSL::X509::Certificate
+      [meaning.public_key, 0, meaning.subject.to_s]
+    when OpenSSL::X509::Request
+      [meaning.public_key, 1, meaning.subject.to_s]
+    end
+  end
+
+  def who_signed(meaning, key_names, keys)
+    signing_key = keys.find { |key| meaning.verify(key) }
+    if signing_key then "#{key_names[signing_key.to_s]}" else "???" end
+  end
+
+  def explain(meaning, key_names, keys)
+    case meaning
+    when OpenSSL::PKey::RSA
+      if meaning.private?
+        "Private key for #{key_names[meaning.public_key.to_s]}"
+      else
+        "Public key for #{key_names[meaning.public_key.to_s]}"
+      end
+    when OpenSSL::X509::Certificate
+      signature_desc = who_signed(meaning, key_names, keys)
+      "Certificate assigning name #{meaning.subject.to_s} to #{key_names[meaning.public_key.to_s]}\n    serial number #{meaning.serial}\n    issued by #{meaning.issuer.to_s}\n    signed by #{signature_desc}"
+    when OpenSSL::X509::Request
+      signature_desc = who_signed(meaning, key_names, keys)
+      "Certificate request for #{meaning.subject.to_s} having key #{key_names[meaning.public_key.to_s]}\n    signed by #{signature_desc}"
+    when OpenSSL::X509::CRL
+      signature_desc = who_signed(meaning, key_names, keys)
+      revoked_serial_numbers = meaning.revoked.map { |r| r.serial }
+      revoked_desc = if revoked_serial_numbers.count > 0 then "serial numbers #{revoked_serial_numbers.inspect}" else "nothing" end
+      "Certificate revocation list revoking #{revoked_desc}\n    issued by #{meaning.issuer.to_s}\n    signed by #{signature_desc}"
+    else
+      "Unknown"
+    end
+  end
+
+  # Yield unique public keys, with a canonical name for each.
+  def collect_public_keys
+    key_data = {} # pem => (priority, name, public_key)
+    @collected_data.collect do |path, meaning|
+      begin
+        next unless public_key_info = extract_public_key_info(path, meaning)
+        public_key, priority, name = public_key_info
+        pem = public_key.to_s
+        existing_priority, existing_name, existing_public_key = key_data[pem]
+        next if existing_priority and existing_priority < priority
+        key_data[pem] = priority, name, public_key
+      rescue
+        puts "exception!"
+      end
+    end
+    name_to_key_hash = {}
+    key_data.each do |pem, data|
+      priority, name, public_key = data
+      if name_to_key_hash[name]
+        suffix_num = 2
+        while name_to_key_hash[name + " (#{suffix_num})"]
+          suffix_num += 1
+        end
+        name = name + " (#{suffix_num})"
+      end
+      name_to_key_hash[name] = public_key
+    end
+    key_names = {}
+    keys = []
+    name_to_key_hash.each do |name, public_key|
+      key_names[public_key.to_s] = "key<#{name}>"
+      keys << public_key
+    end
+    [key_names, keys]
+  end
+end
+
+collector = X509Collector.new
+ARGV.each do |path|
+  collector.investigate_path(path)
+end
+key_names, keys = collector.collect_public_keys
+collector.map do |path, meaning|
+  [collector.explain(meaning, key_names, keys), path]
+end.sort.each do |description, path|
+  puts "#{path}:"
+  puts "  #{description}"
+  puts
+end
diff --git a/ext/envpuppet b/ext/envpuppet
new file mode 100755
index 000000000..d921a19b8
--- /dev/null
+++ b/ext/envpuppet
@@ -0,0 +1,80 @@
+#! /bin/bash
+#
+# Jeff McCune <jeff@puppetlabs.com>
+# 2010-10-20
+#
+# Copyright (c) 2010, Puppet Labs
+# License: BSD 3-clause license
+#
+# This script provides a simple way to execute puppet and related tools
+# directly from a git clone of the upstream repositories.  This allows you to
+# quickly switch branches and test different versions of code without much
+# friction.
+#
+# NOTE: There may be issues if puppet, facter, etc... are already installed
+# into RUBY's site_ruby directory.  If you run into strange problems, make sure
+# the correct ruby libraries are being loaded...
+#
+# Sample Usage:
+# =============
+# cd ~/src
+# git clone git://github.com/puppetlabs/puppet.git
+# git clone git://github.com/puppetlabs/facter.git
+# pushd puppet
+# git checkout tags/2.6.1
+# popd
+# pushd facter
+# git checkout tags/1.5.8
+# export ENVPUPPET_BASEDIR=/home/jeff/src
+# envpuppet puppet --version
+# 2.6.1
+# envpuppet facter --version
+# 1.5.8
+
+set -e
+set -u
+
+if test -d puppet -o -d facter; then
+    echo " WARNING!"
+    echo "  Strange things happen if puppet or facter are in the"
+    echo "  current working directory"
+    echo "  (import errors from ruby are a prime example)"
+    echo " WARNING!"
+    echo ""
+    echo "I suggest changing to ~ or /tmp or something..."
+    echo ""
+    echo "Sleeping 2 seconds."
+    echo ""
+    sleep 2
+fi
+
+# Set this to where you check out puppet and facter
+: ${ENVPUPPET_BASEDIR:="${HOME}/src"}
+
+# git://github.com/reductivelabs/puppet.git
+mypath="${ENVPUPPET_BASEDIR}/puppet/sbin:${ENVPUPPET_BASEDIR}/puppet/bin"
+myrubylib="${ENVPUPPET_BASEDIR}/puppet/lib"
+
+# git://github.com/reductivelabs/facter.git
+mypath="${mypath}:${ENVPUPPET_BASEDIR}/facter/bin"
+myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/facter/lib"
+
+# http://github.com/jamtur01/puppet-scaffold.git
+mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-scaffold/bin"
+myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-scaffold/lib"
+
+# http://github.com/puppetlabs/puppet-module-tool.git
+# Also known as "pmt"  Will become "puppet module"
+mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-module-tool/bin"
+myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-module-tool/lib"
+
+# Use the existing environment, if present.
+# Default to no value to prevent unbound variable issues
+mypath="${mypath}:${PATH:-}"
+myrubylib="${myrubylib}:${RUBYLIB:-}"
+
+# Trim any trailing colons from the path list.
+export PATH="${mypath%%:}"
+export RUBYLIB="${myrubylib%%:}"
+
+exec "$@"
diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb
index 110282d01..35bee6ef8 100644
--- a/ext/puppet-load.rb
+++ b/ext/puppet-load.rb
@@ -14,7 +14,7 @@
 #
 #   puppet-load [-d|--debug] [--concurrency <num>] [--repeat <num>] [-V|--version] [-v|--verbose]
 #               [--node <host.domain.com>] [--facts <factfile>] [--cert <certfile>] [--key <keyfile>]
-#               [--server <server.domain.com>]
+#               [--factsdir <factsdir>] [--server <server.domain.com>]
 #
 # = Description
 #
@@ -35,8 +35,9 @@
 #   Set the puppet master hostname or IP address..
 #
 # node::
-#   Set the fully-qualified domain name of the client.  This is only used for
-#   certificate purposes, but can be used to override the discovered hostname.
+#   Set the fully-qualified domain name of the client. This option can be given multiple
+#   times. In this case puppet-load will ask for catalog compilation of all the given nodes
+#   on a round robin way.
 #
 # help::
 #   Print this help message
@@ -46,6 +47,11 @@
 #   file as found in the clientyaml directory. If none are provided, puppet-load
 #   will look by itself using Puppet facts indirector.
 #
+# factsdir::
+#   Specify a directory where the yaml facts files can be found. If provided puppet-load
+#   will look up facts in this directory. If not found it will resort to using Puppet Facts
+#   indirector.
+#
 # cert::
 #   This option is mandatory. It should be set to the cert PEM file that will be used
 #   to quthenticate the client connections.
@@ -70,8 +76,9 @@
 #
 # = Example usage
 #
+# SINGLE NODE:
 #   1) On the master host, generate a new certificate and private key for our test host:
-#   puppet ca --generate puppet-load.domain.com [*]
+#   puppet ca --generate puppet-load.domain.com
 #
 #   2) Copy the cert and key to the puppet-load host (which can be the same as the master one)
 #
@@ -81,7 +88,7 @@
 #      allow $1
 #      allow puppet-load.domain.com
 #
-#   4) launch the master
+#   4) launch the master(s)
 #
 #   5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host
 #   you want to simulate.
@@ -89,12 +96,30 @@
 #   5) launch puppet-load
 #   puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 
 #
-# [*]: unfortunately at this stage Puppet trusts the certname of the connecting node more than
-#      than the node name request paramater. It means that the master will compile
-#      the puppet-load node and not the --node given.
+# MULTIPLE NODES:
+#   1) On the master host, generate a new certificate and private key for our test host:
+#   puppet ca --generate puppet-load.domain.com
+#
+#   2) Copy the cert and key to the puppet-load host (which can be the same as the master one)
+#
+#   3) On the master host edit or create the auth.conf so that the catalog ACL match:
+#      path ~ ^/catalog/([^/]+)$
+#      method find
+#      allow $1
+#      allow puppet-load.domain.com
+#
+#   4) launch the master(s)
+#
+#   5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host
+#   you want to simulate.
+#
+#   5) launch puppet-load
+#   puppet-load -debug --node server1.domain.com --node server2.domain.com --node server3.domain.com \
+#                      --server master.domain.com --factsdir /var/lib/puppet/yaml/facts --concurrency 2 --repeat 20 
+#
+#   puppet-load will load facts file in the --factsdir directory based on the node name.
 #
 # = TODO
-#   * Allow to simulate any different nodes
 #   * More output stats for error connections (ie report errors, HTTP code...)
 #
 #
@@ -115,6 +140,7 @@ $cmdargs = [
   [ "--concurrency",  "-c", GetoptLong::REQUIRED_ARGUMENT       ],
   [ "--node",     "-n", GetoptLong::REQUIRED_ARGUMENT ],
   [ "--facts",          GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--factsdir",       GetoptLong::REQUIRED_ARGUMENT ],
   [ "--repeat",   "-r", GetoptLong::REQUIRED_ARGUMENT ],
   [ "--cert",     "-C", GetoptLong::REQUIRED_ARGUMENT ],
   [ "--key",      "-k", GetoptLong::REQUIRED_ARGUMENT ],
@@ -131,14 +157,15 @@ Puppet::Util::Log.newdestination(:console)
 times = {}
 
 def read_facts(file)
-  YAML.load(File.read(file))
+  Puppet.debug("reading facts from: #{file}")
+  fact = YAML.load(File.read(file))
 end
 
 
 result = GetoptLong.new(*$cmdargs)
 
 $args = {}
-$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140}
+$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140, :node => [], :factsdir => nil}
 
 begin
   result.each { |opt,arg|
@@ -151,7 +178,9 @@ begin
         exit(14)
       end
     when "--node"
-      $options[:node] = arg
+      $options[:node] << arg
+    when "--factsdir"
+      $options[:factsdir] = arg
     when "--server"
       $options[:server] = arg
     when "--masterport"
@@ -192,21 +221,24 @@ unless $options[:cert] and $options[:key]
   raise "--cert and --key are mandatory to authenticate the client"
 end
 
-unless $options[:facts] and facts = read_facts($options[:facts])
-  unless facts = Puppet::Node::Facts.find($options[:node])
-    raise "Could not find facts for %s" % $options[:node]
-  end
-end
+parameters = []
 
-unless $options[:node]
+unless $options[:node].size > 0
   raise "--node is a mandatory argument. It tells to the master what node to compile"
 end
 
-facts.values["fqdn"] = $options[:node]
-facts.values["hostname"] = $options[:node].sub(/\..+/, '')
-facts.values["domain"] = $options[:node].sub(/^[^.]+\./, '')
+$options[:node].each do |node|
+  factfile = $options[:factsdir] ? File.join($options[:factsdir], node + ".yaml") : $options[:facts]
+  unless fact = read_facts(factfile) or fact = Puppet::Node::Facts.find(node)
+    raise "Could not find facts for %s" % node
+  end
+  fact.values["fqdn"] = node
+  fact.values["hostname"] = node.sub(/\..+/, '')
+  fact.values["domain"] = node.sub(/^[^.]+\./, '')
+
+  parameters << {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(fact.render(:b64_zlib_yaml))}
+end
 
-parameters = {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(facts.render(:b64_zlib_yaml))}
 
 class RequestPool
   include EventMachine::Deferrable
@@ -233,17 +265,21 @@ class RequestPool
   end
 
   def spawn_request(index)
-    EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{$options[:node]}").get(
+    @times[index] = Time.now
+    @sizes[index] = 0
+    nodeidx = index % $options[:node].size
+    node = $options[:node][nodeidx]
+    EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{node}").get(
     :port => $options[:masterport],
-    :query => @parameters,
+    :query => @parameters[nodeidx],
     :timeout => $options[:timeout],
     :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" },
     :ssl => { :private_key_file => $options[:key],
               :cert_chain_file => $options[:cert],
               :verify_peer => false } ) do
-        Puppet.debug("starting client #{index}")
         @times[index] = Time.now
         @sizes[index] = 0
+        Puppet.debug("starting client #{index} for #{node}")
     end
   end
 
@@ -268,7 +304,7 @@ class RequestPool
     }
 
     conn.errback {
-      Puppet.debug("Client #{index} finished with an error: #{conn.response.error}")
+      Puppet.debug("Client #{index} finished with an error: #{conn.error}")
       @times[index] = Time.now - @times[index]
       @responses[:failed].push(conn)
       check_progress