Merge branch 'master' of github.com:domcleal/puppet into master-old

9 files changed, 648 insertions, 235 deletions

diff --git a/ext/cert_inspector b/ext/cert_inspector
new file mode 100755
index 000000000..1effcaa04
--- /dev/null
+++ b/ext/cert_inspector
@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+require 'openssl'
+
+class X509Collector
+  include Enumerable
+
+  def initialize
+    @collected_data = {}
+  end
+
+  def interpret_contents(contents)
+    cls = case contents.split("\n")[0]
+          when /BEGIN X509 CRL/ then OpenSSL::X509::CRL
+          when /BEGIN CERTIFICATE REQUEST/ then OpenSSL::X509::Request
+          when /BEGIN CERTIFICATE/ then OpenSSL::X509::Certificate
+          when /BEGIN RSA (PRIVATE|PUBLIC) KEY/ then OpenSSL::PKey::RSA
+          else return nil
+          end
+    cls.new(contents)
+  rescue
+    nil
+  end
+
+  def expected_non_x509_files
+    ['inventory.txt', 'ca.pass', 'serial']
+  end
+
+  def investigate_path(path)
+    if File.directory?(path)
+      Dir.foreach(path) do |x|
+        next if ['.', '..'].include? x
+        investigate_path File.join(path, x)
+      end
+    else
+      contents = File.read path
+      meaning = interpret_contents contents
+      unless meaning || expected_non_x509_files.include?(File.basename(path))
+        puts "WARNING: file #{path.inspect} could not be interpreted"
+      end
+      @collected_data[path] = meaning if meaning
+    end
+  end
+
+  def each(&block)
+    @collected_data.each(&block)
+  end
+
+  def extract_public_key_info(path, meaning)
+    case meaning
+    when OpenSSL::PKey::RSA
+      if meaning.private?
+        [meaning.public_key, 2, path]
+      else
+        [meaning, 3, path]
+      end
+    when OpenSSL::X509::Certificate
+      [meaning.public_key, 0, meaning.subject.to_s]
+    when OpenSSL::X509::Request
+      [meaning.public_key, 1, meaning.subject.to_s]
+    end
+  end
+
+  def who_signed(meaning, key_names, keys)
+    signing_key = keys.find { |key| meaning.verify(key) }
+    if signing_key then "#{key_names[signing_key.to_s]}" else "???" end
+  end
+
+  def explain(meaning, key_names, keys)
+    case meaning
+    when OpenSSL::PKey::RSA
+      if meaning.private?
+        "Private key for #{key_names[meaning.public_key.to_s]}"
+      else
+        "Public key for #{key_names[meaning.public_key.to_s]}"
+      end
+    when OpenSSL::X509::Certificate
+      signature_desc = who_signed(meaning, key_names, keys)
+      "Certificate assigning name #{meaning.subject.to_s} to #{key_names[meaning.public_key.to_s]}\n    serial number #{meaning.serial}\n    issued by #{meaning.issuer.to_s}\n    signed by #{signature_desc}"
+    when OpenSSL::X509::Request
+      signature_desc = who_signed(meaning, key_names, keys)
+      "Certificate request for #{meaning.subject.to_s} having key #{key_names[meaning.public_key.to_s]}\n    signed by #{signature_desc}"
+    when OpenSSL::X509::CRL
+      signature_desc = who_signed(meaning, key_names, keys)
+      revoked_serial_numbers = meaning.revoked.map { |r| r.serial }
+      revoked_desc = if revoked_serial_numbers.count > 0 then "serial numbers #{revoked_serial_numbers.inspect}" else "nothing" end
+      "Certificate revocation list revoking #{revoked_desc}\n    issued by #{meaning.issuer.to_s}\n    signed by #{signature_desc}"
+    else
+      "Unknown"
+    end
+  end
+
+  # Yield unique public keys, with a canonical name for each.
+  def collect_public_keys
+    key_data = {} # pem => (priority, name, public_key)
+    @collected_data.collect do |path, meaning|
+      begin
+        next unless public_key_info = extract_public_key_info(path, meaning)
+        public_key, priority, name = public_key_info
+        pem = public_key.to_s
+        existing_priority, existing_name, existing_public_key = key_data[pem]
+        next if existing_priority and existing_priority < priority
+        key_data[pem] = priority, name, public_key
+      rescue
+        puts "exception!"
+      end
+    end
+    name_to_key_hash = {}
+    key_data.each do |pem, data|
+      priority, name, public_key = data
+      if name_to_key_hash[name]
+        suffix_num = 2
+        while name_to_key_hash[name + " (#{suffix_num})"]
+          suffix_num += 1
+        end
+        name = name + " (#{suffix_num})"
+      end
+      name_to_key_hash[name] = public_key
+    end
+    key_names = {}
+    keys = []
+    name_to_key_hash.each do |name, public_key|
+      key_names[public_key.to_s] = "key<#{name}>"
+      keys << public_key
+    end
+    [key_names, keys]
+  end
+end
+
+collector = X509Collector.new
+ARGV.each do |path|
+  collector.investigate_path(path)
+end
+key_names, keys = collector.collect_public_keys
+collector.map do |path, meaning|
+  [collector.explain(meaning, key_names, keys), path]
+end.sort.each do |description, path|
+  puts "#{path}:"
+  puts "  #{description}"
+  puts
+end
diff --git a/ext/envpuppet b/ext/envpuppet
new file mode 100755
index 000000000..d921a19b8
--- /dev/null
+++ b/ext/envpuppet
@@ -0,0 +1,80 @@
+#! /bin/bash
+#
+# Jeff McCune <jeff@puppetlabs.com>
+# 2010-10-20
+#
+# Copyright (c) 2010, Puppet Labs
+# License: BSD 3-clause license
+#
+# This script provides a simple way to execute puppet and related tools
+# directly from a git clone of the upstream repositories.  This allows you to
+# quickly switch branches and test different versions of code without much
+# friction.
+#
+# NOTE: There may be issues if puppet, facter, etc... are already installed
+# into RUBY's site_ruby directory.  If you run into strange problems, make sure
+# the correct ruby libraries are being loaded...
+#
+# Sample Usage:
+# =============
+# cd ~/src
+# git clone git://github.com/puppetlabs/puppet.git
+# git clone git://github.com/puppetlabs/facter.git
+# pushd puppet
+# git checkout tags/2.6.1
+# popd
+# pushd facter
+# git checkout tags/1.5.8
+# export ENVPUPPET_BASEDIR=/home/jeff/src
+# envpuppet puppet --version
+# 2.6.1
+# envpuppet facter --version
+# 1.5.8
+
+set -e
+set -u
+
+if test -d puppet -o -d facter; then
+    echo " WARNING!"
+    echo "  Strange things happen if puppet or facter are in the"
+    echo "  current working directory"
+    echo "  (import errors from ruby are a prime example)"
+    echo " WARNING!"
+    echo ""
+    echo "I suggest changing to ~ or /tmp or something..."
+    echo ""
+    echo "Sleeping 2 seconds."
+    echo ""
+    sleep 2
+fi
+
+# Set this to where you check out puppet and facter
+: ${ENVPUPPET_BASEDIR:="${HOME}/src"}
+
+# git://github.com/reductivelabs/puppet.git
+mypath="${ENVPUPPET_BASEDIR}/puppet/sbin:${ENVPUPPET_BASEDIR}/puppet/bin"
+myrubylib="${ENVPUPPET_BASEDIR}/puppet/lib"
+
+# git://github.com/reductivelabs/facter.git
+mypath="${mypath}:${ENVPUPPET_BASEDIR}/facter/bin"
+myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/facter/lib"
+
+# http://github.com/jamtur01/puppet-scaffold.git
+mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-scaffold/bin"
+myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-scaffold/lib"
+
+# http://github.com/puppetlabs/puppet-module-tool.git
+# Also known as "pmt"  Will become "puppet module"
+mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-module-tool/bin"
+myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-module-tool/lib"
+
+# Use the existing environment, if present.
+# Default to no value to prevent unbound variable issues
+mypath="${mypath}:${PATH:-}"
+myrubylib="${myrubylib}:${RUBYLIB:-}"
+
+# Trim any trailing colons from the path list.
+export PATH="${mypath%%:}"
+export RUBYLIB="${myrubylib%%:}"
+
+exec "$@"
diff --git a/ext/extlookup.rb b/ext/extlookup.rb
deleted file mode 100644
index d87583ba7..000000000
--- a/ext/extlookup.rb
+++ /dev/null
@@ -1,181 +0,0 @@
-# Puppet External Data Sources
-#
-# This is a parser function to read data from external files, this version
-# uses CSV files but the concept can easily be adjust for databases, yaml
-# or any other queryable data source.
-#
-# The object of this is to make it obvious when it's being used, rather than
-# magically loading data in when an module is loaded I prefer to look at the code
-# and see statements like:
-#
-#     $snmp_contact = extlookup("snmp_contact")
-#
-# The above snippet will load the snmp_contact value from CSV files, this in its
-# own is useful but a common construct in puppet manifests is something like this:
-#
-# case $domain {
-#      "myclient.com": { $snmp_contact = "John Doe <john@myclient.com>" }
-#      default:        { $snmp_contact = "My Support <support@my.com>" }
-# }
-#
-# Over time there will be a lot of this kind of thing spread all over your manifests
-# and adding an additional client involves grepping through manifests to find all the
-# places where you have constructs like this.
-#
-# This is a data problem and shouldn't be handled in code, a using this function you
-# can do just that.
-#
-# First you configure it in site.pp:
-# $extlookup_datadir = "/etc/puppet/manifests/extdata"
-# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"]
-#
-# The array tells the code how to resolve values, first it will try to find it in
-# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv
-#
-# Now create the following data files in /etc/puppet/manifests/extdata
-#
-# domain_myclient.com.csv:
-#    snmp_contact,John Doe <john@myclient.com>
-#    root_contact,support@%{domain}
-#    client_trusted_ips,192.168.1.130,192.168.10.0/24
-#
-# common.csv:
-#    snmp_contact,My Support <support@my.com>
-#    root_contact,support@my.com
-#
-# Now you can replace the case statement with the simple single line to achieve
-# the exact same outcome:
-#
-#    $snmp_contact = extlookup("snmp_contact")
-#
-# The obove code shows some other features, you can use any fact or variable that
-# is in scope by simply using %{varname} in your data files, you can return arrays
-# by just having multiple values in the csv after the initial variable name.
-#
-# In the event that a variable is nowhere to be found a critical error will be raised
-# that will prevent your manifest from compiling, this is to avoid accidentally putting
-# in empty values etc.  You can however specify a default value:
-#
-#    $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org")
-#
-# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in
-# any data file.
-#
-# You can also specify an additional data file to search first before any others at use
-# time, for example:
-#
-# $version = extlookup("rsyslog_version", "present", "packages")
-#
-# package{"rsyslog": ensure => $version }
-#
-# This will look for a version configured in packages.csv and then in the rest as configured
-# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind
-# of use case makes puppet a lot nicer for managing large amounts of packages since you do not
-# need to edit a load of manifests to do simple things like adjust a desired version number.
-#
-# For more information on installing and writing your own custom functions see:
-#    http://docs.puppetlabs.com/guides/custom_functions.html
-#
-# For further help contact Volcane on #puppet
-require 'csv'
-
-module Puppet::Parser::Functions
-  newfunction(:extlookup, :type => :rvalue) do |args|
-    key = args[0]
-    default = "_ExtUNSET_"
-    datafile = "_ExtUNSET_"
-
-    default = args[1] if args[1]
-    datafile = args[2] if args[2]
-
-    extlookup_datadir = lookupvar('extlookup_datadir')
-    extlookup_precedence = Array.new
-
-    # precedence values can have variables embedded in them
-    # in the form %{fqdn}, you could for example do
-    #
-    #    $extlookup_precedence = ["hosts/%{fqdn}", "common"]
-    #
-    # this will result in /path/to/extdata/hosts/your.box.com.csv
-    # being searched.
-    #
-    # we parse the precedence here because the best place to specify
-    # it would be in site.pp but site.pp is only evaluated at startup
-    # so $fqdn etc would have no meaning there, this way it gets evaluated
-    # each run and has access to the right variables for that run
-    lookupvar('extlookup_precedence').each do |prec|
-      while prec =~ /%\{(.+?)\}/
-        prec.gsub!(/%\{#{$1}\}/, lookupvar($1))
-      end
-
-      extlookup_precedence << prec
-    end
-
-
-    datafiles = Array.new
-
-    # if we got a custom data file, put it first in the array of search files
-    if datafile != ""
-      datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv")
-    end
-
-    extlookup_precedence.each do |d|
-      datafiles << extlookup_datadir + "/#{d}.csv"
-    end
-
-    desired = "_ExtUNSET_"
-
-    datafiles.each do |file|
-      parser.watch_file(file) if File.exists?(file)
-
-      if desired == "_ExtUNSET_"
-        if File.exists?(file)
-          result = CSV.read(file).find_all do |r|
-            r[0] == key
-          end
-
-
-          # return just the single result if theres just one,
-          # else take all the fields in the csv and build an array
-          if result.length > 0
-            if result[0].length == 2
-              val = result[0][1].to_s
-
-              # parse %{}'s in the CSV into local variables using lookupvar()
-              while val =~ /%\{(.+?)\}/
-                val.gsub!(/%\{#{$1}\}/, lookupvar($1))
-              end
-
-              desired = val
-            elsif result[0].length > 1
-              length = result[0].length
-              cells = result[0][1,length]
-
-              # Individual cells in a CSV result are a weird data type and throws
-              # puppets yaml parsing, so just map it all to plain old strings
-              desired = cells.map do |c|
-                # parse %{}'s in the CSV into local variables using lookupvar()
-                while c =~ /%\{(.+?)\}/
-                  c.gsub!(/%\{#{$1}\}/, lookupvar($1))
-                end
-
-                c.to_s
-              end
-            end
-          end
-        end
-      end
-    end
-
-    # don't accidently return nil's and such rather throw a parse error
-    if desired == "_ExtUNSET_" && default == "_ExtUNSET_"
-      raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()"
-    else
-      desired = default if desired == "_ExtUNSET_"
-    end
-
-    desired
-  end
-end
-
-# vi:tabstop=4:expandtab:ai
diff --git a/ext/logcheck/puppet b/ext/logcheck/puppet
index f3280c7b7..7de06aa0f 100644
--- a/ext/logcheck/puppet
+++ b/ext/logcheck/puppet
@@ -1,23 +1,23 @@
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetmasterd\[[0-9]+\]: (Handled resources in|Resource comparison took|Searched for (host|resources|resource params and tags) in) [0-9.]+ seconds
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetmasterd\[[0-9]+\]: Starting Puppet server version [.0-9]+$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetmasterd\[[0-9]+\]: Compiled catalog for [._[:alnum:]-]+ in [.0-9]+ seconds$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetmasterd\[[0-9]+\]: Caught TERM; shutting down$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetmasterd\[[0-9]+\]: Shutting down$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Starting Puppet client version [.0-9]+$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: getting config$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Caching configuration at [\/._[:alnum:]-]+$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Loaded state in [.0-9]+ seconds$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Calling puppetmaster.getconfig$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Retrieved configuration in [.0-9]+ seconds$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Starting configuration run$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Finished configuration run in [.0-9]+ seconds$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Caught (TERM|INT); shutting down$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Shutting down$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Restarting with .*$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Starting catalog run$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Finished catalog run in [.0-9]+ seconds$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Loading fact .*$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Ignoring cache$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Ignoring --listen on onetime run$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetd\[[0-9]+\]: Retrieving plugins$
-^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppetmasterd\[[0-9]+\]: Reopening log files$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-master\[[0-9]+\]: (Handled resources in|Resource comparison took|Searched for (host|resources|resource params and tags) in) [0-9.]+ seconds
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-master\[[0-9]+\]: Starting Puppet server version [.0-9]+$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-master\[[0-9]+\]: Compiled catalog for [._[:alnum:]-]+ in [.0-9]+ seconds$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-master\[[0-9]+\]: Caught TERM; shutting down$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-master\[[0-9]+\]: Shutting down$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Starting Puppet client version [.0-9]+$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: getting config$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Caching configuration at [\/._[:alnum:]-]+$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Loaded state in [.0-9]+ seconds$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Calling puppetmaster.getconfig$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Retrieved configuration in [.0-9]+ seconds$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Starting configuration run$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Finished configuration run in [.0-9]+ seconds$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Caught (TERM|INT); shutting down$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Shutting down$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Restarting with .*$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Starting catalog run$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Finished catalog run in [.0-9]+ seconds$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Loading fact .*$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Ignoring cache$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Ignoring --listen on onetime run$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-agent\[[0-9]+\]: Retrieving plugins$
+^\w{3} [ :0-9]{11} [._[:alnum:]-]+ puppet-master\[[0-9]+\]: Reopening log files$
diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb
new file mode 100644
index 000000000..35bee6ef8
--- /dev/null
+++ b/ext/puppet-load.rb
@@ -0,0 +1,393 @@
+#!/usr/bin/env ruby
+# == Synopsis
+#
+# This tool can exercize a puppetmaster by simulating an arbitraty number of concurrent clients
+# in a lightweight way.
+# 
+# = Prerequisites
+# 
+# This tool requires Event Machine and em-http-request, and an installation of Puppet.
+# Event Machine can be installed from gem.
+# em-http-request can be installed from gem.
+# 
+# = Usage
+#
+#   puppet-load [-d|--debug] [--concurrency <num>] [--repeat <num>] [-V|--version] [-v|--verbose]
+#               [--node <host.domain.com>] [--facts <factfile>] [--cert <certfile>] [--key <keyfile>]
+#               [--factsdir <factsdir>] [--server <server.domain.com>]
+#
+# = Description
+#
+# This is a simple script meant for doing performance tests of puppet masters. It does this
+# by simulating concurrent connections to a puppet master and asking for catalog compilation.
+#
+# = Options
+#
+# Unlike other puppet executables, puppet-load doesn't parse puppet.conf nor use puppet options
+#
+# debug::
+#   Enable full debugging.
+#
+# concurreny::
+#   Number of simulated concurrent clients.
+#
+# server::
+#   Set the puppet master hostname or IP address..
+#
+# node::
+#   Set the fully-qualified domain name of the client. This option can be given multiple
+#   times. In this case puppet-load will ask for catalog compilation of all the given nodes
+#   on a round robin way.
+#
+# help::
+#   Print this help message
+#
+# facts::
+#   This can be used to provide facts for the compilation, directly from a YAML
+#   file as found in the clientyaml directory. If none are provided, puppet-load
+#   will look by itself using Puppet facts indirector.
+#
+# factsdir::
+#   Specify a directory where the yaml facts files can be found. If provided puppet-load
+#   will look up facts in this directory. If not found it will resort to using Puppet Facts
+#   indirector.
+#
+# cert::
+#   This option is mandatory. It should be set to the cert PEM file that will be used
+#   to quthenticate the client connections.
+#
+# key::
+#   This option is mandatory. It should be set to the private key PEM file that will be used
+#   to quthenticate the client connections.
+#
+# timeout::
+#   The number of seconds after which a simulated client is declared in error if it didn't get
+#   a catalog. The default is 180s.
+#
+# repeat::
+#  How many times to perform the test. This means puppet-load will ask for
+#  concurrency * repeat catalogs. 
+#
+# verbose::
+#   Turn on verbose reporting.
+#
+# version::
+#   Print the puppet version number and exit.
+#
+# = Example usage
+#
+# SINGLE NODE:
+#   1) On the master host, generate a new certificate and private key for our test host:
+#   puppet ca --generate puppet-load.domain.com
+#
+#   2) Copy the cert and key to the puppet-load host (which can be the same as the master one)
+#
+#   3) On the master host edit or create the auth.conf so that the catalog ACL match:
+#      path ~ ^/catalog/([^/]+)$
+#      method find
+#      allow $1
+#      allow puppet-load.domain.com
+#
+#   4) launch the master(s)
+#
+#   5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host
+#   you want to simulate.
+#
+#   5) launch puppet-load
+#   puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 
+#
+# MULTIPLE NODES:
+#   1) On the master host, generate a new certificate and private key for our test host:
+#   puppet ca --generate puppet-load.domain.com
+#
+#   2) Copy the cert and key to the puppet-load host (which can be the same as the master one)
+#
+#   3) On the master host edit or create the auth.conf so that the catalog ACL match:
+#      path ~ ^/catalog/([^/]+)$
+#      method find
+#      allow $1
+#      allow puppet-load.domain.com
+#
+#   4) launch the master(s)
+#
+#   5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host
+#   you want to simulate.
+#
+#   5) launch puppet-load
+#   puppet-load -debug --node server1.domain.com --node server2.domain.com --node server3.domain.com \
+#                      --server master.domain.com --factsdir /var/lib/puppet/yaml/facts --concurrency 2 --repeat 20 
+#
+#   puppet-load will load facts file in the --factsdir directory based on the node name.
+#
+# = TODO
+#   * More output stats for error connections (ie report errors, HTTP code...)
+#
+#
+
+# Do an initial trap, so that cancels don't get a stack trace.
+trap(:INT) do
+  $stderr.puts "Cancelling startup"
+  exit(1)
+end
+
+require 'rubygems'
+require 'eventmachine'
+require 'em-http'
+require 'getoptlong'
+require 'puppet'
+
+$cmdargs = [
+  [ "--concurrency",  "-c", GetoptLong::REQUIRED_ARGUMENT       ],
+  [ "--node",     "-n", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--facts",          GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--factsdir",       GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--repeat",   "-r", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--cert",     "-C", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--key",      "-k", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--timeout",  "-t", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--server",   "-s", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--debug",    "-d", GetoptLong::NO_ARGUMENT       ],
+  [ "--help",     "-h", GetoptLong::NO_ARGUMENT       ],
+  [ "--verbose",  "-v", GetoptLong::NO_ARGUMENT       ],
+  [ "--version",  "-V", GetoptLong::NO_ARGUMENT       ],
+]
+
+Puppet::Util::Log.newdestination(:console)
+
+times = {}
+
+def read_facts(file)
+  Puppet.debug("reading facts from: #{file}")
+  fact = YAML.load(File.read(file))
+end
+
+
+result = GetoptLong.new(*$cmdargs)
+
+$args = {}
+$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140, :node => [], :factsdir => nil}
+
+begin
+  result.each { |opt,arg|
+    case opt
+    when "--concurrency"
+      begin
+        $options[:concurrency] = Integer(arg)
+      rescue => detail
+        $stderr.puts "The argument to 'fork' must be an integer"
+        exit(14)
+      end
+    when "--node"
+      $options[:node] << arg
+    when "--factsdir"
+      $options[:factsdir] = arg
+    when "--server"
+      $options[:server] = arg
+    when "--masterport"
+      $options[:masterport] = arg
+    when "--facts"
+      $options[:facts] = arg
+    when "--repeat"
+      $options[:repeat] = Integer(arg)
+    when "--help"
+      if Puppet.features.usage?
+        RDoc::usage && exit
+      else
+        puts "No help available unless you have RDoc::usage installed"
+        exit
+      end
+    when "--version"
+      puts "%s" % Puppet.version
+      exit
+    when "--verbose"
+      Puppet::Util::Log.level = :info
+      Puppet::Util::Log.newdestination(:console)
+    when "--debug"
+      Puppet::Util::Log.level = :debug
+      Puppet::Util::Log.newdestination(:console)
+    when "--cert"
+      $options[:cert] = arg
+    when "--key"
+      $options[:key] = arg
+    end
+  }
+rescue GetoptLong::InvalidOption => detail
+  $stderr.puts detail
+  $stderr.puts "Try '#{$0} --help'"
+  exit(1)
+end
+
+unless $options[:cert] and $options[:key]
+  raise "--cert and --key are mandatory to authenticate the client"
+end
+
+parameters = []
+
+unless $options[:node].size > 0
+  raise "--node is a mandatory argument. It tells to the master what node to compile"
+end
+
+$options[:node].each do |node|
+  factfile = $options[:factsdir] ? File.join($options[:factsdir], node + ".yaml") : $options[:facts]
+  unless fact = read_facts(factfile) or fact = Puppet::Node::Facts.find(node)
+    raise "Could not find facts for %s" % node
+  end
+  fact.values["fqdn"] = node
+  fact.values["hostname"] = node.sub(/\..+/, '')
+  fact.values["domain"] = node.sub(/^[^.]+\./, '')
+
+  parameters << {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(fact.render(:b64_zlib_yaml))}
+end
+
+
+class RequestPool
+  include EventMachine::Deferrable
+
+  attr_reader :requests, :responses, :times, :sizes
+  attr_reader :repeat, :concurrency, :max_request
+
+  def initialize(concurrency, repeat, parameters)
+    @parameters = parameters
+    @current_request = 0
+    @max_request = repeat * concurrency
+    @repeat = repeat
+    @concurrency = concurrency
+    @requests = []
+    @responses = {:succeeded => [], :failed => []}
+    @times = {}
+    @sizes = {}
+
+    # initial spawn
+    (1..concurrency).each do |i|
+      spawn
+    end
+
+  end
+
+  def spawn_request(index)
+    @times[index] = Time.now
+    @sizes[index] = 0
+    nodeidx = index % $options[:node].size
+    node = $options[:node][nodeidx]
+    EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{node}").get(
+    :port => $options[:masterport],
+    :query => @parameters[nodeidx],
+    :timeout => $options[:timeout],
+    :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" },
+    :ssl => { :private_key_file => $options[:key],
+              :cert_chain_file => $options[:cert],
+              :verify_peer => false } ) do
+        @times[index] = Time.now
+        @sizes[index] = 0
+        Puppet.debug("starting client #{index} for #{node}")
+    end
+  end
+
+  def add(index, conn)
+    @requests.push(conn)
+
+    conn.stream { |data|
+      @sizes[index] += data.length
+    }
+
+    conn.callback {
+      @times[index] = Time.now - @times[index]
+      code = conn.response_header.status
+      if code >= 200 && code < 300
+        Puppet.debug("Client #{index} finished successfully")
+        @responses[:succeeded].push(conn)
+      else
+        Puppet.debug("Client #{index} finished with HTTP code #{code}")
+        @responses[:failed].push(conn)
+      end
+      check_progress
+    }
+
+    conn.errback {
+      Puppet.debug("Client #{index} finished with an error: #{conn.error}")
+      @times[index] = Time.now - @times[index]
+      @responses[:failed].push(conn)
+      check_progress
+    }
+  end
+
+  def all_responses
+    @responses[:succeeded] + @responses[:failed]
+  end
+
+  protected
+
+  def check_progress
+    spawn unless all_spawned?
+    succeed if all_finished?
+  end
+
+  def all_spawned?
+    @requests.size >= max_request
+  end
+
+  def all_finished?
+    @responses[:failed].size + @responses[:succeeded].size >= max_request
+  end
+
+  def spawn
+    add(@current_request, spawn_request(@current_request))
+    @current_request += 1
+  end
+end
+
+
+def mean(array)
+  array.inject(0) { |sum, x| sum += x } / array.size.to_f
+end
+
+def median(array)
+  array = array.sort
+  m_pos = array.size / 2
+  return array.size % 2 == 1 ? array[m_pos] : mean(array[m_pos-1..m_pos])
+end
+
+def format_bytes(bytes)
+  if bytes < 1024
+    "%.2f B" % bytes
+  elsif bytes < 1024 * 1024
+    "%.2f KiB" % (bytes/1024.0)
+  else
+    "%.2f MiB" % (bytes/(1024.0*1024.0))
+  end
+end
+
+EM::run {
+
+  start = Time.now
+  multi = RequestPool.new($options[:concurrency], $options[:repeat], parameters)
+
+  multi.callback do
+    duration = Time.now - start
+    puts "#{multi.max_request} requests finished in #{duration} s"
+    puts "#{multi.responses[:failed].size} requests failed"
+    puts "Availability: %3.2f %%" % (100.0*multi.responses[:succeeded].size/(multi.responses[:succeeded].size+multi.responses[:failed].size))
+
+    minmax = multi.times.values.minmax
+    all_time = multi.times.values.reduce(:+)
+
+    puts "\nTime (s):"
+    puts "\tmin: #{minmax[0]} s"
+    puts "\tmax: #{minmax[1]} s"
+    puts "\taverage: #{mean(multi.times.values)} s"
+    puts "\tmedian: #{median(multi.times.values)} s"
+
+    puts "\nConcurrency: %.2f" % (all_time/duration)
+    puts "Transaction Rate (tps): %.2f t/s" % (multi.max_request / duration)
+
+    transferred = multi.sizes.values.reduce(:+)
+
+    puts "\nReceived bytes: #{format_bytes(transferred)}"
+    puts "Throughput: %.5f MiB/s" % (transferred/duration/(1024.0*1024.0))
+
+    # this is the end
+    EventMachine.stop
+  end
+}
+
+
diff --git a/ext/puppetlast b/ext/puppetlast
deleted file mode 100755
index 7434368a3..000000000
--- a/ext/puppetlast
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-# Puppetlast, a script to output the last check-in time of nodes. Also outputs the cached configuration state, if expired or not.
-#
-# AJ "Fujin" Christensen <aj@junglist.gen.nz>
-#
-require 'puppet'
-require 'time'
-
-Puppet[:config] = "/etc/puppet/puppet.conf"
-Puppet.parse_config
-Puppet[:name] = "puppetmasterd"
-Puppet::Node::Facts.terminus_class = :yaml
-
-Puppet::Node::Facts.search("*").sort { |a,b| a.name <=> b.name }.each do |node|
-  puts "#{node.name} checked in #{((Time.now - Time.parse(node.values[:_timestamp].to_s)) / 60).floor} minutes ago.  Version #{node.values['puppetversion']}#{node.expired? ? '  Cache expired.' : ''}"
-end
diff --git a/ext/puppetstoredconfigclean.rb b/ext/puppetstoredconfigclean.rb
index 34dd72f5d..16f39efa1 100644
--- a/ext/puppetstoredconfigclean.rb
+++ b/ext/puppetstoredconfigclean.rb
@@ -19,11 +19,11 @@ end
 
 
       opts = GetoptLong.new(
-                
+
     [ "--config",     "-c",   GetoptLong::REQUIRED_ARGUMENT ],
     [ "--help",        "-h",   GetoptLong::NO_ARGUMENT ],
     [ "--usage",       "-u",   GetoptLong::NO_ARGUMENT ],
-        
+
     [ "--version",     "-v",   GetoptLong::NO_ARGUMENT ]
 )
 
@@ -54,7 +54,7 @@ printusage(1) unless ARGV.size > 0
 require 'puppet/rails'
 Puppet[:config] = config
 Puppet.parse_config
-pm_conf = Puppet.settings.instance_variable_get(:@values)[:puppetmasterd]
+pm_conf = Puppet.settings.instance_variable_get(:@values)[:master]
 
 adapter = pm_conf[:dbadapter]
 args = {:adapter => adapter, :log_level => pm_conf[:rails_loglevel]}
@@ -70,8 +70,6 @@ case adapter
     args[:port]     = pm_conf[:dbport] unless pm_conf[:dbport].to_s.empty?
     socket          = pm_conf[:dbsocket]
     args[:socket]   = socket unless socket.to_s.empty?
-    connections     = pm_conf[:dbconnections].to_i
-    args[:pool]     = connections if connections > 0
   else
     raise ArgumentError, "Invalid db adapter #{adapter}"
 end
diff --git a/ext/rack/files/config.ru b/ext/rack/files/config.ru
index 5f0834a7d..f9c492dd7 100644
--- a/ext/rack/files/config.ru
+++ b/ext/rack/files/config.ru
@@ -4,15 +4,13 @@
 # if puppet is not in your RUBYLIB:
 # $:.unshift('/opt/puppet/lib')
 
-$0 = "puppetmasterd"
-require 'puppet'
+$0 = "master"
 
 # if you want debugging:
 # ARGV << "--debug"
 
 ARGV << "--rack"
-require 'puppet/application/puppetmasterd'
+require 'puppet/application/master'
 # we're usually running inside a Rack::Builder.new {} block,
 # therefore we need to call run *here*.
-run Puppet::Application[:puppetmasterd].run
-
+run Puppet::Application[:master].run
diff --git a/ext/vim/syntax/puppet.vim b/ext/vim/syntax/puppet.vim
index 80cd91c6c..96052104d 100644
--- a/ext/vim/syntax/puppet.vim
+++ b/ext/vim/syntax/puppet.vim
@@ -19,7 +19,7 @@ endif
 " match class/definition/node declarations
 syn region  puppetDefine        start="^\s*\(class\|define\|node\)\s" end="{" contains=puppetDefType,puppetDefName,puppetDefArguments,puppetNodeRe
 syn keyword puppetDefType       class define node inherits contained
-syn region  puppetDefArguments  start="(" end=")" contained contains=puppetArgument
+syn region  puppetDefArguments  start="(" end=")" contained contains=puppetArgument,puppetString
 syn match   puppetArgument      "\w\+" contained
 syn match   puppetArgument      "\$\w\+" contained
 syn match   puppetArgument      "'[^']+'" contained
@@ -33,6 +33,7 @@ syn match   puppetNodeRe        "/.*/" contained
 "FIXME: "Foo-bar" doesn't get highlighted as expected, although "foo-bar" does.
 syn match   puppetInstance      "[A-Za-z0-9_-]\+\(::[A-Za-z0-9_-]\+\)*\s*{" contains=puppetTypeName,puppetTypeDefault
 syn match   puppetInstance      "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*[[{]" contains=puppetTypeName,puppetTypeDefault
+syn match   puppetInstance      "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*<\?<|" contains=puppetTypeName,puppetTypeDefault
 syn match   puppetTypeName      "[a-z]\w*" contained
 syn match   puppetTypeDefault   "[A-Z]\w*" contained
 
@@ -56,19 +57,19 @@ syn region  puppetFunction      start="^\s*\(alert\|crit\|debug\|emerg\|err\|fai
 " rvalues
 syn region  puppetFunction      start="^\s*\(defined\|file\|fqdn_rand\|generate\|inline_template\|regsubst\|sha1\|shellquote\|split\|sprintf\|tagged\|template\|versioncmp\)\s*(" end=")" contained contains=puppetString
 
-syn match   puppetVariable      "$\w\+"
-syn match   puppetVariable      "${\w\+}"
+syn match   puppetVariable      "$[a-zA-Z0-9_:]\+"
+syn match   puppetVariable      "${[a-zA-Z0-9_:]\+}"
 
 " match anything between simple/double quotes.
 " don't match variables if preceded by a backslash.
 syn region  puppetString        start=+'+ skip=+\\\\\|\\'+ end=+'+
 syn region  puppetString        start=+"+ skip=+\\\\\|\\"+ end=+"+ contains=puppetVariable,puppetNotVariable
-syn match   puppetString        "/.*/"
+syn match   puppetString        "/[^/]*/"
 syn match   puppetNotVariable   "\\$\w\+" contained
 syn match   puppetNotVariable   "\\${\w\+}" contained
 
 syn keyword puppetKeyword       import inherits include
-syn keyword puppetControl       case default if else
+syn keyword puppetControl       case default if else elsif
 syn keyword puppetSpecial       true false undef
 
 " comments last overriding everything else