Merge branch 'next'

This synchronizes the 2.7 master branch with 2.6.1RC1

3 files changed, 360 insertions, 183 deletions

diff --git a/ext/extlookup.rb b/ext/extlookup.rb
deleted file mode 100644
index d87583ba7..000000000
--- a/ext/extlookup.rb
+++ /dev/null
@@ -1,181 +0,0 @@
-# Puppet External Data Sources
-#
-# This is a parser function to read data from external files, this version
-# uses CSV files but the concept can easily be adjust for databases, yaml
-# or any other queryable data source.
-#
-# The object of this is to make it obvious when it's being used, rather than
-# magically loading data in when an module is loaded I prefer to look at the code
-# and see statements like:
-#
-#     $snmp_contact = extlookup("snmp_contact")
-#
-# The above snippet will load the snmp_contact value from CSV files, this in its
-# own is useful but a common construct in puppet manifests is something like this:
-#
-# case $domain {
-#      "myclient.com": { $snmp_contact = "John Doe <john@myclient.com>" }
-#      default:        { $snmp_contact = "My Support <support@my.com>" }
-# }
-#
-# Over time there will be a lot of this kind of thing spread all over your manifests
-# and adding an additional client involves grepping through manifests to find all the
-# places where you have constructs like this.
-#
-# This is a data problem and shouldn't be handled in code, a using this function you
-# can do just that.
-#
-# First you configure it in site.pp:
-# $extlookup_datadir = "/etc/puppet/manifests/extdata"
-# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"]
-#
-# The array tells the code how to resolve values, first it will try to find it in
-# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv
-#
-# Now create the following data files in /etc/puppet/manifests/extdata
-#
-# domain_myclient.com.csv:
-#    snmp_contact,John Doe <john@myclient.com>
-#    root_contact,support@%{domain}
-#    client_trusted_ips,192.168.1.130,192.168.10.0/24
-#
-# common.csv:
-#    snmp_contact,My Support <support@my.com>
-#    root_contact,support@my.com
-#
-# Now you can replace the case statement with the simple single line to achieve
-# the exact same outcome:
-#
-#    $snmp_contact = extlookup("snmp_contact")
-#
-# The obove code shows some other features, you can use any fact or variable that
-# is in scope by simply using %{varname} in your data files, you can return arrays
-# by just having multiple values in the csv after the initial variable name.
-#
-# In the event that a variable is nowhere to be found a critical error will be raised
-# that will prevent your manifest from compiling, this is to avoid accidentally putting
-# in empty values etc.  You can however specify a default value:
-#
-#    $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org")
-#
-# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in
-# any data file.
-#
-# You can also specify an additional data file to search first before any others at use
-# time, for example:
-#
-# $version = extlookup("rsyslog_version", "present", "packages")
-#
-# package{"rsyslog": ensure => $version }
-#
-# This will look for a version configured in packages.csv and then in the rest as configured
-# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind
-# of use case makes puppet a lot nicer for managing large amounts of packages since you do not
-# need to edit a load of manifests to do simple things like adjust a desired version number.
-#
-# For more information on installing and writing your own custom functions see:
-#    http://docs.puppetlabs.com/guides/custom_functions.html
-#
-# For further help contact Volcane on #puppet
-require 'csv'
-
-module Puppet::Parser::Functions
-  newfunction(:extlookup, :type => :rvalue) do |args|
-    key = args[0]
-    default = "_ExtUNSET_"
-    datafile = "_ExtUNSET_"
-
-    default = args[1] if args[1]
-    datafile = args[2] if args[2]
-
-    extlookup_datadir = lookupvar('extlookup_datadir')
-    extlookup_precedence = Array.new
-
-    # precedence values can have variables embedded in them
-    # in the form %{fqdn}, you could for example do
-    #
-    #    $extlookup_precedence = ["hosts/%{fqdn}", "common"]
-    #
-    # this will result in /path/to/extdata/hosts/your.box.com.csv
-    # being searched.
-    #
-    # we parse the precedence here because the best place to specify
-    # it would be in site.pp but site.pp is only evaluated at startup
-    # so $fqdn etc would have no meaning there, this way it gets evaluated
-    # each run and has access to the right variables for that run
-    lookupvar('extlookup_precedence').each do |prec|
-      while prec =~ /%\{(.+?)\}/
-        prec.gsub!(/%\{#{$1}\}/, lookupvar($1))
-      end
-
-      extlookup_precedence << prec
-    end
-
-
-    datafiles = Array.new
-
-    # if we got a custom data file, put it first in the array of search files
-    if datafile != ""
-      datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv")
-    end
-
-    extlookup_precedence.each do |d|
-      datafiles << extlookup_datadir + "/#{d}.csv"
-    end
-
-    desired = "_ExtUNSET_"
-
-    datafiles.each do |file|
-      parser.watch_file(file) if File.exists?(file)
-
-      if desired == "_ExtUNSET_"
-        if File.exists?(file)
-          result = CSV.read(file).find_all do |r|
-            r[0] == key
-          end
-
-
-          # return just the single result if theres just one,
-          # else take all the fields in the csv and build an array
-          if result.length > 0
-            if result[0].length == 2
-              val = result[0][1].to_s
-
-              # parse %{}'s in the CSV into local variables using lookupvar()
-              while val =~ /%\{(.+?)\}/
-                val.gsub!(/%\{#{$1}\}/, lookupvar($1))
-              end
-
-              desired = val
-            elsif result[0].length > 1
-              length = result[0].length
-              cells = result[0][1,length]
-
-              # Individual cells in a CSV result are a weird data type and throws
-              # puppets yaml parsing, so just map it all to plain old strings
-              desired = cells.map do |c|
-                # parse %{}'s in the CSV into local variables using lookupvar()
-                while c =~ /%\{(.+?)\}/
-                  c.gsub!(/%\{#{$1}\}/, lookupvar($1))
-                end
-
-                c.to_s
-              end
-            end
-          end
-        end
-      end
-    end
-
-    # don't accidently return nil's and such rather throw a parse error
-    if desired == "_ExtUNSET_" && default == "_ExtUNSET_"
-      raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()"
-    else
-      desired = default if desired == "_ExtUNSET_"
-    end
-
-    desired
-  end
-end
-
-# vi:tabstop=4:expandtab:ai
diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb
new file mode 100644
index 000000000..110282d01
--- /dev/null
+++ b/ext/puppet-load.rb
@@ -0,0 +1,357 @@
+#!/usr/bin/env ruby
+# == Synopsis
+#
+# This tool can exercize a puppetmaster by simulating an arbitraty number of concurrent clients
+# in a lightweight way.
+# 
+# = Prerequisites
+# 
+# This tool requires Event Machine and em-http-request, and an installation of Puppet.
+# Event Machine can be installed from gem.
+# em-http-request can be installed from gem.
+# 
+# = Usage
+#
+#   puppet-load [-d|--debug] [--concurrency <num>] [--repeat <num>] [-V|--version] [-v|--verbose]
+#               [--node <host.domain.com>] [--facts <factfile>] [--cert <certfile>] [--key <keyfile>]
+#               [--server <server.domain.com>]
+#
+# = Description
+#
+# This is a simple script meant for doing performance tests of puppet masters. It does this
+# by simulating concurrent connections to a puppet master and asking for catalog compilation.
+#
+# = Options
+#
+# Unlike other puppet executables, puppet-load doesn't parse puppet.conf nor use puppet options
+#
+# debug::
+#   Enable full debugging.
+#
+# concurreny::
+#   Number of simulated concurrent clients.
+#
+# server::
+#   Set the puppet master hostname or IP address..
+#
+# node::
+#   Set the fully-qualified domain name of the client.  This is only used for
+#   certificate purposes, but can be used to override the discovered hostname.
+#
+# help::
+#   Print this help message
+#
+# facts::
+#   This can be used to provide facts for the compilation, directly from a YAML
+#   file as found in the clientyaml directory. If none are provided, puppet-load
+#   will look by itself using Puppet facts indirector.
+#
+# cert::
+#   This option is mandatory. It should be set to the cert PEM file that will be used
+#   to quthenticate the client connections.
+#
+# key::
+#   This option is mandatory. It should be set to the private key PEM file that will be used
+#   to quthenticate the client connections.
+#
+# timeout::
+#   The number of seconds after which a simulated client is declared in error if it didn't get
+#   a catalog. The default is 180s.
+#
+# repeat::
+#  How many times to perform the test. This means puppet-load will ask for
+#  concurrency * repeat catalogs. 
+#
+# verbose::
+#   Turn on verbose reporting.
+#
+# version::
+#   Print the puppet version number and exit.
+#
+# = Example usage
+#
+#   1) On the master host, generate a new certificate and private key for our test host:
+#   puppet ca --generate puppet-load.domain.com [*]
+#
+#   2) Copy the cert and key to the puppet-load host (which can be the same as the master one)
+#
+#   3) On the master host edit or create the auth.conf so that the catalog ACL match:
+#      path ~ ^/catalog/([^/]+)$
+#      method find
+#      allow $1
+#      allow puppet-load.domain.com
+#
+#   4) launch the master
+#
+#   5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host
+#   you want to simulate.
+#
+#   5) launch puppet-load
+#   puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 
+#
+# [*]: unfortunately at this stage Puppet trusts the certname of the connecting node more than
+#      than the node name request paramater. It means that the master will compile
+#      the puppet-load node and not the --node given.
+#
+# = TODO
+#   * Allow to simulate any different nodes
+#   * More output stats for error connections (ie report errors, HTTP code...)
+#
+#
+
+# Do an initial trap, so that cancels don't get a stack trace.
+trap(:INT) do
+  $stderr.puts "Cancelling startup"
+  exit(1)
+end
+
+require 'rubygems'
+require 'eventmachine'
+require 'em-http'
+require 'getoptlong'
+require 'puppet'
+
+$cmdargs = [
+  [ "--concurrency",  "-c", GetoptLong::REQUIRED_ARGUMENT       ],
+  [ "--node",     "-n", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--facts",          GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--repeat",   "-r", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--cert",     "-C", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--key",      "-k", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--timeout",  "-t", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--server",   "-s", GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--debug",    "-d", GetoptLong::NO_ARGUMENT       ],
+  [ "--help",     "-h", GetoptLong::NO_ARGUMENT       ],
+  [ "--verbose",  "-v", GetoptLong::NO_ARGUMENT       ],
+  [ "--version",  "-V", GetoptLong::NO_ARGUMENT       ],
+]
+
+Puppet::Util::Log.newdestination(:console)
+
+times = {}
+
+def read_facts(file)
+  YAML.load(File.read(file))
+end
+
+
+result = GetoptLong.new(*$cmdargs)
+
+$args = {}
+$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140}
+
+begin
+  result.each { |opt,arg|
+    case opt
+    when "--concurrency"
+      begin
+        $options[:concurrency] = Integer(arg)
+      rescue => detail
+        $stderr.puts "The argument to 'fork' must be an integer"
+        exit(14)
+      end
+    when "--node"
+      $options[:node] = arg
+    when "--server"
+      $options[:server] = arg
+    when "--masterport"
+      $options[:masterport] = arg
+    when "--facts"
+      $options[:facts] = arg
+    when "--repeat"
+      $options[:repeat] = Integer(arg)
+    when "--help"
+      if Puppet.features.usage?
+        RDoc::usage && exit
+      else
+        puts "No help available unless you have RDoc::usage installed"
+        exit
+      end
+    when "--version"
+      puts "%s" % Puppet.version
+      exit
+    when "--verbose"
+      Puppet::Util::Log.level = :info
+      Puppet::Util::Log.newdestination(:console)
+    when "--debug"
+      Puppet::Util::Log.level = :debug
+      Puppet::Util::Log.newdestination(:console)
+    when "--cert"
+      $options[:cert] = arg
+    when "--key"
+      $options[:key] = arg
+    end
+  }
+rescue GetoptLong::InvalidOption => detail
+  $stderr.puts detail
+  $stderr.puts "Try '#{$0} --help'"
+  exit(1)
+end
+
+unless $options[:cert] and $options[:key]
+  raise "--cert and --key are mandatory to authenticate the client"
+end
+
+unless $options[:facts] and facts = read_facts($options[:facts])
+  unless facts = Puppet::Node::Facts.find($options[:node])
+    raise "Could not find facts for %s" % $options[:node]
+  end
+end
+
+unless $options[:node]
+  raise "--node is a mandatory argument. It tells to the master what node to compile"
+end
+
+facts.values["fqdn"] = $options[:node]
+facts.values["hostname"] = $options[:node].sub(/\..+/, '')
+facts.values["domain"] = $options[:node].sub(/^[^.]+\./, '')
+
+parameters = {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(facts.render(:b64_zlib_yaml))}
+
+class RequestPool
+  include EventMachine::Deferrable
+
+  attr_reader :requests, :responses, :times, :sizes
+  attr_reader :repeat, :concurrency, :max_request
+
+  def initialize(concurrency, repeat, parameters)
+    @parameters = parameters
+    @current_request = 0
+    @max_request = repeat * concurrency
+    @repeat = repeat
+    @concurrency = concurrency
+    @requests = []
+    @responses = {:succeeded => [], :failed => []}
+    @times = {}
+    @sizes = {}
+
+    # initial spawn
+    (1..concurrency).each do |i|
+      spawn
+    end
+
+  end
+
+  def spawn_request(index)
+    EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{$options[:node]}").get(
+    :port => $options[:masterport],
+    :query => @parameters,
+    :timeout => $options[:timeout],
+    :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" },
+    :ssl => { :private_key_file => $options[:key],
+              :cert_chain_file => $options[:cert],
+              :verify_peer => false } ) do
+        Puppet.debug("starting client #{index}")
+        @times[index] = Time.now
+        @sizes[index] = 0
+    end
+  end
+
+  def add(index, conn)
+    @requests.push(conn)
+
+    conn.stream { |data|
+      @sizes[index] += data.length
+    }
+
+    conn.callback {
+      @times[index] = Time.now - @times[index]
+      code = conn.response_header.status
+      if code >= 200 && code < 300
+        Puppet.debug("Client #{index} finished successfully")
+        @responses[:succeeded].push(conn)
+      else
+        Puppet.debug("Client #{index} finished with HTTP code #{code}")
+        @responses[:failed].push(conn)
+      end
+      check_progress
+    }
+
+    conn.errback {
+      Puppet.debug("Client #{index} finished with an error: #{conn.response.error}")
+      @times[index] = Time.now - @times[index]
+      @responses[:failed].push(conn)
+      check_progress
+    }
+  end
+
+  def all_responses
+    @responses[:succeeded] + @responses[:failed]
+  end
+
+  protected
+
+  def check_progress
+    spawn unless all_spawned?
+    succeed if all_finished?
+  end
+
+  def all_spawned?
+    @requests.size >= max_request
+  end
+
+  def all_finished?
+    @responses[:failed].size + @responses[:succeeded].size >= max_request
+  end
+
+  def spawn
+    add(@current_request, spawn_request(@current_request))
+    @current_request += 1
+  end
+end
+
+
+def mean(array)
+  array.inject(0) { |sum, x| sum += x } / array.size.to_f
+end
+
+def median(array)
+  array = array.sort
+  m_pos = array.size / 2
+  return array.size % 2 == 1 ? array[m_pos] : mean(array[m_pos-1..m_pos])
+end
+
+def format_bytes(bytes)
+  if bytes < 1024
+    "%.2f B" % bytes
+  elsif bytes < 1024 * 1024
+    "%.2f KiB" % (bytes/1024.0)
+  else
+    "%.2f MiB" % (bytes/(1024.0*1024.0))
+  end
+end
+
+EM::run {
+
+  start = Time.now
+  multi = RequestPool.new($options[:concurrency], $options[:repeat], parameters)
+
+  multi.callback do
+    duration = Time.now - start
+    puts "#{multi.max_request} requests finished in #{duration} s"
+    puts "#{multi.responses[:failed].size} requests failed"
+    puts "Availability: %3.2f %%" % (100.0*multi.responses[:succeeded].size/(multi.responses[:succeeded].size+multi.responses[:failed].size))
+
+    minmax = multi.times.values.minmax
+    all_time = multi.times.values.reduce(:+)
+
+    puts "\nTime (s):"
+    puts "\tmin: #{minmax[0]} s"
+    puts "\tmax: #{minmax[1]} s"
+    puts "\taverage: #{mean(multi.times.values)} s"
+    puts "\tmedian: #{median(multi.times.values)} s"
+
+    puts "\nConcurrency: %.2f" % (all_time/duration)
+    puts "Transaction Rate (tps): %.2f t/s" % (multi.max_request / duration)
+
+    transferred = multi.sizes.values.reduce(:+)
+
+    puts "\nReceived bytes: #{format_bytes(transferred)}"
+    puts "Throughput: %.5f MiB/s" % (transferred/duration/(1024.0*1024.0))
+
+    # this is the end
+    EventMachine.stop
+  end
+}
+
+
diff --git a/ext/vim/syntax/puppet.vim b/ext/vim/syntax/puppet.vim
index 80cd91c6c..0025e2d1c 100644
--- a/ext/vim/syntax/puppet.vim
+++ b/ext/vim/syntax/puppet.vim
@@ -19,7 +19,7 @@ endif
 " match class/definition/node declarations
 syn region  puppetDefine        start="^\s*\(class\|define\|node\)\s" end="{" contains=puppetDefType,puppetDefName,puppetDefArguments,puppetNodeRe
 syn keyword puppetDefType       class define node inherits contained
-syn region  puppetDefArguments  start="(" end=")" contained contains=puppetArgument
+syn region  puppetDefArguments  start="(" end=")" contained contains=puppetArgument,puppetString
 syn match   puppetArgument      "\w\+" contained
 syn match   puppetArgument      "\$\w\+" contained
 syn match   puppetArgument      "'[^']+'" contained
@@ -33,6 +33,7 @@ syn match   puppetNodeRe        "/.*/" contained
 "FIXME: "Foo-bar" doesn't get highlighted as expected, although "foo-bar" does.
 syn match   puppetInstance      "[A-Za-z0-9_-]\+\(::[A-Za-z0-9_-]\+\)*\s*{" contains=puppetTypeName,puppetTypeDefault
 syn match   puppetInstance      "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*[[{]" contains=puppetTypeName,puppetTypeDefault
+syn match   puppetInstance      "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*<\?<|" contains=puppetTypeName,puppetTypeDefault
 syn match   puppetTypeName      "[a-z]\w*" contained
 syn match   puppetTypeDefault   "[A-Z]\w*" contained
 
@@ -68,7 +69,7 @@ syn match   puppetNotVariable   "\\$\w\+" contained
 syn match   puppetNotVariable   "\\${\w\+}" contained
 
 syn keyword puppetKeyword       import inherits include
-syn keyword puppetControl       case default if else
+syn keyword puppetControl       case default if else elsif
 syn keyword puppetSpecial       true false undef
 
 " comments last overriding everything else