Merge branch 'next'

This synchronizes the 2.7 master branch with 2.6.1RC1

20 files changed, 267 insertions, 43 deletions

diff --git a/lib/puppet/application.rb b/lib/puppet/application.rb
index 05b7d466f..0a8fbc155 100644
--- a/lib/puppet/application.rb
+++ b/lib/puppet/application.rb
@@ -264,7 +264,6 @@ class Application
         self.send(fname, value)
       end
     end
-    @option_parser.default_argv = self.command_line.args
     @option_parser
   end
 
@@ -337,7 +336,7 @@ class Application
 
     # scan command line argument
     begin
-      self.option_parser.parse!
+      self.option_parser.parse!(self.command_line.args)
     rescue OptionParser::ParseError => detail
       $stderr.puts detail
       $stderr.puts "Try 'puppet #{command_line.subcommand_name} --help'"
diff --git a/lib/puppet/network/http/rack.rb b/lib/puppet/network/http/rack.rb
index c18a07559..5b4ef7e1c 100644
--- a/lib/puppet/network/http/rack.rb
+++ b/lib/puppet/network/http/rack.rb
@@ -1,3 +1,6 @@
+require 'rack'
+require 'rack/request'
+require 'rack/response'
 
 require 'puppet/network/http'
 require 'puppet/network/http/rack/rest'
diff --git a/lib/puppet/network/rest_authconfig.rb b/lib/puppet/network/rest_authconfig.rb
index 38a8f9a09..82d5a9de2 100644
--- a/lib/puppet/network/rest_authconfig.rb
+++ b/lib/puppet/network/rest_authconfig.rb
@@ -3,6 +3,7 @@ require 'puppet/network/authconfig'
 module Puppet
   class Network::RestAuthConfig < Network::AuthConfig
 
+    extend MonitorMixin
     attr_accessor :rights
 
     DEFAULT_ACL = [
@@ -20,9 +21,11 @@ module Puppet
     ]
 
     def self.main
-      add_acl = @main.nil?
-      super
-      @main.insert_default_acl if add_acl and !@main.exists?
+      synchronize do
+        add_acl = @main.nil?
+        super
+        @main.insert_default_acl if add_acl and !@main.exists?
+      end
       @main
     end
 
diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb
index 762599cff..3f67474f9 100644
--- a/lib/puppet/node/environment.rb
+++ b/lib/puppet/node/environment.rb
@@ -1,4 +1,5 @@
 require 'puppet/util/cacher'
+require 'monitor'
 
 # Just define it, so this class has fewer load dependencies.
 class Puppet::Node
@@ -67,14 +68,23 @@ class Puppet::Node::Environment
 
   def initialize(name)
     @name = name
+    extend MonitorMixin
   end
 
   def known_resource_types
-    if @known_resource_types.nil? or @known_resource_types.stale?
-      @known_resource_types = Puppet::Resource::TypeCollection.new(self)
-      @known_resource_types.perform_initial_import
-    end
-    @known_resource_types
+    # This makes use of short circuit evaluation to get the right thread-safe
+    # per environment semantics with an efficient most common cases; we almost
+    # always just return our thread's known-resource types.  Only at the start
+    # of a compilation (after our thread var has been set to nil) or when the
+    # environment has changed do we delve deeper. 
+    Thread.current[:known_resource_types] = nil if (krt = Thread.current[:known_resource_types]) && krt.environment != self
+    Thread.current[:known_resource_types] ||= synchronize {
+      if @known_resource_types.nil? or @known_resource_types.stale?
+        @known_resource_types = Puppet::Resource::TypeCollection.new(self)
+        @known_resource_types.perform_initial_import
+      end
+      @known_resource_types
+    }
   end
 
   def module(name)
diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb
index a901c0dd6..61bb13cb6 100644
--- a/lib/puppet/parser/compiler.rb
+++ b/lib/puppet/parser/compiler.rb
@@ -19,7 +19,11 @@ class Puppet::Parser::Compiler
   rescue => detail
     puts detail.backtrace if Puppet[:trace]
     raise Puppet::Error, "#{detail} on node #{node.name}"
-  end
+  ensure
+    # We get these from the environment and only cache them in a thread 
+    # variable for the duration of the compilation.
+    Thread.current[:known_resource_types] = nil
+ end
 
   attr_reader :node, :facts, :collections, :catalog, :node_scope, :resources, :relationships
 
diff --git a/lib/puppet/parser/functions/extlookup.rb b/lib/puppet/parser/functions/extlookup.rb
new file mode 100644
index 000000000..ee230e7ce
--- /dev/null
+++ b/lib/puppet/parser/functions/extlookup.rb
@@ -0,0 +1,173 @@
+# Puppet External Data Sources
+#
+# This is a parser function to read data from external files, this version
+# uses CSV files but the concept can easily be adjust for databases, yaml
+# or any other queryable data source.
+#
+# The object of this is to make it obvious when it's being used, rather than
+# magically loading data in when an module is loaded I prefer to look at the code
+# and see statements like:
+#
+#     $snmp_contact = extlookup("snmp_contact")
+#
+# The above snippet will load the snmp_contact value from CSV files, this in its
+# own is useful but a common construct in puppet manifests is something like this:
+#
+# case $domain {
+#      "myclient.com": { $snmp_contact = "John Doe <john@myclient.com>" }
+#      default:        { $snmp_contact = "My Support <support@my.com>" }
+# }
+#
+# Over time there will be a lot of this kind of thing spread all over your manifests
+# and adding an additional client involves grepping through manifests to find all the
+# places where you have constructs like this.
+#
+# This is a data problem and shouldn't be handled in code, a using this function you
+# can do just that.
+#
+# First you configure it in site.pp:
+# $extlookup_datadir = "/etc/puppet/manifests/extdata"
+# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"]
+#
+# The array tells the code how to resolve values, first it will try to find it in
+# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv
+#
+# Now create the following data files in /etc/puppet/manifests/extdata
+#
+# domain_myclient.com.csv:
+#    snmp_contact,John Doe <john@myclient.com>
+#    root_contact,support@%{domain}
+#    client_trusted_ips,192.168.1.130,192.168.10.0/24
+#
+# common.csv:
+#    snmp_contact,My Support <support@my.com>
+#    root_contact,support@my.com
+#
+# Now you can replace the case statement with the simple single line to achieve
+# the exact same outcome:
+#
+#    $snmp_contact = extlookup("snmp_contact")
+#
+# The obove code shows some other features, you can use any fact or variable that
+# is in scope by simply using %{varname} in your data files, you can return arrays
+# by just having multiple values in the csv after the initial variable name.
+#
+# In the event that a variable is nowhere to be found a critical error will be raised
+# that will prevent your manifest from compiling, this is to avoid accidentally putting
+# in empty values etc.  You can however specify a default value:
+#
+#    $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org")
+#
+# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in
+# any data file.
+#
+# You can also specify an additional data file to search first before any others at use
+# time, for example:
+#
+# $version = extlookup("rsyslog_version", "present", "packages")
+#
+# package{"rsyslog": ensure => $version }
+#
+# This will look for a version configured in packages.csv and then in the rest as configured
+# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind
+# of use case makes puppet a lot nicer for managing large amounts of packages since you do not
+# need to edit a load of manifests to do simple things like adjust a desired version number.
+#
+# For more information on installing and writing your own custom functions see:
+#    http://docs.puppetlabs.com/guides/custom_functions.html
+#
+# For further help contact Volcane on #puppet
+require 'csv'
+
+module Puppet::Parser::Functions
+  newfunction(:extlookup, :type => :rvalue) do |args|
+    key = args[0]
+
+    default  = args[1]
+    datafile = args[2]
+
+    raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3
+
+    extlookup_datadir = lookupvar('extlookup_datadir')
+    extlookup_precedence = Array.new
+
+    # precedence values can have variables embedded in them
+    # in the form %{fqdn}, you could for example do
+    #
+    #    $extlookup_precedence = ["hosts/%{fqdn}", "common"]
+    #
+    # this will result in /path/to/extdata/hosts/your.box.com.csv
+    # being searched.
+    #
+    # we parse the precedence here because the best place to specify
+    # it would be in site.pp but site.pp is only evaluated at startup
+    # so $fqdn etc would have no meaning there, this way it gets evaluated
+    # each run and has access to the right variables for that run
+    lookupvar('extlookup_precedence').each do |prec|
+      while prec =~ /%\{(.+?)\}/
+        prec.gsub!(/%\{#{$1}\}/, lookupvar($1))
+      end
+
+      extlookup_precedence << prec
+    end
+
+
+    datafiles = Array.new
+
+    # if we got a custom data file, put it first in the array of search files
+    if datafile != ""
+      datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv")
+    end
+
+    extlookup_precedence.each do |d|
+      datafiles << extlookup_datadir + "/#{d}.csv"
+    end
+
+    desired = nil
+
+    datafiles.each do |file|
+      parser = Puppet::Parser::Parser.new(environment)
+      parser.watch_file(file) if File.exists?(file)
+
+      if desired.nil?
+        if File.exists?(file)
+          result = CSV.read(file).find_all do |r|
+            r[0] == key
+          end
+
+
+          # return just the single result if theres just one,
+          # else take all the fields in the csv and build an array
+          if result.length > 0
+            if result[0].length == 2
+              val = result[0][1].to_s
+
+              # parse %{}'s in the CSV into local variables using lookupvar()
+              while val =~ /%\{(.+?)\}/
+                val.gsub!(/%\{#{$1}\}/, lookupvar($1))
+              end
+
+              desired = val
+            elsif result[0].length > 1
+              length = result[0].length
+              cells = result[0][1,length]
+
+              # Individual cells in a CSV result are a weird data type and throws
+              # puppets yaml parsing, so just map it all to plain old strings
+              desired = cells.map do |c|
+                # parse %{}'s in the CSV into local variables using lookupvar()
+                while c =~ /%\{(.+?)\}/
+                  c.gsub!(/%\{#{$1}\}/, lookupvar($1))
+                end
+
+                c.to_s
+              end
+            end
+          end
+        end
+      end
+    end
+
+    desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()"
+  end
+end
diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb
index 1e10ff96c..aa04f17a0 100644
--- a/lib/puppet/parser/lexer.rb
+++ b/lib/puppet/parser/lexer.rb
@@ -221,7 +221,7 @@ class Puppet::Parser::Lexer
   TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true
 
   TOKENS.add_token :SQUOTE, "'" do |lexer, value|
-    [TOKENS[:STRING], lexer.slurpstring(value).first ]
+    [TOKENS[:STRING], lexer.slurpstring(value,["'"],:ignore_invalid_esapes).first ]
   end
 
   DQ_initial_token_types      = {'$' => :DQPRE,'"' => :STRING}
@@ -517,8 +517,7 @@ class Puppet::Parser::Lexer
 
   # we've encountered the start of a string...
   # slurp in the rest of the string and return it
-  Valid_escapes_in_strings = %w{ \\  $ ' " n t s }+["\n"]
-  def slurpstring(terminators)
+  def slurpstring(terminators,escapes=%w{ \\  $ ' " n t s }+["\n"],ignore_invalid_escapes=false)
     # we search for the next quote that isn't preceded by a
     # backslash; the caret is there to match empty strings
     str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'"
@@ -529,10 +528,10 @@ class Puppet::Parser::Lexer
       when 't'; "\t"
       when 's'; " "
       else
-        if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'")
+        if escapes.include? ch
           ch
         else
-          Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}"
+          Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" unless ignore_invalid_escapes
           "\\#{ch}"
         end
       end
diff --git a/lib/puppet/parser/resource.rb b/lib/puppet/parser/resource.rb
index c956a1106..3c451929e 100644
--- a/lib/puppet/parser/resource.rb
+++ b/lib/puppet/parser/resource.rb
@@ -64,6 +64,7 @@ class Puppet::Parser::Resource < Puppet::Resource
 
   # Retrieve the associated definition and evaluate it.
   def evaluate
+    @evaluated = true
     if klass = resource_type and ! builtin_type?
       finish
       return klass.evaluate_code(self)
@@ -72,8 +73,6 @@ class Puppet::Parser::Resource < Puppet::Resource
     else
       self.fail "Cannot find definition #{type}"
     end
-  ensure
-    @evaluated = true
   end
 
   # Mark this resource as both exported and virtual,
diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb
index 35ad49593..09aa636e1 100644
--- a/lib/puppet/parser/type_loader.rb
+++ b/lib/puppet/parser/type_loader.rb
@@ -88,7 +88,7 @@ class Puppet::Parser::TypeLoader
         nil
       end
       if result = yield(filename)
-        Puppet.info "Automatically imported #{name} from #{filename} into #{environment}"
+        Puppet.debug "Automatically imported #{name} from #{filename} into #{environment}"
         result.module_name = modname if modname and result.respond_to?(:module_name=)
         return result
       end
diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb
index 4b4342d11..a8668d844 100644
--- a/lib/puppet/resource/catalog.rb
+++ b/lib/puppet/resource/catalog.rb
@@ -57,7 +57,7 @@ class Puppet::Resource::Catalog < Puppet::SimpleGraph
   end
 
   def title_key_for_ref( ref )
-    ref =~ /^([\w:]+)\[(.*)\]$/m
+    ref =~ /^([-\w:]+)\[(.*)\]$/m
     [$1, $2]
   end
 
diff --git a/lib/puppet/resource/type.rb b/lib/puppet/resource/type.rb
index 85c0979c1..6296d26e5 100644
--- a/lib/puppet/resource/type.rb
+++ b/lib/puppet/resource/type.rb
@@ -174,12 +174,29 @@ class Puppet::Resource::Type
     @name.is_a?(Regexp)
   end
 
+  # MQR TODO:
+  #
+  # The change(s) introduced by the fix for #4270 are mostly silly & should be 
+  # removed, though we didn't realize it at the time.  If it can be established/
+  # ensured that nodes never call parent_type and that resource_types are always
+  # (as they should be) members of exactly one resource_type_collection the 
+  # following method could / should be replaced with:
+  #
+  # def parent_type
+  #   @parent_type ||= parent && (
+  #     resource_type_collection.find_or_load([name],parent,type.to_sym) ||
+  #     fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{resource_type_collection.environment}"
+  #   )
+  # end
+  #
+  # ...and then the rest of the changes around passing in scope reverted.
+  #
   def parent_type(scope = nil)
     return nil unless parent
 
     unless @parent_type
       raise "Must pass scope to parent_type when called first time" unless scope
-      unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", scope.namespaces, parent)
+      unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", [name], parent)
         fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{scope.environment}"
       end
     end
diff --git a/lib/puppet/resource/type_collection.rb b/lib/puppet/resource/type_collection.rb
index 6a933362f..90b6df9c1 100644
--- a/lib/puppet/resource/type_collection.rb
+++ b/lib/puppet/resource/type_collection.rb
@@ -153,6 +153,7 @@ class Puppet::Resource::TypeCollection
   end
 
   def perform_initial_import
+    return if Puppet.settings[:ignoreimport]
     parser = Puppet::Parser::Parser.new(environment)
     if code = Puppet.settings.uninterpolated_value(:code, environment.to_s) and code != ""
       parser.string = code
diff --git a/lib/puppet/simple_graph.rb b/lib/puppet/simple_graph.rb
index 945b1beb3..55b39fadf 100644
--- a/lib/puppet/simple_graph.rb
+++ b/lib/puppet/simple_graph.rb
@@ -318,16 +318,15 @@ class Puppet::SimpleGraph
     # to container leaves, but that would result in many more
     # relationships.
     stage_class = Puppet::Type.type(:stage)
+    whit_class  = Puppet::Type.type(:whit)
     containers = other.topsort.find_all { |v| (v.is_a?(type) or v.is_a?(stage_class)) and vertex?(v) }
     containers.each do |container|
       # Get the list of children from the other graph.
       children = other.adjacent(container, :direction => :out)
 
-      # Just remove the container if it's empty.
-      if children.empty?
-        remove_vertex!(container)
-        next
-      end
+      # MQR TODO: Luke suggests that it should be possible to refactor the system so that
+      #           container nodes are retained, thus obviating the need for the whit. 
+      children = [whit_class.new(:name => container.name, :catalog => other)] if children.empty?
 
       # First create new edges for each of the :in edges
       [:in, :out].each do |dir|
diff --git a/lib/puppet/type/file/content.rb b/lib/puppet/type/file/content.rb
index 74b380f55..0d0bb5571 100755
--- a/lib/puppet/type/file/content.rb
+++ b/lib/puppet/type/file/content.rb
@@ -184,7 +184,7 @@ module Puppet
     end
 
     def chunk_file_from_source(source_or_content)
-      request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path)
+      request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path.sub(/^\//,''))
       connection = Puppet::Network::HttpPool.http_instance(source_or_content.server, source_or_content.port)
       connection.request_get(indirection2uri(request), add_accept_encoding({"Accept" => "raw"})) do |response|
         case response.code
diff --git a/lib/puppet/type/whit.rb b/lib/puppet/type/whit.rb
new file mode 100644
index 000000000..6e5ba9eab
--- /dev/null
+++ b/lib/puppet/type/whit.rb
@@ -0,0 +1,7 @@
+Puppet::Type.newtype(:whit) do
+  desc "The smallest possible resource type, for when you need a resource and naught else."
+
+  newparam :name do
+    desc "The name of the whit, because it must have one."
+  end
+end
diff --git a/lib/puppet/type/yumrepo.rb b/lib/puppet/type/yumrepo.rb
index e0ed3ad37..5d86b1c75 100644
--- a/lib/puppet/type/yumrepo.rb
+++ b/lib/puppet/type/yumrepo.rb
@@ -85,7 +85,7 @@ module Puppet
       clear
       inifile.each_section do |s|
         next if s.name == "main"
-        obj = create(:name => s.name, :audit => check)
+        obj = new(:name => s.name, :audit => check)
         current_values = obj.retrieve
         obj.eachproperty do |property|
           if current_values[property].nil?
diff --git a/lib/puppet/util/cacher.rb b/lib/puppet/util/cacher.rb
index 8785c694f..3dddec0d4 100644
--- a/lib/puppet/util/cacher.rb
+++ b/lib/puppet/util/cacher.rb
@@ -1,3 +1,5 @@
+require 'monitor'
+
 module Puppet::Util::Cacher
   module Expirer
     attr_reader :timestamp
@@ -49,7 +51,7 @@ module Puppet::Util::Cacher
       define_method(name.to_s + "=") do |value|
         # Make sure the cache timestamp is set
         cache_timestamp
-        value_cache[name] = value
+        value_cache.synchronize { value_cache[name] = value }
       end
 
       if ttl = options[:ttl]
@@ -70,6 +72,7 @@ module Puppet::Util::Cacher
 
   # Methods that get added to instances.
   module InstanceMethods
+
     def expire
       # Only expire if we have an expirer.  This is
       # mostly so that we can comfortably handle cases
@@ -92,15 +95,17 @@ module Puppet::Util::Cacher
     end
 
     def cached_value(name)
-      # Allow a nil expirer, in which case we regenerate the value every time.
-      if expired_by_expirer?(name)
-        value_cache.clear
-        @cache_timestamp = Time.now
-      elsif expired_by_ttl?(name)
-        value_cache.delete(name)
+      value_cache.synchronize do
+        # Allow a nil expirer, in which case we regenerate the value every time.
+        if expired_by_expirer?(name)
+          value_cache.clear
+          @cache_timestamp = Time.now
+        elsif expired_by_ttl?(name)
+          value_cache.delete(name)
+        end
+        value_cache[name] = send("init_#{name}") unless value_cache.include?(name)
+        value_cache[name]
       end
-      value_cache[name] = send("init_#{name}") unless value_cache.include?(name)
-      value_cache[name]
     end
 
     def expired_by_expirer?(name)
@@ -121,7 +126,7 @@ module Puppet::Util::Cacher
     end
 
     def value_cache
-      @value_cache ||= {}
+      @value_cache ||= {}.extend(MonitorMixin)
     end
   end
 end
diff --git a/lib/puppet/util/monkey_patches.rb b/lib/puppet/util/monkey_patches.rb
index e035afd9f..9664ff310 100644
--- a/lib/puppet/util/monkey_patches.rb
+++ b/lib/puppet/util/monkey_patches.rb
@@ -1,4 +1,7 @@
-Process.maxgroups = 1024
+
+unless defined? JRUBY_VERSION
+  Process.maxgroups = 1024
+end
 
 module RDoc
   def self.caller(skip=nil)
diff --git a/lib/puppet/util/rdoc.rb b/lib/puppet/util/rdoc.rb
index 4a80b069b..085d8ec93 100644
--- a/lib/puppet/util/rdoc.rb
+++ b/lib/puppet/util/rdoc.rb
@@ -41,7 +41,7 @@ module Puppet::Util::RDoc
   def manifestdoc(files)
     Puppet[:ignoreimport] = true
     files.select { |f| FileTest.file?(f) }.each do |f|
-      parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment])
+      parser = Puppet::Parser::Parser.new(Puppet::Node::Environment.new(Puppet[:environment]))
       parser.file = f
       ast = parser.parse
       output(f, ast)
diff --git a/lib/puppet/util/rdoc/parser.rb b/lib/puppet/util/rdoc/parser.rb
index 573d1766f..63df38ab9 100644
--- a/lib/puppet/util/rdoc/parser.rb
+++ b/lib/puppet/util/rdoc/parser.rb
@@ -15,6 +15,8 @@ module RDoc
 class Parser
   extend ParserFactory
 
+  SITE = "__site__"
+
   attr_accessor :ast, :input_file_name, :top_level
 
   # parser registration into RDoc
@@ -74,7 +76,7 @@ class Parser
 
   # split_module tries to find if +path+ belongs to the module path
   # if it does, it returns the module name, otherwise if we are sure
-  # it is part of the global manifest path, "<site>" is returned.
+  # it is part of the global manifest path, "__site__" is returned.
   # And finally if this path couldn't be mapped anywhere, nil is returned.
   def split_module(path)
     # find a module
@@ -105,7 +107,7 @@ class Parser
     end
     # we are under a global manifests
     Puppet.debug "rdoc: global manifests"
-    "<site>"
+    SITE
   end
 
   # create documentation for the top level +container+
@@ -128,7 +130,7 @@ class Parser
     Puppet.debug "rdoc: scanning for #{name}"
 
     container.module_name = name
-    container.global=true if name == "<site>"
+    container.global=true if name == SITE
 
     @stats.num_modules += 1
     container, name  = get_class_or_module(container,name)