From 64386cf185c6d99ab5639ca8b2e0f0f4c47021af Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 20 Jul 2010 15:01:59 +1000 Subject: Fixed Indirection reference --- lib/puppet/reference/indirection.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/reference/indirection.rb b/lib/puppet/reference/indirection.rb index 549299b36..d14510c16 100644 --- a/lib/puppet/reference/indirection.rb +++ b/lib/puppet/reference/indirection.rb @@ -1,5 +1,5 @@ require 'puppet/indirector/indirection' -require 'puppet/checksum' +require 'puppet/util/checksums' require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' -- cgit From a0a63c3735f53cbf895a9e30e6a2c1dfd2ae4c4e Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 20 Jul 2010 15:25:05 +1000 Subject: Fixed network and indirection reference --- lib/puppet/rails/host.rb | 1 + lib/puppet/reference/network.rb | 2 ++ 2 files changed, 3 insertions(+) (limited to 'lib/puppet') diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb index 6c008bccf..986cebd0a 100644 --- a/lib/puppet/rails/host.rb +++ b/lib/puppet/rails/host.rb @@ -1,3 +1,4 @@ +require 'puppet/rails' require 'puppet/rails/resource' require 'puppet/rails/fact_name' require 'puppet/rails/source_file' diff --git a/lib/puppet/reference/network.rb b/lib/puppet/reference/network.rb index a3bff656b..15e4f6769 100644 --- a/lib/puppet/reference/network.rb +++ b/lib/puppet/reference/network.rb @@ -1,3 +1,5 @@ +require 'puppet/network/handler' + network = Puppet::Util::Reference.newreference :network, :depth => 2, :doc => "Available network handlers and clients" do ret = "" Puppet::Network::Handler.subclasses.sort { |a,b| a.to_s <=> b.to_s }.each do |name| -- cgit From 59a23d69c20a8ab1f7dc9ff1d109305f9027641c Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sat, 24 Jul 2010 14:48:13 -0700 Subject: Fix for #3382 -- Empty classes as graph placeholders As Brice discovered, the problem was that we simply ignored empty classes in the graph when determining application order. This patch instead replaces them with a resource of a new type which we've frequently noted the (internal) need for: a whit, the smallest possible resource, which has no properties or other semantics apart from its existence and its name. This resource then ensures application order through the normal mechanisms. --- lib/puppet/simple_graph.rb | 9 ++++----- lib/puppet/type/whit.rb | 7 +++++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 lib/puppet/type/whit.rb (limited to 'lib/puppet') diff --git a/lib/puppet/simple_graph.rb b/lib/puppet/simple_graph.rb index 945b1beb3..55b39fadf 100644 --- a/lib/puppet/simple_graph.rb +++ b/lib/puppet/simple_graph.rb @@ -318,16 +318,15 @@ class Puppet::SimpleGraph # to container leaves, but that would result in many more # relationships. stage_class = Puppet::Type.type(:stage) + whit_class = Puppet::Type.type(:whit) containers = other.topsort.find_all { |v| (v.is_a?(type) or v.is_a?(stage_class)) and vertex?(v) } containers.each do |container| # Get the list of children from the other graph. children = other.adjacent(container, :direction => :out) - # Just remove the container if it's empty. - if children.empty? - remove_vertex!(container) - next - end + # MQR TODO: Luke suggests that it should be possible to refactor the system so that + # container nodes are retained, thus obviating the need for the whit. + children = [whit_class.new(:name => container.name, :catalog => other)] if children.empty? # First create new edges for each of the :in edges [:in, :out].each do |dir| diff --git a/lib/puppet/type/whit.rb b/lib/puppet/type/whit.rb new file mode 100644 index 000000000..6e5ba9eab --- /dev/null +++ b/lib/puppet/type/whit.rb @@ -0,0 +1,7 @@ +Puppet::Type.newtype(:whit) do + desc "The smallest possible resource type, for when you need a resource and naught else." + + newparam :name do + desc "The name of the whit, because it must have one." + end +end -- cgit From 9778f2a47922a66e59d571c1c98552223a817ce1 Mon Sep 17 00:00:00 2001 From: Nick Lewis Date: Fri, 23 Jul 2010 17:04:22 -0700 Subject: [#4242] Fixed recursion due to parents including their children Resources mark themselves as evaluated to prevent being evaluated again. Unfortunately, they were not marking themselves until after they had finished being completely evaluated. Thus, there was nothing actually stopping recursive evaluations. This patch just makes resources mark themselves as evaluated when they start evaluating, and adds tests. The original setting of evaluated was done in an ensure block, so this doesn't change the behavior of a resource which fails to evaluate. The only places evaluated? is checked aren't affected by this change, as they wouldn't want to evaluate it when it's already being evaluated anyway. --- lib/puppet/parser/resource.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/parser/resource.rb b/lib/puppet/parser/resource.rb index c956a1106..3c451929e 100644 --- a/lib/puppet/parser/resource.rb +++ b/lib/puppet/parser/resource.rb @@ -64,6 +64,7 @@ class Puppet::Parser::Resource < Puppet::Resource # Retrieve the associated definition and evaluate it. def evaluate + @evaluated = true if klass = resource_type and ! builtin_type? finish return klass.evaluate_code(self) @@ -72,8 +73,6 @@ class Puppet::Parser::Resource < Puppet::Resource else self.fail "Cannot find definition #{type}" end - ensure - @evaluated = true end # Mark this resource as both exported and virtual, -- cgit From 28bb1950cf040eaf5196e39f43b3a9d0d61175a7 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 16 Jul 2010 14:42:01 +1000 Subject: Fixed yumrepo type deprecation wanring ` --- lib/puppet/type/yumrepo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/type/yumrepo.rb b/lib/puppet/type/yumrepo.rb index e0ed3ad37..5d86b1c75 100644 --- a/lib/puppet/type/yumrepo.rb +++ b/lib/puppet/type/yumrepo.rb @@ -85,7 +85,7 @@ module Puppet clear inifile.each_section do |s| next if s.name == "main" - obj = create(:name => s.name, :audit => check) + obj = new(:name => s.name, :audit => check) current_values = obj.retrieve obj.eachproperty do |property| if current_values[property].nil? -- cgit From 7ad7eb12fade4695abf08392e80ca57003353140 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Wed, 21 Jul 2010 23:10:46 +0200 Subject: Fix #4286 - rename puppetdoc global module to __site__ < and > might be invalid or borderline chars to use for a file name or an url. This patch changes those characters to __. Signed-off-by: Brice Figureau --- lib/puppet/util/rdoc/parser.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/util/rdoc/parser.rb b/lib/puppet/util/rdoc/parser.rb index 573d1766f..63df38ab9 100644 --- a/lib/puppet/util/rdoc/parser.rb +++ b/lib/puppet/util/rdoc/parser.rb @@ -15,6 +15,8 @@ module RDoc class Parser extend ParserFactory + SITE = "__site__" + attr_accessor :ast, :input_file_name, :top_level # parser registration into RDoc @@ -74,7 +76,7 @@ class Parser # split_module tries to find if +path+ belongs to the module path # if it does, it returns the module name, otherwise if we are sure - # it is part of the global manifest path, "" is returned. + # it is part of the global manifest path, "__site__" is returned. # And finally if this path couldn't be mapped anywhere, nil is returned. def split_module(path) # find a module @@ -105,7 +107,7 @@ class Parser end # we are under a global manifests Puppet.debug "rdoc: global manifests" - "" + SITE end # create documentation for the top level +container+ @@ -128,7 +130,7 @@ class Parser Puppet.debug "rdoc: scanning for #{name}" container.module_name = name - container.global=true if name == "" + container.global=true if name == SITE @stats.num_modules += 1 container, name = get_class_or_module(container,name) -- cgit From 63ec207cc88828b8b0ad421c7fcdd8a5715e7fd3 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sat, 24 Jul 2010 10:35:21 -0700 Subject: Minimal fix for #4297, with notes for follow-up In retrospect it appears that the fix for #4270 was incomplete and somewhat off target. This patch fixes the one demonstrably incorrect part (the namespace) and adds a comment outlining what remains to be done to clean up the code; these additional changes, while needed for maintanability, are inappropriate for a quick turnaround crucial bug fix release such as 2.6.1, at which this patch is targeted. --- lib/puppet/resource/type.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/resource/type.rb b/lib/puppet/resource/type.rb index 85c0979c1..6296d26e5 100644 --- a/lib/puppet/resource/type.rb +++ b/lib/puppet/resource/type.rb @@ -174,12 +174,29 @@ class Puppet::Resource::Type @name.is_a?(Regexp) end + # MQR TODO: + # + # The change(s) introduced by the fix for #4270 are mostly silly & should be + # removed, though we didn't realize it at the time. If it can be established/ + # ensured that nodes never call parent_type and that resource_types are always + # (as they should be) members of exactly one resource_type_collection the + # following method could / should be replaced with: + # + # def parent_type + # @parent_type ||= parent && ( + # resource_type_collection.find_or_load([name],parent,type.to_sym) || + # fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{resource_type_collection.environment}" + # ) + # end + # + # ...and then the rest of the changes around passing in scope reverted. + # def parent_type(scope = nil) return nil unless parent unless @parent_type raise "Must pass scope to parent_type when called first time" unless scope - unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", scope.namespaces, parent) + unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", [name], parent) fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{scope.environment}" end end -- cgit From 23830504dfeb48b2d162e44b84b6f9dfa97e4e7e Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Thu, 22 Jul 2010 20:36:24 +0200 Subject: Fix #4302 - Compilation speed regression compared to 2.6 Each time the compiler was accessing the loaded types, we were checking if the manifests had changed. This incurred a large performance cost compared to 0.25 and introduced race conditions if manifests changed while a thread was in the middle of a compilation. This tentative fix, based on Brice's, makes sure each thread will get access to the same loaded types collection for the durration of a compilation, even if the manifests change. We now only check for changed files at the start of a compilation or if the environment changes, and we maintain a per environment thread lock so that only one thread at a time can be reloading any particular environment (and the need-check is done inside the synchronize block so that only the first will actually load it). As long as the manifests don't change, the threads will share the same collection, so there is only duplication in memory for a brief window surrounding a change. Signed-off-by: Brice Figureau Second-author: Markus Roberts --- lib/puppet/node/environment.rb | 20 +++++++++++++++----- lib/puppet/parser/compiler.rb | 5 +++++ 2 files changed, 20 insertions(+), 5 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 762599cff..3f67474f9 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -1,4 +1,5 @@ require 'puppet/util/cacher' +require 'monitor' # Just define it, so this class has fewer load dependencies. class Puppet::Node @@ -67,14 +68,23 @@ class Puppet::Node::Environment def initialize(name) @name = name + extend MonitorMixin end def known_resource_types - if @known_resource_types.nil? or @known_resource_types.stale? - @known_resource_types = Puppet::Resource::TypeCollection.new(self) - @known_resource_types.perform_initial_import - end - @known_resource_types + # This makes use of short circuit evaluation to get the right thread-safe + # per environment semantics with an efficient most common cases; we almost + # always just return our thread's known-resource types. Only at the start + # of a compilation (after our thread var has been set to nil) or when the + # environment has changed do we delve deeper. + Thread.current[:known_resource_types] = nil if (krt = Thread.current[:known_resource_types]) && krt.environment != self + Thread.current[:known_resource_types] ||= synchronize { + if @known_resource_types.nil? or @known_resource_types.stale? + @known_resource_types = Puppet::Resource::TypeCollection.new(self) + @known_resource_types.perform_initial_import + end + @known_resource_types + } end def module(name) diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index a901c0dd6..760d5a75a 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -15,6 +15,11 @@ class Puppet::Parser::Compiler include Puppet::Resource::TypeCollectionHelper def self.compile(node) + # At the start of a new compile we don't assume anything about + # known_resouce_types; we'll get these from the environment and + # cache them in a thread variable for the duration of the + # compilation. + Thread.current[:known_resource_types] = nil new(node).compile.to_resource rescue => detail puts detail.backtrace if Puppet[:trace] -- cgit From 1d494a3104e9794cc09ba27c701ced68a74fa398 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Fri, 23 Jul 2010 08:47:48 -0700 Subject: Tweak to fix for #4302--dangling ref to known_resource_types Since we were clearing the thread variable containing the compiler's reference to it's environment's known resource types at the start of each compile the reference remaining at the end of a compilation could never be used and was thus just garbage that we were arbitrarily retaining. This patch moves the clearing of the thread var to the _end_ of compilation so that it's always nil except in the middle of a compile. This raises an interesting question; should the ref just live on the compiler object and we could dispense with the thread-var? It might require things that now only know about the environment to need a ref to the compiler and introduce other thread issues (e.g. we might just end up needing a :current_compiler thread variable, for no net gain in simplicity). --- lib/puppet/parser/compiler.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 760d5a75a..61bb13cb6 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -15,16 +15,15 @@ class Puppet::Parser::Compiler include Puppet::Resource::TypeCollectionHelper def self.compile(node) - # At the start of a new compile we don't assume anything about - # known_resouce_types; we'll get these from the environment and - # cache them in a thread variable for the duration of the - # compilation. - Thread.current[:known_resource_types] = nil new(node).compile.to_resource rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "#{detail} on node #{node.name}" - end + ensure + # We get these from the environment and only cache them in a thread + # variable for the duration of the compilation. + Thread.current[:known_resource_types] = nil + end attr_reader :node, :facts, :collections, :catalog, :node_scope, :resources, :relationships -- cgit From 000fd1e83782c70fc9d9b032b52d96800cab2121 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Fri, 23 Jul 2010 11:31:40 -0700 Subject: Fix for #4303 -- reverting to old escaping in '-strings Single quoted used to allow escape on single quotes and pass all other characters through without comment; now the do again. --- lib/puppet/parser/lexer.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb index 1e10ff96c..aa04f17a0 100644 --- a/lib/puppet/parser/lexer.rb +++ b/lib/puppet/parser/lexer.rb @@ -221,7 +221,7 @@ class Puppet::Parser::Lexer TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true TOKENS.add_token :SQUOTE, "'" do |lexer, value| - [TOKENS[:STRING], lexer.slurpstring(value).first ] + [TOKENS[:STRING], lexer.slurpstring(value,["'"],:ignore_invalid_esapes).first ] end DQ_initial_token_types = {'$' => :DQPRE,'"' => :STRING} @@ -517,8 +517,7 @@ class Puppet::Parser::Lexer # we've encountered the start of a string... # slurp in the rest of the string and return it - Valid_escapes_in_strings = %w{ \\ $ ' " n t s }+["\n"] - def slurpstring(terminators) + def slurpstring(terminators,escapes=%w{ \\ $ ' " n t s }+["\n"],ignore_invalid_escapes=false) # we search for the next quote that isn't preceded by a # backslash; the caret is there to match empty strings str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'" @@ -529,10 +528,10 @@ class Puppet::Parser::Lexer when 't'; "\t" when 's'; " " else - if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'") + if escapes.include? ch ch else - Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" + Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" unless ignore_invalid_escapes "\\#{ch}" end end -- cgit From 636079f114034f5dced1772da206d1b1b1a537ed Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 23 Jul 2010 02:32:50 +1000 Subject: Fixed #4304 - Changed logging level for auto import message --- lib/puppet/parser/type_loader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 35ad49593..09aa636e1 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -88,7 +88,7 @@ class Puppet::Parser::TypeLoader nil end if result = yield(filename) - Puppet.info "Automatically imported #{name} from #{filename} into #{environment}" + Puppet.debug "Automatically imported #{name} from #{filename} into #{environment}" result.module_name = modname if modname and result.respond_to?(:module_name=) return result end -- cgit From 9569136329f87eeb102e589a3d5e9c26d8632aad Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Fri, 23 Jul 2010 09:50:29 -0700 Subject: Fix for 4314 -- Need to allow '-' in class name for refs I almost changed this to correspond to the lexer pattern ([a-z0-9][-\w]*) but that isn't quite right either (it would reintroduce the '::' problem). Another option I considered was [^\[]+, but that has it's own problems. In the end I just made the minimal change, adding '-' to the acceptable characters. --- lib/puppet/resource/catalog.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb index 4b4342d11..a8668d844 100644 --- a/lib/puppet/resource/catalog.rb +++ b/lib/puppet/resource/catalog.rb @@ -57,7 +57,7 @@ class Puppet::Resource::Catalog < Puppet::SimpleGraph end def title_key_for_ref( ref ) - ref =~ /^([\w:]+)\[(.*)\]$/m + ref =~ /^([-\w:]+)\[(.*)\]$/m [$1, $2] end -- cgit From 03313b81c696d12c756a5ff5adced5bed162bbfc Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Wed, 21 Jul 2010 22:56:58 +0200 Subject: Fix #4319 - source file url sent to the master is invalid We were sending an incorrect (containing a //) url for sourced file content since the file streaming patches. Depending on the webserver in front of puppet it could fail (for instance nginx+mongrel). This patch fixes the offending // in each sourced file urls. Signed-off-by: Brice Figureau --- lib/puppet/type/file/content.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/type/file/content.rb b/lib/puppet/type/file/content.rb index 74b380f55..0d0bb5571 100755 --- a/lib/puppet/type/file/content.rb +++ b/lib/puppet/type/file/content.rb @@ -184,7 +184,7 @@ module Puppet end def chunk_file_from_source(source_or_content) - request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path) + request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path.sub(/^\//,'')) connection = Puppet::Network::HttpPool.http_instance(source_or_content.server, source_or_content.port) connection.request_get(indirection2uri(request), add_accept_encoding({"Accept" => "raw"})) do |response| case response.code -- cgit From d38e52224f19b4fcedef722b2aab8fc94a59b481 Mon Sep 17 00:00:00 2001 From: Jesse Wolfe Date: Thu, 22 Jul 2010 14:33:58 -0700 Subject: [#4333] old optparse doesn't support default_argv= optparse hasn't always had the concept of default_argv. Fortunately, we don't really need it. --- lib/puppet/application.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/application.rb b/lib/puppet/application.rb index 05b7d466f..0a8fbc155 100644 --- a/lib/puppet/application.rb +++ b/lib/puppet/application.rb @@ -264,7 +264,6 @@ class Application self.send(fname, value) end end - @option_parser.default_argv = self.command_line.args @option_parser end @@ -337,7 +336,7 @@ class Application # scan command line argument begin - self.option_parser.parse! + self.option_parser.parse!(self.command_line.args) rescue OptionParser::ParseError => detail $stderr.puts detail $stderr.puts "Try 'puppet #{command_line.subcommand_name} --help'" -- cgit From 13c71b97d93edf033897dc09093861f68c9dd23a Mon Sep 17 00:00:00 2001 From: Jesse Wolfe Date: Fri, 23 Jul 2010 13:59:40 -0700 Subject: extlookup() is a builtin This patch promotes extlookup() to being a builtin function. It also adds test and makes some minor tweaks to the code. The behavior of extlookup has been left unchanged. --- lib/puppet/parser/functions/extlookup.rb | 173 +++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 lib/puppet/parser/functions/extlookup.rb (limited to 'lib/puppet') diff --git a/lib/puppet/parser/functions/extlookup.rb b/lib/puppet/parser/functions/extlookup.rb new file mode 100644 index 000000000..ee230e7ce --- /dev/null +++ b/lib/puppet/parser/functions/extlookup.rb @@ -0,0 +1,173 @@ +# Puppet External Data Sources +# +# This is a parser function to read data from external files, this version +# uses CSV files but the concept can easily be adjust for databases, yaml +# or any other queryable data source. +# +# The object of this is to make it obvious when it's being used, rather than +# magically loading data in when an module is loaded I prefer to look at the code +# and see statements like: +# +# $snmp_contact = extlookup("snmp_contact") +# +# The above snippet will load the snmp_contact value from CSV files, this in its +# own is useful but a common construct in puppet manifests is something like this: +# +# case $domain { +# "myclient.com": { $snmp_contact = "John Doe " } +# default: { $snmp_contact = "My Support " } +# } +# +# Over time there will be a lot of this kind of thing spread all over your manifests +# and adding an additional client involves grepping through manifests to find all the +# places where you have constructs like this. +# +# This is a data problem and shouldn't be handled in code, a using this function you +# can do just that. +# +# First you configure it in site.pp: +# $extlookup_datadir = "/etc/puppet/manifests/extdata" +# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"] +# +# The array tells the code how to resolve values, first it will try to find it in +# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv +# +# Now create the following data files in /etc/puppet/manifests/extdata +# +# domain_myclient.com.csv: +# snmp_contact,John Doe +# root_contact,support@%{domain} +# client_trusted_ips,192.168.1.130,192.168.10.0/24 +# +# common.csv: +# snmp_contact,My Support +# root_contact,support@my.com +# +# Now you can replace the case statement with the simple single line to achieve +# the exact same outcome: +# +# $snmp_contact = extlookup("snmp_contact") +# +# The obove code shows some other features, you can use any fact or variable that +# is in scope by simply using %{varname} in your data files, you can return arrays +# by just having multiple values in the csv after the initial variable name. +# +# In the event that a variable is nowhere to be found a critical error will be raised +# that will prevent your manifest from compiling, this is to avoid accidentally putting +# in empty values etc. You can however specify a default value: +# +# $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org") +# +# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in +# any data file. +# +# You can also specify an additional data file to search first before any others at use +# time, for example: +# +# $version = extlookup("rsyslog_version", "present", "packages") +# +# package{"rsyslog": ensure => $version } +# +# This will look for a version configured in packages.csv and then in the rest as configured +# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind +# of use case makes puppet a lot nicer for managing large amounts of packages since you do not +# need to edit a load of manifests to do simple things like adjust a desired version number. +# +# For more information on installing and writing your own custom functions see: +# http://docs.puppetlabs.com/guides/custom_functions.html +# +# For further help contact Volcane on #puppet +require 'csv' + +module Puppet::Parser::Functions + newfunction(:extlookup, :type => :rvalue) do |args| + key = args[0] + + default = args[1] + datafile = args[2] + + raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3 + + extlookup_datadir = lookupvar('extlookup_datadir') + extlookup_precedence = Array.new + + # precedence values can have variables embedded in them + # in the form %{fqdn}, you could for example do + # + # $extlookup_precedence = ["hosts/%{fqdn}", "common"] + # + # this will result in /path/to/extdata/hosts/your.box.com.csv + # being searched. + # + # we parse the precedence here because the best place to specify + # it would be in site.pp but site.pp is only evaluated at startup + # so $fqdn etc would have no meaning there, this way it gets evaluated + # each run and has access to the right variables for that run + lookupvar('extlookup_precedence').each do |prec| + while prec =~ /%\{(.+?)\}/ + prec.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + extlookup_precedence << prec + end + + + datafiles = Array.new + + # if we got a custom data file, put it first in the array of search files + if datafile != "" + datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv") + end + + extlookup_precedence.each do |d| + datafiles << extlookup_datadir + "/#{d}.csv" + end + + desired = nil + + datafiles.each do |file| + parser = Puppet::Parser::Parser.new(environment) + parser.watch_file(file) if File.exists?(file) + + if desired.nil? + if File.exists?(file) + result = CSV.read(file).find_all do |r| + r[0] == key + end + + + # return just the single result if theres just one, + # else take all the fields in the csv and build an array + if result.length > 0 + if result[0].length == 2 + val = result[0][1].to_s + + # parse %{}'s in the CSV into local variables using lookupvar() + while val =~ /%\{(.+?)\}/ + val.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + desired = val + elsif result[0].length > 1 + length = result[0].length + cells = result[0][1,length] + + # Individual cells in a CSV result are a weird data type and throws + # puppets yaml parsing, so just map it all to plain old strings + desired = cells.map do |c| + # parse %{}'s in the CSV into local variables using lookupvar() + while c =~ /%\{(.+?)\}/ + c.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + c.to_s + end + end + end + end + end + end + + desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" + end +end -- cgit From 67bdf89ac91a753171ad354b070546c62e09ce1a Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 15:26:01 +0200 Subject: Fix #4348 - Puppet doc single manifest broken The refactoring of using environment instances instead of strings for initializing the parser, rdoc wasn't updated, thus was unable to initialize the parser. Signed-off-by: Brice Figureau --- lib/puppet/util/rdoc.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/util/rdoc.rb b/lib/puppet/util/rdoc.rb index 4a80b069b..085d8ec93 100644 --- a/lib/puppet/util/rdoc.rb +++ b/lib/puppet/util/rdoc.rb @@ -41,7 +41,7 @@ module Puppet::Util::RDoc def manifestdoc(files) Puppet[:ignoreimport] = true files.select { |f| FileTest.file?(f) }.each do |f| - parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment]) + parser = Puppet::Parser::Parser.new(Puppet::Node::Environment.new(Puppet[:environment])) parser.file = f ast = parser.parse output(f, ast) -- cgit From 760e418d254a8d2198d2c6eb466d783a5930ef47 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sun, 25 Jul 2010 22:14:31 -0700 Subject: Fix #4349 - Parsing with ignoreimport=true was always loading site.pp With the type collection refactoring, when accessing a fresh collection puppet tries to import the site.pp manifest (perfrom_initial_import). In the case of puppetdoc, we are parsing site.pp by ourselves, so we ended parsing it twice, resulting in an "import loop detected" error. Signed-off-by: Brice Figureau --- lib/puppet/resource/type_collection.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/puppet') diff --git a/lib/puppet/resource/type_collection.rb b/lib/puppet/resource/type_collection.rb index 6a933362f..90b6df9c1 100644 --- a/lib/puppet/resource/type_collection.rb +++ b/lib/puppet/resource/type_collection.rb @@ -153,6 +153,7 @@ class Puppet::Resource::TypeCollection end def perform_initial_import + return if Puppet.settings[:ignoreimport] parser = Puppet::Parser::Parser.new(environment) if code = Puppet.settings.uninterpolated_value(:code, environment.to_s) and code != "" parser.string = code -- cgit From 7d42f77b83a7597a879a0fc2080c18b7c76c6fc3 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 17:42:42 +0200 Subject: JRuby doesn't implement Process.maxgroups So let's not call it :) Signed-off-by: Brice Figureau --- lib/puppet/util/monkey_patches.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib/puppet') diff --git a/lib/puppet/util/monkey_patches.rb b/lib/puppet/util/monkey_patches.rb index e035afd9f..9664ff310 100644 --- a/lib/puppet/util/monkey_patches.rb +++ b/lib/puppet/util/monkey_patches.rb @@ -1,4 +1,7 @@ -Process.maxgroups = 1024 + +unless defined? JRUBY_VERSION + Process.maxgroups = 1024 +end module RDoc def self.caller(skip=nil) -- cgit From 3163932f70e824906d48edc9665b4017a4669797 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 19:04:44 +0200 Subject: Fix #4244 - Cached Attributes is not thread safe The underlying hash is not protected and thus two threads accessing the cached value at the same time and one expiring the value can result in a race condition. This patch synchronizes the access to the value_cache underlying hash. Signed-off-by: Brice Figureau --- lib/puppet/util/cacher.rb | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/util/cacher.rb b/lib/puppet/util/cacher.rb index 8785c694f..3dddec0d4 100644 --- a/lib/puppet/util/cacher.rb +++ b/lib/puppet/util/cacher.rb @@ -1,3 +1,5 @@ +require 'monitor' + module Puppet::Util::Cacher module Expirer attr_reader :timestamp @@ -49,7 +51,7 @@ module Puppet::Util::Cacher define_method(name.to_s + "=") do |value| # Make sure the cache timestamp is set cache_timestamp - value_cache[name] = value + value_cache.synchronize { value_cache[name] = value } end if ttl = options[:ttl] @@ -70,6 +72,7 @@ module Puppet::Util::Cacher # Methods that get added to instances. module InstanceMethods + def expire # Only expire if we have an expirer. This is # mostly so that we can comfortably handle cases @@ -92,15 +95,17 @@ module Puppet::Util::Cacher end def cached_value(name) - # Allow a nil expirer, in which case we regenerate the value every time. - if expired_by_expirer?(name) - value_cache.clear - @cache_timestamp = Time.now - elsif expired_by_ttl?(name) - value_cache.delete(name) + value_cache.synchronize do + # Allow a nil expirer, in which case we regenerate the value every time. + if expired_by_expirer?(name) + value_cache.clear + @cache_timestamp = Time.now + elsif expired_by_ttl?(name) + value_cache.delete(name) + end + value_cache[name] = send("init_#{name}") unless value_cache.include?(name) + value_cache[name] end - value_cache[name] = send("init_#{name}") unless value_cache.include?(name) - value_cache[name] end def expired_by_expirer?(name) @@ -121,7 +126,7 @@ module Puppet::Util::Cacher end def value_cache - @value_cache ||= {} + @value_cache ||= {}.extend(MonitorMixin) end end end -- cgit From 4065e81f76020c29ed8ccfc1cf6c481aa202cb23 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 18:09:33 +0200 Subject: Fix race condition in rack autoloading of request/response Ruby autoloader seems to not be thread-safe. Since rack uses it to lazily load the Rack::Request and Rack::Response classes, on jruby it fails if the first compilation is done with multiple concurrent threads. Signed-off-by: Brice Figureau --- lib/puppet/network/http/rack.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/puppet') diff --git a/lib/puppet/network/http/rack.rb b/lib/puppet/network/http/rack.rb index c18a07559..5b4ef7e1c 100644 --- a/lib/puppet/network/http/rack.rb +++ b/lib/puppet/network/http/rack.rb @@ -1,3 +1,6 @@ +require 'rack' +require 'rack/request' +require 'rack/response' require 'puppet/network/http' require 'puppet/network/http/rack/rest' -- cgit From ef9a4a6df163c370a524d9ab76453c96bb99e8a4 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 19:19:04 +0200 Subject: Fix #4245 - default insertion of ACL is not thread safe This can happen under jruby with native threads. Signed-off-by: Brice Figureau --- lib/puppet/network/rest_authconfig.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib/puppet') diff --git a/lib/puppet/network/rest_authconfig.rb b/lib/puppet/network/rest_authconfig.rb index 38a8f9a09..82d5a9de2 100644 --- a/lib/puppet/network/rest_authconfig.rb +++ b/lib/puppet/network/rest_authconfig.rb @@ -3,6 +3,7 @@ require 'puppet/network/authconfig' module Puppet class Network::RestAuthConfig < Network::AuthConfig + extend MonitorMixin attr_accessor :rights DEFAULT_ACL = [ @@ -20,9 +21,11 @@ module Puppet ] def self.main - add_acl = @main.nil? - super - @main.insert_default_acl if add_acl and !@main.exists? + synchronize do + add_acl = @main.nil? + super + @main.insert_default_acl if add_acl and !@main.exists? + end @main end -- cgit