From 64386cf185c6d99ab5639ca8b2e0f0f4c47021af Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 20 Jul 2010 15:01:59 +1000 Subject: Fixed Indirection reference --- lib/puppet/reference/indirection.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/reference/indirection.rb b/lib/puppet/reference/indirection.rb index 549299b36..d14510c16 100644 --- a/lib/puppet/reference/indirection.rb +++ b/lib/puppet/reference/indirection.rb @@ -1,5 +1,5 @@ require 'puppet/indirector/indirection' -require 'puppet/checksum' +require 'puppet/util/checksums' require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' -- cgit From a0a63c3735f53cbf895a9e30e6a2c1dfd2ae4c4e Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 20 Jul 2010 15:25:05 +1000 Subject: Fixed network and indirection reference --- lib/puppet/rails/host.rb | 1 + lib/puppet/reference/network.rb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb index 6c008bccf..986cebd0a 100644 --- a/lib/puppet/rails/host.rb +++ b/lib/puppet/rails/host.rb @@ -1,3 +1,4 @@ +require 'puppet/rails' require 'puppet/rails/resource' require 'puppet/rails/fact_name' require 'puppet/rails/source_file' diff --git a/lib/puppet/reference/network.rb b/lib/puppet/reference/network.rb index a3bff656b..15e4f6769 100644 --- a/lib/puppet/reference/network.rb +++ b/lib/puppet/reference/network.rb @@ -1,3 +1,5 @@ +require 'puppet/network/handler' + network = Puppet::Util::Reference.newreference :network, :depth => 2, :doc => "Available network handlers and clients" do ret = "" Puppet::Network::Handler.subclasses.sort { |a,b| a.to_s <=> b.to_s }.each do |name| -- cgit From 865282ae7b9332fdbdfa51b2814755b8a13d244b Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Wed, 21 Jul 2010 09:50:21 +1000 Subject: Fixed example config.ru --- ext/rack/files/config.ru | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/rack/files/config.ru b/ext/rack/files/config.ru index 5f0834a7d..995a9f8c5 100644 --- a/ext/rack/files/config.ru +++ b/ext/rack/files/config.ru @@ -11,8 +11,7 @@ require 'puppet' # ARGV << "--debug" ARGV << "--rack" -require 'puppet/application/puppetmasterd' +require 'puppet/application/master' # we're usually running inside a Rack::Builder.new {} block, # therefore we need to call run *here*. -run Puppet::Application[:puppetmasterd].run - +run Puppet::Application[:master].run -- cgit From 59a23d69c20a8ab1f7dc9ff1d109305f9027641c Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sat, 24 Jul 2010 14:48:13 -0700 Subject: Fix for #3382 -- Empty classes as graph placeholders As Brice discovered, the problem was that we simply ignored empty classes in the graph when determining application order. This patch instead replaces them with a resource of a new type which we've frequently noted the (internal) need for: a whit, the smallest possible resource, which has no properties or other semantics apart from its existence and its name. This resource then ensures application order through the normal mechanisms. --- lib/puppet/simple_graph.rb | 9 ++++----- lib/puppet/type/whit.rb | 7 +++++++ spec/unit/simple_graph_spec.rb | 14 ++++++++++++-- 3 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 lib/puppet/type/whit.rb diff --git a/lib/puppet/simple_graph.rb b/lib/puppet/simple_graph.rb index 945b1beb3..55b39fadf 100644 --- a/lib/puppet/simple_graph.rb +++ b/lib/puppet/simple_graph.rb @@ -318,16 +318,15 @@ class Puppet::SimpleGraph # to container leaves, but that would result in many more # relationships. stage_class = Puppet::Type.type(:stage) + whit_class = Puppet::Type.type(:whit) containers = other.topsort.find_all { |v| (v.is_a?(type) or v.is_a?(stage_class)) and vertex?(v) } containers.each do |container| # Get the list of children from the other graph. children = other.adjacent(container, :direction => :out) - # Just remove the container if it's empty. - if children.empty? - remove_vertex!(container) - next - end + # MQR TODO: Luke suggests that it should be possible to refactor the system so that + # container nodes are retained, thus obviating the need for the whit. + children = [whit_class.new(:name => container.name, :catalog => other)] if children.empty? # First create new edges for each of the :in edges [:in, :out].each do |dir| diff --git a/lib/puppet/type/whit.rb b/lib/puppet/type/whit.rb new file mode 100644 index 000000000..6e5ba9eab --- /dev/null +++ b/lib/puppet/type/whit.rb @@ -0,0 +1,7 @@ +Puppet::Type.newtype(:whit) do + desc "The smallest possible resource type, for when you need a resource and naught else." + + newparam :name do + desc "The name of the whit, because it must have one." + end +end diff --git a/spec/unit/simple_graph_spec.rb b/spec/unit/simple_graph_spec.rb index 83e674139..2ca8888c5 100755 --- a/spec/unit/simple_graph_spec.rb +++ b/spec/unit/simple_graph_spec.rb @@ -439,7 +439,8 @@ describe Puppet::SimpleGraph do @middle = Container.new("middle", ["e", "f", @two]) @top = Container.new("top", ["g", "h", @middle, @one, @three]) @empty = Container.new("empty", []) - + + @whit = Puppet::Type.type(:whit) @stage = Puppet::Type.type(:stage).new(:name => "foo") @contgraph = @top.to_graph @@ -499,8 +500,17 @@ describe Puppet::SimpleGraph do end end + it "should contain a whit-resource to mark the place held by the empty container" do + @depgraph.vertices.find_all { |v| v.is_a?(@whit) }.length.should == 1 + end + + it "should replace edges to empty containers with edges to their residual whit" do + emptys_whit = @depgraph.vertices.find_all { |v| v.is_a?(@whit) }.first + @depgraph.should be_edge("c", emptys_whit) + end + it "should no longer contain anything but the non-container objects" do - @depgraph.vertices.find_all { |v| ! v.is_a?(String) }.should be_empty + @depgraph.vertices.find_all { |v| ! v.is_a?(String) and ! v.is_a?(@whit)}.should be_empty end it "should copy labels" do -- cgit From 9778f2a47922a66e59d571c1c98552223a817ce1 Mon Sep 17 00:00:00 2001 From: Nick Lewis Date: Fri, 23 Jul 2010 17:04:22 -0700 Subject: [#4242] Fixed recursion due to parents including their children Resources mark themselves as evaluated to prevent being evaluated again. Unfortunately, they were not marking themselves until after they had finished being completely evaluated. Thus, there was nothing actually stopping recursive evaluations. This patch just makes resources mark themselves as evaluated when they start evaluating, and adds tests. The original setting of evaluated was done in an ensure block, so this doesn't change the behavior of a resource which fails to evaluate. The only places evaluated? is checked aren't affected by this change, as they wouldn't want to evaluate it when it's already being evaluated anyway. --- lib/puppet/parser/resource.rb | 3 +- spec/unit/parser/functions/include_spec.rb | 50 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 spec/unit/parser/functions/include_spec.rb diff --git a/lib/puppet/parser/resource.rb b/lib/puppet/parser/resource.rb index c956a1106..3c451929e 100644 --- a/lib/puppet/parser/resource.rb +++ b/lib/puppet/parser/resource.rb @@ -64,6 +64,7 @@ class Puppet::Parser::Resource < Puppet::Resource # Retrieve the associated definition and evaluate it. def evaluate + @evaluated = true if klass = resource_type and ! builtin_type? finish return klass.evaluate_code(self) @@ -72,8 +73,6 @@ class Puppet::Parser::Resource < Puppet::Resource else self.fail "Cannot find definition #{type}" end - ensure - @evaluated = true end # Mark this resource as both exported and virtual, diff --git a/spec/unit/parser/functions/include_spec.rb b/spec/unit/parser/functions/include_spec.rb new file mode 100644 index 000000000..4954ac63a --- /dev/null +++ b/spec/unit/parser/functions/include_spec.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../../spec_helper' + +describe "the 'include' function" do + + before :each do + Puppet::Node::Environment.stubs(:current).returns(nil) + @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("foo")) + @scope = Puppet::Parser::Scope.new(:compiler => @compiler) + end + + it "should exist" do + Puppet::Parser::Functions.function("include").should == "function_include" + end + + it "should include a single class" do + inc = "foo" + @compiler.expects(:evaluate_classes).with {|klasses,parser,lazy| klasses == [inc]}.returns([inc]) + @scope.function_include("foo") + end + + it "should include multiple classes" do + inc = ["foo","bar"] + @compiler.expects(:evaluate_classes).with {|klasses,parser,lazy| klasses == inc}.returns(inc) + @scope.function_include(["foo","bar"]) + end + + it "should not lazily evaluate the included class" do + @compiler.expects(:evaluate_classes).with {|klasses,parser,lazy| lazy == false}.returns("foo") + @scope.function_include("foo") + end + + it "should allow a parent to include its child" do + @parent_type = Puppet::Resource::Type.new(:hostclass, "parent") + @parent_resource = Puppet::Parser::Resource.new(:hostclass, "parent", :scope => @scope) + @subscope = @parent_type.subscope(@scope,@parent_resource) + @scope.environment.known_resource_types.stubs(:find_hostclass).with{|nses,name| name.downcase == "parent"}.returns(@parent_type) + + @type = Puppet::Resource::Type.new(:hostclass, "foo") + @type.stubs(:parent_scope).returns(@subscope) + @type.parent = "parent" + @resource = Puppet::Parser::Resource.new(:hostclass, "foo", :scope => @subscope) + @resource.stubs(:resource_type).returns(@type) + @scope.environment.known_resource_types.stubs(:find_hostclass).with{|nses,name| name.downcase == "foo"}.returns(@parent_type) + Puppet::Resource.stubs(:new).returns(@resource) + Puppet::Parser::Resource.stubs(:new).returns(@resource) + lambda { @subscope.function_include("foo") }.should_not raise_error + end +end -- cgit From 067a46db27a746a680f8afbb32bb5317ce7a7056 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sat, 24 Jul 2010 21:55:42 -0700 Subject: Temporary tweak to tests for #4242 The tests introduced with the fix for #4242 have isolation issues apparently due to the creation of a stage resource which is global and persistent. This patch stubs the creation, marks the one test which is thus invalidated pending, and adds comments noting the problem. --- spec/unit/parser/functions/include_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/unit/parser/functions/include_spec.rb b/spec/unit/parser/functions/include_spec.rb index 4954ac63a..e5f051906 100644 --- a/spec/unit/parser/functions/include_spec.rb +++ b/spec/unit/parser/functions/include_spec.rb @@ -8,6 +8,10 @@ describe "the 'include' function" do Puppet::Node::Environment.stubs(:current).returns(nil) @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("foo")) @scope = Puppet::Parser::Scope.new(:compiler => @compiler) + # MQR TODO: Without the following stub these tests cause hundreds of spurious errors in + # subsequent tests. With it, there are no spurious failures and all but one + # of the tests (marked pending, bellow) fail. This needs a better solution. + Puppet::Parser::Resource.stubs(:new).with('stage', :main, :scope => @scope).returns 'foo' end it "should exist" do @@ -32,6 +36,7 @@ describe "the 'include' function" do end it "should allow a parent to include its child" do + pending "Resolution of MQR TODO item, above" @parent_type = Puppet::Resource::Type.new(:hostclass, "parent") @parent_resource = Puppet::Parser::Resource.new(:hostclass, "parent", :scope => @scope) @subscope = @parent_type.subscope(@scope,@parent_resource) -- cgit From 28bb1950cf040eaf5196e39f43b3a9d0d61175a7 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 16 Jul 2010 14:42:01 +1000 Subject: Fixed yumrepo type deprecation wanring ` --- lib/puppet/type/yumrepo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/type/yumrepo.rb b/lib/puppet/type/yumrepo.rb index e0ed3ad37..5d86b1c75 100644 --- a/lib/puppet/type/yumrepo.rb +++ b/lib/puppet/type/yumrepo.rb @@ -85,7 +85,7 @@ module Puppet clear inifile.each_section do |s| next if s.name == "main" - obj = create(:name => s.name, :audit => check) + obj = new(:name => s.name, :audit => check) current_values = obj.retrieve obj.eachproperty do |property| if current_values[property].nil? -- cgit From 7ad7eb12fade4695abf08392e80ca57003353140 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Wed, 21 Jul 2010 23:10:46 +0200 Subject: Fix #4286 - rename puppetdoc global module to __site__ < and > might be invalid or borderline chars to use for a file name or an url. This patch changes those characters to __. Signed-off-by: Brice Figureau --- lib/puppet/util/rdoc/parser.rb | 8 +++++--- spec/unit/util/rdoc/parser_spec.rb | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/puppet/util/rdoc/parser.rb b/lib/puppet/util/rdoc/parser.rb index 573d1766f..63df38ab9 100644 --- a/lib/puppet/util/rdoc/parser.rb +++ b/lib/puppet/util/rdoc/parser.rb @@ -15,6 +15,8 @@ module RDoc class Parser extend ParserFactory + SITE = "__site__" + attr_accessor :ast, :input_file_name, :top_level # parser registration into RDoc @@ -74,7 +76,7 @@ class Parser # split_module tries to find if +path+ belongs to the module path # if it does, it returns the module name, otherwise if we are sure - # it is part of the global manifest path, "" is returned. + # it is part of the global manifest path, "__site__" is returned. # And finally if this path couldn't be mapped anywhere, nil is returned. def split_module(path) # find a module @@ -105,7 +107,7 @@ class Parser end # we are under a global manifests Puppet.debug "rdoc: global manifests" - "" + SITE end # create documentation for the top level +container+ @@ -128,7 +130,7 @@ class Parser Puppet.debug "rdoc: scanning for #{name}" container.module_name = name - container.global=true if name == "" + container.global=true if name == SITE @stats.num_modules += 1 container, name = get_class_or_module(container,name) diff --git a/spec/unit/util/rdoc/parser_spec.rb b/spec/unit/util/rdoc/parser_spec.rb index a9c8190a6..79195e657 100755 --- a/spec/unit/util/rdoc/parser_spec.rb +++ b/spec/unit/util/rdoc/parser_spec.rb @@ -85,8 +85,8 @@ describe RDoc::Parser do @parser.scan_top_level(@topcontainer) end - it "should set the module as global if we parse the global manifests (ie module)" do - @parser.stubs(:split_module).returns("") + it "should set the module as global if we parse the global manifests (ie __site__ module)" do + @parser.stubs(:split_module).returns(RDoc::Parser::SITE) @parser.stubs(:parse_elements) @topcontainer.expects(:global=).with(true) @@ -131,7 +131,7 @@ describe RDoc::Parser do it "should return for manifests not under module path" do File.stubs(:expand_path).returns("/path/to/manifests/init.pp") File.stubs(:identical?).returns(false) - @parser.split_module("/path/to/manifests/init.pp").should == "" + @parser.split_module("/path/to/manifests/init.pp").should == RDoc::Parser::SITE end end -- cgit From 63ec207cc88828b8b0ad421c7fcdd8a5715e7fd3 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sat, 24 Jul 2010 10:35:21 -0700 Subject: Minimal fix for #4297, with notes for follow-up In retrospect it appears that the fix for #4270 was incomplete and somewhat off target. This patch fixes the one demonstrably incorrect part (the namespace) and adds a comment outlining what remains to be done to clean up the code; these additional changes, while needed for maintanability, are inappropriate for a quick turnaround crucial bug fix release such as 2.6.1, at which this patch is targeted. --- lib/puppet/resource/type.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/puppet/resource/type.rb b/lib/puppet/resource/type.rb index 85c0979c1..6296d26e5 100644 --- a/lib/puppet/resource/type.rb +++ b/lib/puppet/resource/type.rb @@ -174,12 +174,29 @@ class Puppet::Resource::Type @name.is_a?(Regexp) end + # MQR TODO: + # + # The change(s) introduced by the fix for #4270 are mostly silly & should be + # removed, though we didn't realize it at the time. If it can be established/ + # ensured that nodes never call parent_type and that resource_types are always + # (as they should be) members of exactly one resource_type_collection the + # following method could / should be replaced with: + # + # def parent_type + # @parent_type ||= parent && ( + # resource_type_collection.find_or_load([name],parent,type.to_sym) || + # fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{resource_type_collection.environment}" + # ) + # end + # + # ...and then the rest of the changes around passing in scope reverted. + # def parent_type(scope = nil) return nil unless parent unless @parent_type raise "Must pass scope to parent_type when called first time" unless scope - unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", scope.namespaces, parent) + unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", [name], parent) fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{scope.environment}" end end -- cgit From 23830504dfeb48b2d162e44b84b6f9dfa97e4e7e Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Thu, 22 Jul 2010 20:36:24 +0200 Subject: Fix #4302 - Compilation speed regression compared to 2.6 Each time the compiler was accessing the loaded types, we were checking if the manifests had changed. This incurred a large performance cost compared to 0.25 and introduced race conditions if manifests changed while a thread was in the middle of a compilation. This tentative fix, based on Brice's, makes sure each thread will get access to the same loaded types collection for the durration of a compilation, even if the manifests change. We now only check for changed files at the start of a compilation or if the environment changes, and we maintain a per environment thread lock so that only one thread at a time can be reloading any particular environment (and the need-check is done inside the synchronize block so that only the first will actually load it). As long as the manifests don't change, the threads will share the same collection, so there is only duplication in memory for a brief window surrounding a change. Signed-off-by: Brice Figureau Second-author: Markus Roberts --- lib/puppet/node/environment.rb | 20 +++++++++++++++----- lib/puppet/parser/compiler.rb | 5 +++++ spec/unit/node/environment_spec.rb | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 762599cff..3f67474f9 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -1,4 +1,5 @@ require 'puppet/util/cacher' +require 'monitor' # Just define it, so this class has fewer load dependencies. class Puppet::Node @@ -67,14 +68,23 @@ class Puppet::Node::Environment def initialize(name) @name = name + extend MonitorMixin end def known_resource_types - if @known_resource_types.nil? or @known_resource_types.stale? - @known_resource_types = Puppet::Resource::TypeCollection.new(self) - @known_resource_types.perform_initial_import - end - @known_resource_types + # This makes use of short circuit evaluation to get the right thread-safe + # per environment semantics with an efficient most common cases; we almost + # always just return our thread's known-resource types. Only at the start + # of a compilation (after our thread var has been set to nil) or when the + # environment has changed do we delve deeper. + Thread.current[:known_resource_types] = nil if (krt = Thread.current[:known_resource_types]) && krt.environment != self + Thread.current[:known_resource_types] ||= synchronize { + if @known_resource_types.nil? or @known_resource_types.stale? + @known_resource_types = Puppet::Resource::TypeCollection.new(self) + @known_resource_types.perform_initial_import + end + @known_resource_types + } end def module(name) diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index a901c0dd6..760d5a75a 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -15,6 +15,11 @@ class Puppet::Parser::Compiler include Puppet::Resource::TypeCollectionHelper def self.compile(node) + # At the start of a new compile we don't assume anything about + # known_resouce_types; we'll get these from the environment and + # cache them in a thread variable for the duration of the + # compilation. + Thread.current[:known_resource_types] = nil new(node).compile.to_resource rescue => detail puts detail.backtrace if Puppet[:trace] diff --git a/spec/unit/node/environment_spec.rb b/spec/unit/node/environment_spec.rb index b400865a2..6edcce56c 100755 --- a/spec/unit/node/environment_spec.rb +++ b/spec/unit/node/environment_spec.rb @@ -53,6 +53,7 @@ describe Puppet::Node::Environment do @env = Puppet::Node::Environment.new("dev") @collection = Puppet::Resource::TypeCollection.new(@env) @collection.stubs(:perform_initial_import) + Thread.current[:known_resource_types] = nil end it "should create a resource type collection if none exists" do @@ -71,13 +72,41 @@ describe Puppet::Node::Environment do @env.known_resource_types end - it "should create and return a new collection rather than returning a stale collection" do - @env.known_resource_types.expects(:stale?).returns true + it "should return the same collection even if stale if it's the same thread" do + Puppet::Resource::TypeCollection.stubs(:new).returns @collection + @env.known_resource_types.stubs(:stale?).returns true - Puppet::Resource::TypeCollection.expects(:new).returns @collection + @env.known_resource_types.should equal(@collection) + end + + it "should return the current thread associated collection if there is one" do + Thread.current[:known_resource_types] = @collection @env.known_resource_types.should equal(@collection) end + + it "should give to all threads the same collection if it didn't change" do + Puppet::Resource::TypeCollection.expects(:new).with(@env).returns @collection + @env.known_resource_types + + t = Thread.new { + @env.known_resource_types.should equal(@collection) + } + t.join + end + + it "should give to new threads a new collection if it isn't stale" do + Puppet::Resource::TypeCollection.expects(:new).with(@env).returns @collection + @env.known_resource_types.expects(:stale?).returns(true) + + Puppet::Resource::TypeCollection.expects(:new).returns @collection + + t = Thread.new { + @env.known_resource_types.should equal(@collection) + } + t.join + end + end [:modulepath, :manifestdir].each do |setting| -- cgit From 1d494a3104e9794cc09ba27c701ced68a74fa398 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Fri, 23 Jul 2010 08:47:48 -0700 Subject: Tweak to fix for #4302--dangling ref to known_resource_types Since we were clearing the thread variable containing the compiler's reference to it's environment's known resource types at the start of each compile the reference remaining at the end of a compilation could never be used and was thus just garbage that we were arbitrarily retaining. This patch moves the clearing of the thread var to the _end_ of compilation so that it's always nil except in the middle of a compile. This raises an interesting question; should the ref just live on the compiler object and we could dispense with the thread-var? It might require things that now only know about the environment to need a ref to the compiler and introduce other thread issues (e.g. we might just end up needing a :current_compiler thread variable, for no net gain in simplicity). --- lib/puppet/parser/compiler.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 760d5a75a..61bb13cb6 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -15,16 +15,15 @@ class Puppet::Parser::Compiler include Puppet::Resource::TypeCollectionHelper def self.compile(node) - # At the start of a new compile we don't assume anything about - # known_resouce_types; we'll get these from the environment and - # cache them in a thread variable for the duration of the - # compilation. - Thread.current[:known_resource_types] = nil new(node).compile.to_resource rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "#{detail} on node #{node.name}" - end + ensure + # We get these from the environment and only cache them in a thread + # variable for the duration of the compilation. + Thread.current[:known_resource_types] = nil + end attr_reader :node, :facts, :collections, :catalog, :node_scope, :resources, :relationships -- cgit From 000fd1e83782c70fc9d9b032b52d96800cab2121 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Fri, 23 Jul 2010 11:31:40 -0700 Subject: Fix for #4303 -- reverting to old escaping in '-strings Single quoted used to allow escape on single quotes and pass all other characters through without comment; now the do again. --- lib/puppet/parser/lexer.rb | 9 ++++----- spec/unit/parser/lexer_spec.rb | 3 +++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb index 1e10ff96c..aa04f17a0 100644 --- a/lib/puppet/parser/lexer.rb +++ b/lib/puppet/parser/lexer.rb @@ -221,7 +221,7 @@ class Puppet::Parser::Lexer TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true TOKENS.add_token :SQUOTE, "'" do |lexer, value| - [TOKENS[:STRING], lexer.slurpstring(value).first ] + [TOKENS[:STRING], lexer.slurpstring(value,["'"],:ignore_invalid_esapes).first ] end DQ_initial_token_types = {'$' => :DQPRE,'"' => :STRING} @@ -517,8 +517,7 @@ class Puppet::Parser::Lexer # we've encountered the start of a string... # slurp in the rest of the string and return it - Valid_escapes_in_strings = %w{ \\ $ ' " n t s }+["\n"] - def slurpstring(terminators) + def slurpstring(terminators,escapes=%w{ \\ $ ' " n t s }+["\n"],ignore_invalid_escapes=false) # we search for the next quote that isn't preceded by a # backslash; the caret is there to match empty strings str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'" @@ -529,10 +528,10 @@ class Puppet::Parser::Lexer when 't'; "\t" when 's'; " " else - if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'") + if escapes.include? ch ch else - Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" + Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" unless ignore_invalid_escapes "\\#{ch}" end end diff --git a/spec/unit/parser/lexer_spec.rb b/spec/unit/parser/lexer_spec.rb index d3d2a0a31..a85d1b257 100755 --- a/spec/unit/parser/lexer_spec.rb +++ b/spec/unit/parser/lexer_spec.rb @@ -410,6 +410,9 @@ describe Puppet::Parser::Lexer,"when lexing strings" do %q{'single quoted string')} => [[:STRING,'single quoted string']], %q{"double quoted string"} => [[:STRING,'double quoted string']], %q{'single quoted string with an escaped "\\'"'} => [[:STRING,'single quoted string with an escaped "\'"']], + %q{'single quoted string with an escaped "\$"'} => [[:STRING,'single quoted string with an escaped "\$"']], + %q{'single quoted string with an escaped "\."'} => [[:STRING,'single quoted string with an escaped "\."']], + %q{'single quoted string with an escaped "\\\\"'} => [[:STRING,'single quoted string with an escaped "\\\\"']], %q{"string with an escaped '\\"'"} => [[:STRING,"string with an escaped '\"'"]], %q{"string with an escaped '\\$'"} => [[:STRING,"string with an escaped '$'"]], %q{"string with $v (but no braces)"} => [[:DQPRE,"string with "],[:VARIABLE,'v'],[:DQPOST,' (but no braces)']], -- cgit From 636079f114034f5dced1772da206d1b1b1a537ed Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 23 Jul 2010 02:32:50 +1000 Subject: Fixed #4304 - Changed logging level for auto import message --- lib/puppet/parser/type_loader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 35ad49593..09aa636e1 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -88,7 +88,7 @@ class Puppet::Parser::TypeLoader nil end if result = yield(filename) - Puppet.info "Automatically imported #{name} from #{filename} into #{environment}" + Puppet.debug "Automatically imported #{name} from #{filename} into #{environment}" result.module_name = modname if modname and result.respond_to?(:module_name=) return result end -- cgit From 9569136329f87eeb102e589a3d5e9c26d8632aad Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Fri, 23 Jul 2010 09:50:29 -0700 Subject: Fix for 4314 -- Need to allow '-' in class name for refs I almost changed this to correspond to the lexer pattern ([a-z0-9][-\w]*) but that isn't quite right either (it would reintroduce the '::' problem). Another option I considered was [^\[]+, but that has it's own problems. In the end I just made the minimal change, adding '-' to the acceptable characters. --- lib/puppet/resource/catalog.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb index 4b4342d11..a8668d844 100644 --- a/lib/puppet/resource/catalog.rb +++ b/lib/puppet/resource/catalog.rb @@ -57,7 +57,7 @@ class Puppet::Resource::Catalog < Puppet::SimpleGraph end def title_key_for_ref( ref ) - ref =~ /^([\w:]+)\[(.*)\]$/m + ref =~ /^([-\w:]+)\[(.*)\]$/m [$1, $2] end -- cgit From c047c8d38f5fbb8159a357f64d6c444d66407423 Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Fri, 16 Jul 2010 07:36:45 +0200 Subject: vim: added elsif --- ext/vim/syntax/puppet.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/vim/syntax/puppet.vim b/ext/vim/syntax/puppet.vim index 80cd91c6c..81efa92ba 100644 --- a/ext/vim/syntax/puppet.vim +++ b/ext/vim/syntax/puppet.vim @@ -68,7 +68,7 @@ syn match puppetNotVariable "\\$\w\+" contained syn match puppetNotVariable "\\${\w\+}" contained syn keyword puppetKeyword import inherits include -syn keyword puppetControl case default if else +syn keyword puppetControl case default if else elsif syn keyword puppetSpecial true false undef " comments last overriding everything else -- cgit From be2141a6e37fabe78b57be699f8ae9d84acdebcf Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Fri, 16 Jul 2010 13:03:54 +0200 Subject: vim: match collected resources. --- ext/vim/syntax/puppet.vim | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/vim/syntax/puppet.vim b/ext/vim/syntax/puppet.vim index 81efa92ba..b24cbf623 100644 --- a/ext/vim/syntax/puppet.vim +++ b/ext/vim/syntax/puppet.vim @@ -33,6 +33,7 @@ syn match puppetNodeRe "/.*/" contained "FIXME: "Foo-bar" doesn't get highlighted as expected, although "foo-bar" does. syn match puppetInstance "[A-Za-z0-9_-]\+\(::[A-Za-z0-9_-]\+\)*\s*{" contains=puppetTypeName,puppetTypeDefault syn match puppetInstance "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*[[{]" contains=puppetTypeName,puppetTypeDefault +syn match puppetInstance "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*<\?<|" contains=puppetTypeName,puppetTypeDefault syn match puppetTypeName "[a-z]\w*" contained syn match puppetTypeDefault "[A-Z]\w*" contained -- cgit From ac3a0d2d6c10f4a1616865d2c2e69b68a4bbcb3e Mon Sep 17 00:00:00 2001 From: Marc Fournier Date: Fri, 16 Jul 2010 08:22:30 +0200 Subject: vim: highlight default parameters in definition/classes --- ext/vim/syntax/puppet.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/vim/syntax/puppet.vim b/ext/vim/syntax/puppet.vim index b24cbf623..0025e2d1c 100644 --- a/ext/vim/syntax/puppet.vim +++ b/ext/vim/syntax/puppet.vim @@ -19,7 +19,7 @@ endif " match class/definition/node declarations syn region puppetDefine start="^\s*\(class\|define\|node\)\s" end="{" contains=puppetDefType,puppetDefName,puppetDefArguments,puppetNodeRe syn keyword puppetDefType class define node inherits contained -syn region puppetDefArguments start="(" end=")" contained contains=puppetArgument +syn region puppetDefArguments start="(" end=")" contained contains=puppetArgument,puppetString syn match puppetArgument "\w\+" contained syn match puppetArgument "\$\w\+" contained syn match puppetArgument "'[^']+'" contained -- cgit From 03313b81c696d12c756a5ff5adced5bed162bbfc Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Wed, 21 Jul 2010 22:56:58 +0200 Subject: Fix #4319 - source file url sent to the master is invalid We were sending an incorrect (containing a //) url for sourced file content since the file streaming patches. Depending on the webserver in front of puppet it could fail (for instance nginx+mongrel). This patch fixes the offending // in each sourced file urls. Signed-off-by: Brice Figureau --- lib/puppet/type/file/content.rb | 2 +- spec/unit/type/file/content_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/puppet/type/file/content.rb b/lib/puppet/type/file/content.rb index 74b380f55..0d0bb5571 100755 --- a/lib/puppet/type/file/content.rb +++ b/lib/puppet/type/file/content.rb @@ -184,7 +184,7 @@ module Puppet end def chunk_file_from_source(source_or_content) - request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path) + request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path.sub(/^\//,'')) connection = Puppet::Network::HttpPool.http_instance(source_or_content.server, source_or_content.port) connection.request_get(indirection2uri(request), add_accept_encoding({"Accept" => "raw"})) do |response| case response.code diff --git a/spec/unit/type/file/content_spec.rb b/spec/unit/type/file/content_spec.rb index e4b9f9bc6..a01a14a59 100755 --- a/spec/unit/type/file/content_spec.rb +++ b/spec/unit/type/file/content_spec.rb @@ -362,7 +362,7 @@ describe content do end it "should send the correct indirection uri" do - @conn.expects(:request_get).with { |uri,headers| uri == "/production/file_content//path/to/source" }.yields(@response) + @conn.expects(:request_get).with { |uri,headers| uri == "/production/file_content/path/to/source" }.yields(@response) @content.write(@fh) end -- cgit From 86b0882c8a772b2e84e8da0609ec911d085ac0fc Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sun, 25 Jul 2010 10:52:32 +1000 Subject: Fixed #4326 - Updated SUSE packaging --- conf/suse/client.init | 8 +-- conf/suse/fileserver.conf | 12 ++++ conf/suse/logrotate | 10 +++ conf/suse/puppet.conf | 25 +++++++ conf/suse/puppet.spec | 163 +++++++++++++++++++++++++++++++------------- conf/suse/puppet.suse.patch | 16 ----- conf/suse/ruby-env.patch | 17 +++++ conf/suse/server.init | 8 +-- 8 files changed, 186 insertions(+), 73 deletions(-) create mode 100644 conf/suse/fileserver.conf create mode 100644 conf/suse/logrotate create mode 100644 conf/suse/puppet.conf delete mode 100644 conf/suse/puppet.suse.patch create mode 100644 conf/suse/ruby-env.patch diff --git a/conf/suse/client.init b/conf/suse/client.init index d72b09b2c..71585cd7b 100644 --- a/conf/suse/client.init +++ b/conf/suse/client.init @@ -13,10 +13,10 @@ ### BEGIN INIT INFO # Provides: puppet -# Required-Start: $local_fs $network $syslog -# Should-Start: $null -# Required-Stop: $null -# Should-Stop: $null +# Required-Start: $local_fs $remote_fs $network $syslog +# Should-Start: puppet +# Required-Stop: $local_fs $remote_fs $network $syslog +# Should-Stop: puppet # Default-Start: 3 4 5 # Default-Stop: 0 1 2 6 # Short-Description: puppet diff --git a/conf/suse/fileserver.conf b/conf/suse/fileserver.conf new file mode 100644 index 000000000..67e387ca0 --- /dev/null +++ b/conf/suse/fileserver.conf @@ -0,0 +1,12 @@ +# This file consists of arbitrarily named sections/modules +# defining where files are served from and to whom + +# Define a section 'files' +# Adapt the allow/deny settings to your needs. Order +# for allow/deny does not matter, allow always takes precedence +# over deny +# [files] +# path /var/lib/puppet/files +# allow *.example.com +# deny *.evil.example.com +# allow 192.168.0.0/24 diff --git a/conf/suse/logrotate b/conf/suse/logrotate new file mode 100644 index 000000000..c3a4d437a --- /dev/null +++ b/conf/suse/logrotate @@ -0,0 +1,10 @@ +/var/log/puppet/*log { + missingok + notifempty + create 0644 puppet puppet + sharedscripts + postrotate + [ -e /etc/init.d/puppetmaster ] && /etc/init.d/puppetmaster condrestart >/dev/null 2>&1 || true + [ -e /etc/init.d/puppet ] && /etc/init.d/puppet reload > /dev/null 2>&1 || true + endscript +} diff --git a/conf/suse/puppet.conf b/conf/suse/puppet.conf new file mode 100644 index 000000000..47501a388 --- /dev/null +++ b/conf/suse/puppet.conf @@ -0,0 +1,25 @@ +[main] + # The Puppet log directory. + # The default value is '$vardir/log'. + logdir = /var/log/puppet + + # Where Puppet PID files are kept. + # The default value is '$vardir/run'. + rundir = /var/run/puppet + + # Where SSL certificates are kept. + # The default value is '$confdir/ssl'. + ssldir = $vardir/ssl + +[agent] + # The file in which puppetd stores a list of the classes + # associated with the retrieved configuratiion. Can be loaded in + # the separate ``puppet`` executable using the ``--loadclasses`` + # option. + # The default value is '$confdir/classes.txt'. + classfile = $vardir/classes.txt + + # Where puppetd caches the local configuration. An + # extension indicating the cache format is added automatically. + # The default value is '$confdir/localconfig'. + localconfig = $vardir/localconfig diff --git a/conf/suse/puppet.spec b/conf/suse/puppet.spec index 2a0e6e359..bd4b35885 100644 --- a/conf/suse/puppet.spec +++ b/conf/suse/puppet.spec @@ -1,31 +1,35 @@ %{!?ruby_sitelibdir: %define ruby_sitelibdir %(ruby -rrbconfig -e 'puts Config::CONFIG["sitelibdir"]')} %define pbuild %{_builddir}/%{name}-%{version} -%define suseconfdir conf/suse -%define confdir conf/redhat +%define confdir conf/suse Summary: A network tool for managing many disparate systems Name: puppet -Version: 0.25.4 +Version: 2.6.0 Release: 1%{?dist} License: GPL -Group: System Environment/Base +Group: Productivity/Networking/System -URL: http://reductivelabs.com/projects/puppet/ -Source: http://reductivelabs.com/downloads/puppet/%{name}-%{version}.tar.gz -Patch0: puppet.suse.patch +URL: http://puppetlabs.com/projects/puppet/ +Source0: http://puppetlabs.com/downloads/puppet/%{name}-%{version}.tar.gz +Source1: client.init +Source2: server.init +Patch0: ruby-env.patch + +PreReq: %{insserv_prereq} %{fillup_prereq} Requires: ruby >= 1.8.2 -Requires: facter >= 1.3.7 +Requires: facter >= 1.5 +Requires: cron BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildRequires: ruby >= 1.8.2 +BuildRequires: ruby >= 1.8.7 %description -Puppet lets you centrally manage every important aspect of your system using a -cross-platform specification language that manages all the separate elements -normally aggregated in different files, like users, cron jobs, and hosts, +Puppet lets you centrally manage every important aspect of your system using a +cross-platform specification language that manages all the separate elements +normally aggregated in different files, like users, cron jobs, and hosts, along with obviously discrete elements like packages, services, and files. %package server -Group: System Environment/Base +Group: Productivity/Networking/System Summary: Server for the puppet system management tool Requires: puppet = %{version}-%{release} @@ -45,45 +49,47 @@ done %install %{__install} -d -m0755 %{buildroot}%{_sbindir} %{__install} -d -m0755 %{buildroot}%{_bindir} +%{__install} -d -m0755 %{buildroot}%{_confdir} %{__install} -d -m0755 %{buildroot}%{ruby_sitelibdir} %{__install} -d -m0755 %{buildroot}%{_sysconfdir}/puppet/manifests %{__install} -d -m0755 %{buildroot}%{_docdir}/%{name}-%{version} %{__install} -d -m0755 %{buildroot}%{_localstatedir}/lib/puppet %{__install} -d -m0755 %{buildroot}%{_localstatedir}/run/puppet %{__install} -d -m0755 %{buildroot}%{_localstatedir}/log/puppet -%{__install} -Dp -m0755 %{pbuild}/bin/* %{buildroot}%{_sbindir} +%{__install} -Dp -m0755 %{pbuild}/bin/* %{buildroot}%{_bindir} %{__install} -Dp -m0755 %{pbuild}/sbin/* %{buildroot}%{_sbindir} -%{__mv} %{buildroot}%{_sbindir}/puppet %{buildroot}%{_bindir}/puppet -%{__mv} %{buildroot}%{_sbindir}/puppetrun %{buildroot}%{_bindir}/puppetrun %{__install} -Dp -m0644 %{pbuild}/lib/puppet.rb %{buildroot}%{ruby_sitelibdir}/puppet.rb %{__cp} -a %{pbuild}/lib/puppet %{buildroot}%{ruby_sitelibdir} find %{buildroot}%{ruby_sitelibdir} -type f -perm +ugo+x -exec chmod a-x '{}' \; -%{__install} -Dp -m0644 %{confdir}/client.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppet +%{__cp} -a %{pbuild}/conf/redhat/client.sysconfig %{buildroot}%{_confdir}/client.sysconfig +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/client.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppet %{__install} -Dp -m0755 %SOURCE1 %{buildroot}%{_initrddir}/puppet -%{__install} -Dp -m0644 %{confdir}/server.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppetmaster +%{__cp} -a %{pbuild}/conf/redhat/server.sysconfig %{buildroot}%{_confdir}/server.sysconfig +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/server.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppetmaster %{__install} -Dp -m0755 %SOURCE2 %{buildroot}%{_initrddir}/puppetmaster -%{__install} -Dp -m0644 %{confdir}/fileserver.conf %{buildroot}%{_sysconfdir}/puppet/fileserver.conf -%{__install} -Dp -m0644 %{confdir}/puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppet.conf -#%{__ln_s} puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppetmasterd.conf -#%{__ln_s} puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppetca.conf -%{__install} -Dp -m0644 %{confdir}/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/puppet +%{__cp} -a %{pbuild}/conf/redhat/fileserver.conf %{buildroot}%{_confdir}/fileserver.conf +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/fileserver.conf %{buildroot}%{_sysconfdir}/puppet/fileserver.conf +%{__cp} -a %{pbuild}/conf/redhat/puppet.conf %{buildroot}%{_confdir}/puppet.conf +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppet.conf +%{__cp} -a %{pbuild}/conf/redhat/logrotate %{buildroot}%{_confdir}/logrotate +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/puppet %{__ln_s} %{_initrddir}/puppet %{buildroot}%{_sbindir}/rcpuppet %{__ln_s} %{_initrddir}/puppetmaster %{buildroot}%{_sbindir}/rcpuppetmaster %files %defattr(-, root, root, 0755) %{_bindir}/puppet -%{_sbindir}/filebucket -%{_sbindir}/ralsh +%{_bindir}/puppetdoc +%{_bindir}/filebucket +%{_bindir}/ralsh +%{_bindir}/pi %{_sbindir}/puppetd %{_sbindir}/rcpuppet -%{_sbindir}/pi %{ruby_sitelibdir}/* %{_initrddir}/puppet /var/adm/fillup-templates/sysconfig.puppet %config(noreplace) %{_sysconfdir}/puppet/puppet.conf %doc CHANGELOG COPYING LICENSE README examples -%exclude %{_sbindir}/puppetdoc %config(noreplace) %{_sysconfdir}/logrotate.d/puppet %dir %{_sysconfdir}/puppet # These need to be owned by puppet so the server can @@ -95,56 +101,115 @@ find %{buildroot}%{ruby_sitelibdir} -type f -perm +ugo+x -exec chmod a-x '{}' \; %files server %defattr(-, root, root, 0755) %{_sbindir}/puppetmasterd +%{_sbindir}/rcpuppetmaster %{_sbindir}/puppetqd -%{_bindir}/puppetrun +%{_sbindir}/puppetrun +%{_sbindir}/puppetca %{_initrddir}/puppetmaster %config(noreplace) %{_sysconfdir}/puppet/* -%config(noreplace) %{_sysconfdir}/sysconfig/puppetmaster -%{_sbindir}/puppetca +%exclude %{_sysconfdir}/puppet/puppet.conf +/var/adm/fillup-templates/sysconfig.puppetmaster +%dir %{_sysconfdir}/puppet %pre /usr/sbin/groupadd -r puppet 2>/dev/null || : /usr/sbin/useradd -g puppet -c "Puppet" \ - -s /sbin/nologin -r -d /var/lib/puppet puppet 2> /dev/null || : + -s /sbin/nologin -r -d /var/puppet puppet 2> /dev/null || : %post -/sbin/chkconfig --add puppet -exit 0 +%{fillup_and_insserv -y puppet} %post server -/sbin/chkconfig --add puppetmaster +%{fillup_and_insserv -n -y puppetmaster} %preun -if [ "$1" = 0 ] ; then - /sbin/service puppet stop > /dev/null 2>&1 - /sbin/chkconfig --del puppet -fi +%stop_on_removal puppet %preun server -if [ "$1" = 0 ] ; then - /sbin/service puppetmaster stop > /dev/null 2>&1 - /sbin/chkconfig --del puppetmaster -fi +%stop_on_removal puppetmaster + +%postun +%restart_on_update puppet +%{insserv_cleanup} %postun server -if [ "$1" -ge 1 ]; then - /sbin/service puppetmaster try-restart > /dev/null 2>&1 -fi +%restart_on_update puppetmaster +%{insserv_cleanup} %clean %{__rm} -rf %{buildroot} %changelog -* Sat Feb 16 2008 James Turnbull - 0.24.1-1 -- Fixed puppet configuation file references to match single puppet.conf file -- Update versions for 0.24.1 release +* Wed Jul 21 2010 Ben Kevan - 2.6.0 +- New version and ruby version bump +- Add puppetdoc to %_bindir (unknown why original suse package, excluded or forgot to add) +- Corrected patch for ruby environment +- Move binaries back to the correct directories + +* Wed Jul 14 2010 Ben Kevan - 0.25.5 +- New version. +- Use original client, server.init names +- Revert to puppetmaster +- Fixed client.init and server.init and included $null and Should-Stop for both + +* Tue Mar 2 2010 Martin Vuk - 0.25.4 +- New version. + +* Sun Aug 9 2009 Noah Fontes +- Fix build on SLES 9. +- Enable puppet and puppet-server services by default. + +* Sat Aug 8 2009 Noah Fontes +- Fix a lot of relevant warnings from rpmlint. +- Build on OpenSUSE 11.1 correctly. +- Rename puppetmaster init scripts to puppet-server to correspond to the package name. + +* Wed Apr 22 2009 Leo Eraly - 0.24.8 +- New version. + +* Tue Dec 9 2008 Leo Eraly - 0.24.6 +- New version. + +* Fri Sep 5 2008 Leo Eraly - 0.24.5 +- New version. + +* Fri Jun 20 2008 Martin Vuk - 0.24.4 +- Removed symlinks to old configuration files + +* Fri Dec 14 2007 Martin Vuk - 0.24.0 +- New version. + +* Fri Jun 29 2007 Martin Vuk - 0.23.0 +- New version. + +* Wed May 2 2007 Martin Vuk - 0.22.4 +- New version. Includes provider for rug package manager. + +* Wed Apr 25 2007 Martin Vuk - 0.22.3 +- New version. Added links /sbin/rcpuppet and /sbin/rcpuppetmaster + +* Sun Jan 7 2007 Martin Vuk - 0.22.0 +- version bump + +* Tue Oct 3 2006 Martin Vuk - 0.19.3-3 +- Made package arch dependant. + +* Sat Sep 23 2006 Martin Vuk - 0.19.3-1 +- New version + +* Sun Sep 17 2006 Martin Vuk - 0.19.1-1 +- New version + +* Tue Aug 30 2006 Martin Vuk - 0.19.0-1 +- New version +- No need to patch anymore :-), since my changes went into official release. * Tue Aug 3 2006 Martin Vuk - 0.18.4-3 - Replaced puppet-bin.patch with %build section from David's spec * Tue Aug 1 2006 Martin Vuk - 0.18.4-2 - Added supprot for enabling services in SuSE - + * Tue Aug 1 2006 Martin Vuk - 0.18.4-1 - New version and support for SuSE diff --git a/conf/suse/puppet.suse.patch b/conf/suse/puppet.suse.patch deleted file mode 100644 index 41974679f..000000000 --- a/conf/suse/puppet.suse.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- sbin/puppetmasterd.orig 2009-11-23 10:38:16.632264990 -0500 -+++ sbin/puppetmasterd 2009-11-23 10:38:28.884265155 -0500 -@@ -1,4 +1,4 @@ --#!/usr/bin/env ruby -+#!/usr/bin/ruby - - # - # = Synopsis ---- sbin/puppetd.orig 2009-11-23 10:38:23.240265893 -0500 -+++ sbin/puppetd 2009-11-23 10:38:35.044265271 -0500 -@@ -1,4 +1,4 @@ --#!/usr/bin/env ruby -+#!/usr/bin/ruby - # == Synopsis - # - # Retrieve the client configuration from the central puppet server and apply diff --git a/conf/suse/ruby-env.patch b/conf/suse/ruby-env.patch new file mode 100644 index 000000000..0a7a8a748 --- /dev/null +++ b/conf/suse/ruby-env.patch @@ -0,0 +1,17 @@ +--- sbin/puppetd 2010-07-18 14:17:38.000000000 -0700 ++++ sbin/puppetd 2010-07-21 17:31:57.057138615 -0700 +@@ -1,4 +1,4 @@ +-#!/usr/bin/env ruby ++#!/usr/bin/ruby + + require 'puppet/application/agent' + Puppet::Application[:agent].run + +--- sbin/puppetmasterd 2010-07-18 14:17:38.000000000 -0700 ++++ sbin/puppetmasterd 2010-07-21 17:32:10.178513969 -0700 +@@ -1,4 +1,4 @@ +-#!/usr/bin/env ruby ++#!/usr/bin/ruby + + require 'puppet/application/master' + Puppet::Application[:master].run diff --git a/conf/suse/server.init b/conf/suse/server.init index e3a1d902a..50b9fef81 100644 --- a/conf/suse/server.init +++ b/conf/suse/server.init @@ -11,10 +11,10 @@ ### BEGIN INIT INFO # Provides: puppetmaster -# Required-Start: $local_fs $network $syslog -# Should-Start: $null -# Required-Stop: $null -# Should-Stop: $null +# Required-Start: $local_fs $remote_fs $network $syslog +# Should-Start: puppetmaster +# Required-Stop: $local_fs $remote_fs $network $syslog +# Should-Stop: puppetmaster # Default-Start: 3 4 5 # Default-Stop: 0 1 2 6 # Short-Description: puppetmaster -- cgit From d38e52224f19b4fcedef722b2aab8fc94a59b481 Mon Sep 17 00:00:00 2001 From: Jesse Wolfe Date: Thu, 22 Jul 2010 14:33:58 -0700 Subject: [#4333] old optparse doesn't support default_argv= optparse hasn't always had the concept of default_argv. Fortunately, we don't really need it. --- lib/puppet/application.rb | 3 +-- spec/unit/application_spec.rb | 14 ++------------ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/lib/puppet/application.rb b/lib/puppet/application.rb index 05b7d466f..0a8fbc155 100644 --- a/lib/puppet/application.rb +++ b/lib/puppet/application.rb @@ -264,7 +264,6 @@ class Application self.send(fname, value) end end - @option_parser.default_argv = self.command_line.args @option_parser end @@ -337,7 +336,7 @@ class Application # scan command line argument begin - self.option_parser.parse! + self.option_parser.parse!(self.command_line.args) rescue OptionParser::ParseError => detail $stderr.puts detail $stderr.puts "Try 'puppet #{command_line.subcommand_name} --help'" diff --git a/spec/unit/application_spec.rb b/spec/unit/application_spec.rb index 3c354cce9..433809172 100755 --- a/spec/unit/application_spec.rb +++ b/spec/unit/application_spec.rb @@ -194,7 +194,6 @@ describe Puppet::Application do it "should create a new option parser when needed" do option_parser = stub "option parser" option_parser.stubs(:on) - option_parser.stubs(:default_argv=) OptionParser.expects(:new).returns(option_parser).once @app.option_parser.should == option_parser @app.option_parser.should == option_parser @@ -203,7 +202,6 @@ describe Puppet::Application do it "should pass the banner to the option parser" do option_parser = stub "option parser" option_parser.stubs(:on) - option_parser.stubs(:default_argv=) @app.class.instance_eval do banner "banner" end @@ -213,15 +211,6 @@ describe Puppet::Application do @app.option_parser end - it "should set the optionparser's args to the command line args" do - option_parser = stub "option parser" - option_parser.stubs(:on) - option_parser.expects(:default_argv=).with(%w{ fake args }) - @app.command_line.stubs(:args).returns(%w{ fake args }) - OptionParser.expects(:new).returns(option_parser) - @app.option_parser - end - it "should get options from Puppet.settings.optparse_addargs" do Puppet.settings.expects(:optparse_addargs).returns([]) @@ -237,7 +226,8 @@ describe Puppet::Application do end it "should ask OptionParser to parse the command-line argument" do - @app.option_parser.expects(:parse!) + @app.command_line.stubs(:args).returns(%w{ fake args }) + @app.option_parser.expects(:parse!).with(%w{ fake args }) @app.parse_options end -- cgit From 13c71b97d93edf033897dc09093861f68c9dd23a Mon Sep 17 00:00:00 2001 From: Jesse Wolfe Date: Fri, 23 Jul 2010 13:59:40 -0700 Subject: extlookup() is a builtin This patch promotes extlookup() to being a builtin function. It also adds test and makes some minor tweaks to the code. The behavior of extlookup has been left unchanged. --- ext/extlookup.rb | 181 --------------------------- lib/puppet/parser/functions/extlookup.rb | 173 +++++++++++++++++++++++++ spec/unit/parser/functions/extlookup_spec.rb | 85 +++++++++++++ 3 files changed, 258 insertions(+), 181 deletions(-) delete mode 100644 ext/extlookup.rb create mode 100644 lib/puppet/parser/functions/extlookup.rb create mode 100755 spec/unit/parser/functions/extlookup_spec.rb diff --git a/ext/extlookup.rb b/ext/extlookup.rb deleted file mode 100644 index d87583ba7..000000000 --- a/ext/extlookup.rb +++ /dev/null @@ -1,181 +0,0 @@ -# Puppet External Data Sources -# -# This is a parser function to read data from external files, this version -# uses CSV files but the concept can easily be adjust for databases, yaml -# or any other queryable data source. -# -# The object of this is to make it obvious when it's being used, rather than -# magically loading data in when an module is loaded I prefer to look at the code -# and see statements like: -# -# $snmp_contact = extlookup("snmp_contact") -# -# The above snippet will load the snmp_contact value from CSV files, this in its -# own is useful but a common construct in puppet manifests is something like this: -# -# case $domain { -# "myclient.com": { $snmp_contact = "John Doe " } -# default: { $snmp_contact = "My Support " } -# } -# -# Over time there will be a lot of this kind of thing spread all over your manifests -# and adding an additional client involves grepping through manifests to find all the -# places where you have constructs like this. -# -# This is a data problem and shouldn't be handled in code, a using this function you -# can do just that. -# -# First you configure it in site.pp: -# $extlookup_datadir = "/etc/puppet/manifests/extdata" -# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"] -# -# The array tells the code how to resolve values, first it will try to find it in -# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv -# -# Now create the following data files in /etc/puppet/manifests/extdata -# -# domain_myclient.com.csv: -# snmp_contact,John Doe -# root_contact,support@%{domain} -# client_trusted_ips,192.168.1.130,192.168.10.0/24 -# -# common.csv: -# snmp_contact,My Support -# root_contact,support@my.com -# -# Now you can replace the case statement with the simple single line to achieve -# the exact same outcome: -# -# $snmp_contact = extlookup("snmp_contact") -# -# The obove code shows some other features, you can use any fact or variable that -# is in scope by simply using %{varname} in your data files, you can return arrays -# by just having multiple values in the csv after the initial variable name. -# -# In the event that a variable is nowhere to be found a critical error will be raised -# that will prevent your manifest from compiling, this is to avoid accidentally putting -# in empty values etc. You can however specify a default value: -# -# $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org") -# -# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in -# any data file. -# -# You can also specify an additional data file to search first before any others at use -# time, for example: -# -# $version = extlookup("rsyslog_version", "present", "packages") -# -# package{"rsyslog": ensure => $version } -# -# This will look for a version configured in packages.csv and then in the rest as configured -# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind -# of use case makes puppet a lot nicer for managing large amounts of packages since you do not -# need to edit a load of manifests to do simple things like adjust a desired version number. -# -# For more information on installing and writing your own custom functions see: -# http://docs.puppetlabs.com/guides/custom_functions.html -# -# For further help contact Volcane on #puppet -require 'csv' - -module Puppet::Parser::Functions - newfunction(:extlookup, :type => :rvalue) do |args| - key = args[0] - default = "_ExtUNSET_" - datafile = "_ExtUNSET_" - - default = args[1] if args[1] - datafile = args[2] if args[2] - - extlookup_datadir = lookupvar('extlookup_datadir') - extlookup_precedence = Array.new - - # precedence values can have variables embedded in them - # in the form %{fqdn}, you could for example do - # - # $extlookup_precedence = ["hosts/%{fqdn}", "common"] - # - # this will result in /path/to/extdata/hosts/your.box.com.csv - # being searched. - # - # we parse the precedence here because the best place to specify - # it would be in site.pp but site.pp is only evaluated at startup - # so $fqdn etc would have no meaning there, this way it gets evaluated - # each run and has access to the right variables for that run - lookupvar('extlookup_precedence').each do |prec| - while prec =~ /%\{(.+?)\}/ - prec.gsub!(/%\{#{$1}\}/, lookupvar($1)) - end - - extlookup_precedence << prec - end - - - datafiles = Array.new - - # if we got a custom data file, put it first in the array of search files - if datafile != "" - datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv") - end - - extlookup_precedence.each do |d| - datafiles << extlookup_datadir + "/#{d}.csv" - end - - desired = "_ExtUNSET_" - - datafiles.each do |file| - parser.watch_file(file) if File.exists?(file) - - if desired == "_ExtUNSET_" - if File.exists?(file) - result = CSV.read(file).find_all do |r| - r[0] == key - end - - - # return just the single result if theres just one, - # else take all the fields in the csv and build an array - if result.length > 0 - if result[0].length == 2 - val = result[0][1].to_s - - # parse %{}'s in the CSV into local variables using lookupvar() - while val =~ /%\{(.+?)\}/ - val.gsub!(/%\{#{$1}\}/, lookupvar($1)) - end - - desired = val - elsif result[0].length > 1 - length = result[0].length - cells = result[0][1,length] - - # Individual cells in a CSV result are a weird data type and throws - # puppets yaml parsing, so just map it all to plain old strings - desired = cells.map do |c| - # parse %{}'s in the CSV into local variables using lookupvar() - while c =~ /%\{(.+?)\}/ - c.gsub!(/%\{#{$1}\}/, lookupvar($1)) - end - - c.to_s - end - end - end - end - end - end - - # don't accidently return nil's and such rather throw a parse error - if desired == "_ExtUNSET_" && default == "_ExtUNSET_" - raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" - else - desired = default if desired == "_ExtUNSET_" - end - - desired - end -end - -# vi:tabstop=4:expandtab:ai diff --git a/lib/puppet/parser/functions/extlookup.rb b/lib/puppet/parser/functions/extlookup.rb new file mode 100644 index 000000000..ee230e7ce --- /dev/null +++ b/lib/puppet/parser/functions/extlookup.rb @@ -0,0 +1,173 @@ +# Puppet External Data Sources +# +# This is a parser function to read data from external files, this version +# uses CSV files but the concept can easily be adjust for databases, yaml +# or any other queryable data source. +# +# The object of this is to make it obvious when it's being used, rather than +# magically loading data in when an module is loaded I prefer to look at the code +# and see statements like: +# +# $snmp_contact = extlookup("snmp_contact") +# +# The above snippet will load the snmp_contact value from CSV files, this in its +# own is useful but a common construct in puppet manifests is something like this: +# +# case $domain { +# "myclient.com": { $snmp_contact = "John Doe " } +# default: { $snmp_contact = "My Support " } +# } +# +# Over time there will be a lot of this kind of thing spread all over your manifests +# and adding an additional client involves grepping through manifests to find all the +# places where you have constructs like this. +# +# This is a data problem and shouldn't be handled in code, a using this function you +# can do just that. +# +# First you configure it in site.pp: +# $extlookup_datadir = "/etc/puppet/manifests/extdata" +# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"] +# +# The array tells the code how to resolve values, first it will try to find it in +# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv +# +# Now create the following data files in /etc/puppet/manifests/extdata +# +# domain_myclient.com.csv: +# snmp_contact,John Doe +# root_contact,support@%{domain} +# client_trusted_ips,192.168.1.130,192.168.10.0/24 +# +# common.csv: +# snmp_contact,My Support +# root_contact,support@my.com +# +# Now you can replace the case statement with the simple single line to achieve +# the exact same outcome: +# +# $snmp_contact = extlookup("snmp_contact") +# +# The obove code shows some other features, you can use any fact or variable that +# is in scope by simply using %{varname} in your data files, you can return arrays +# by just having multiple values in the csv after the initial variable name. +# +# In the event that a variable is nowhere to be found a critical error will be raised +# that will prevent your manifest from compiling, this is to avoid accidentally putting +# in empty values etc. You can however specify a default value: +# +# $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org") +# +# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in +# any data file. +# +# You can also specify an additional data file to search first before any others at use +# time, for example: +# +# $version = extlookup("rsyslog_version", "present", "packages") +# +# package{"rsyslog": ensure => $version } +# +# This will look for a version configured in packages.csv and then in the rest as configured +# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind +# of use case makes puppet a lot nicer for managing large amounts of packages since you do not +# need to edit a load of manifests to do simple things like adjust a desired version number. +# +# For more information on installing and writing your own custom functions see: +# http://docs.puppetlabs.com/guides/custom_functions.html +# +# For further help contact Volcane on #puppet +require 'csv' + +module Puppet::Parser::Functions + newfunction(:extlookup, :type => :rvalue) do |args| + key = args[0] + + default = args[1] + datafile = args[2] + + raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3 + + extlookup_datadir = lookupvar('extlookup_datadir') + extlookup_precedence = Array.new + + # precedence values can have variables embedded in them + # in the form %{fqdn}, you could for example do + # + # $extlookup_precedence = ["hosts/%{fqdn}", "common"] + # + # this will result in /path/to/extdata/hosts/your.box.com.csv + # being searched. + # + # we parse the precedence here because the best place to specify + # it would be in site.pp but site.pp is only evaluated at startup + # so $fqdn etc would have no meaning there, this way it gets evaluated + # each run and has access to the right variables for that run + lookupvar('extlookup_precedence').each do |prec| + while prec =~ /%\{(.+?)\}/ + prec.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + extlookup_precedence << prec + end + + + datafiles = Array.new + + # if we got a custom data file, put it first in the array of search files + if datafile != "" + datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv") + end + + extlookup_precedence.each do |d| + datafiles << extlookup_datadir + "/#{d}.csv" + end + + desired = nil + + datafiles.each do |file| + parser = Puppet::Parser::Parser.new(environment) + parser.watch_file(file) if File.exists?(file) + + if desired.nil? + if File.exists?(file) + result = CSV.read(file).find_all do |r| + r[0] == key + end + + + # return just the single result if theres just one, + # else take all the fields in the csv and build an array + if result.length > 0 + if result[0].length == 2 + val = result[0][1].to_s + + # parse %{}'s in the CSV into local variables using lookupvar() + while val =~ /%\{(.+?)\}/ + val.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + desired = val + elsif result[0].length > 1 + length = result[0].length + cells = result[0][1,length] + + # Individual cells in a CSV result are a weird data type and throws + # puppets yaml parsing, so just map it all to plain old strings + desired = cells.map do |c| + # parse %{}'s in the CSV into local variables using lookupvar() + while c =~ /%\{(.+?)\}/ + c.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + c.to_s + end + end + end + end + end + end + + desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" + end +end diff --git a/spec/unit/parser/functions/extlookup_spec.rb b/spec/unit/parser/functions/extlookup_spec.rb new file mode 100755 index 000000000..bf2880345 --- /dev/null +++ b/spec/unit/parser/functions/extlookup_spec.rb @@ -0,0 +1,85 @@ +#! /usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../../spec_helper' +require 'tempfile' + +describe "the extlookup function" do + + before :each do + @scope = Puppet::Parser::Scope.new + + @scope.stubs(:environment).returns(Puppet::Node::Environment.new('production')) + end + + it "should exist" do + Puppet::Parser::Functions.function("extlookup").should == "function_extlookup" + end + + it "should raise a ParseError if there is less than 1 arguments" do + lambda { @scope.function_extlookup([]) }.should( raise_error(Puppet::ParseError)) + end + + it "should raise a ParseError if there is more than 3 arguments" do + lambda { @scope.function_extlookup(["foo", "bar", "baz", "gazonk"]) }.should( raise_error(Puppet::ParseError)) + end + + it "should return the default" do + result = @scope.function_extlookup([ "key", "default"]) + result.should == "default" + end + + it "should lookup the key in a supplied datafile" do + t = Tempfile.new('extlookup.csv') do + t.puts 'key,value' + t.puts 'nonkey,nonvalue' + t.close + + result = @scope.function_extlookup([ "key", "default", t.path]) + result.should == "value" + end + end + + it "should return an array if the datafile contains more than two columns" do + t = Tempfile.new('extlookup.csv') do + t.puts 'key,value1,value2' + t.puts 'nonkey,nonvalue,nonvalue' + t.close + + result = @scope.function_extlookup([ "key", "default", t.path]) + result.should == ["value1", "value2"] + end + end + + it "should raise an error if there's no matching key and no default" do + t = Tempfile.new('extlookup.csv') do + t.puts 'key,value' + t.puts 'nonkey,nonvalue' + t.close + + result = @scope.function_extlookup([ "key", nil, t.path]) + result.should == "value" + end + end + + describe "should look in $extlookup_datadir for data files listed by $extlookup_precedence" do + before do + @scope.stubs(:lookupvar).with('extlookup_datadir').returns("/tmp") + @scope.stubs(:lookupvar).with('extlookup_precedence').returns(["one","two"]) + File.open("/tmp/one.csv","w"){|one| one.puts "key,value1" } + File.open("/tmp/two.csv","w") do |two| + two.puts "key,value2" + two.puts "key2,value_two" + end + end + + it "when the key is in the first file" do + result = @scope.function_extlookup([ "key" ]) + result.should == "value1" + end + + it "when the key is in the second file" do + result = @scope.function_extlookup([ "key2" ]) + result.should == "value_two" + end + end +end -- cgit From 67bdf89ac91a753171ad354b070546c62e09ce1a Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 15:26:01 +0200 Subject: Fix #4348 - Puppet doc single manifest broken The refactoring of using environment instances instead of strings for initializing the parser, rdoc wasn't updated, thus was unable to initialize the parser. Signed-off-by: Brice Figureau --- lib/puppet/util/rdoc.rb | 2 +- spec/unit/util/rdoc_spec.rb | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/puppet/util/rdoc.rb b/lib/puppet/util/rdoc.rb index 4a80b069b..085d8ec93 100644 --- a/lib/puppet/util/rdoc.rb +++ b/lib/puppet/util/rdoc.rb @@ -41,7 +41,7 @@ module Puppet::Util::RDoc def manifestdoc(files) Puppet[:ignoreimport] = true files.select { |f| FileTest.file?(f) }.each do |f| - parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment]) + parser = Puppet::Parser::Parser.new(Puppet::Node::Environment.new(Puppet[:environment])) parser.file = f ast = parser.parse output(f, ast) diff --git a/spec/unit/util/rdoc_spec.rb b/spec/unit/util/rdoc_spec.rb index 65df26156..58c2034ee 100755 --- a/spec/unit/util/rdoc_spec.rb +++ b/spec/unit/util/rdoc_spec.rb @@ -75,6 +75,19 @@ describe Puppet::Util::RDoc do Puppet::Util::RDoc.manifestdoc([]) end + it "should use a parser with the correct environment" do + FileTest.stubs(:file?).returns(true) + Puppet::Util::RDoc.stubs(:output) + + parser = stub_everything + Puppet::Parser::Parser.stubs(:new).with{ |env| env.is_a?(Puppet::Node::Environment) }.returns(parser) + + parser.expects(:file=).with("file") + parser.expects(:parse) + + Puppet::Util::RDoc.manifestdoc(["file"]) + end + it "should puppet parse all given files" do FileTest.stubs(:file?).returns(true) Puppet::Util::RDoc.stubs(:output) -- cgit From 760e418d254a8d2198d2c6eb466d783a5930ef47 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sun, 25 Jul 2010 22:14:31 -0700 Subject: Fix #4349 - Parsing with ignoreimport=true was always loading site.pp With the type collection refactoring, when accessing a fresh collection puppet tries to import the site.pp manifest (perfrom_initial_import). In the case of puppetdoc, we are parsing site.pp by ourselves, so we ended parsing it twice, resulting in an "import loop detected" error. Signed-off-by: Brice Figureau --- lib/puppet/resource/type_collection.rb | 1 + spec/unit/resource/type_collection_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/lib/puppet/resource/type_collection.rb b/lib/puppet/resource/type_collection.rb index 6a933362f..90b6df9c1 100644 --- a/lib/puppet/resource/type_collection.rb +++ b/lib/puppet/resource/type_collection.rb @@ -153,6 +153,7 @@ class Puppet::Resource::TypeCollection end def perform_initial_import + return if Puppet.settings[:ignoreimport] parser = Puppet::Parser::Parser.new(environment) if code = Puppet.settings.uninterpolated_value(:code, environment.to_s) and code != "" parser.string = code diff --git a/spec/unit/resource/type_collection_spec.rb b/spec/unit/resource/type_collection_spec.rb index 09643cd24..45fc05da0 100644 --- a/spec/unit/resource/type_collection_spec.rb +++ b/spec/unit/resource/type_collection_spec.rb @@ -400,6 +400,14 @@ describe Puppet::Resource::TypeCollection do @parser.expects(:parse).raises ArgumentError lambda { @code.perform_initial_import }.should raise_error(Puppet::Error) end + + it "should not do anything if the ignore_import settings is set" do + Puppet.settings[:ignoreimport] = true + @parser.expects(:string=).never + @parser.expects(:file=).never + @parser.expects(:parse).never + @code.perform_initial_import + end end describe "when determining the configuration version" do -- cgit From 7d42f77b83a7597a879a0fc2080c18b7c76c6fc3 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 17:42:42 +0200 Subject: JRuby doesn't implement Process.maxgroups So let's not call it :) Signed-off-by: Brice Figureau --- lib/puppet/util/monkey_patches.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/puppet/util/monkey_patches.rb b/lib/puppet/util/monkey_patches.rb index e035afd9f..9664ff310 100644 --- a/lib/puppet/util/monkey_patches.rb +++ b/lib/puppet/util/monkey_patches.rb @@ -1,4 +1,7 @@ -Process.maxgroups = 1024 + +unless defined? JRUBY_VERSION + Process.maxgroups = 1024 +end module RDoc def self.caller(skip=nil) -- cgit From 3163932f70e824906d48edc9665b4017a4669797 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 19:04:44 +0200 Subject: Fix #4244 - Cached Attributes is not thread safe The underlying hash is not protected and thus two threads accessing the cached value at the same time and one expiring the value can result in a race condition. This patch synchronizes the access to the value_cache underlying hash. Signed-off-by: Brice Figureau --- lib/puppet/util/cacher.rb | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/puppet/util/cacher.rb b/lib/puppet/util/cacher.rb index 8785c694f..3dddec0d4 100644 --- a/lib/puppet/util/cacher.rb +++ b/lib/puppet/util/cacher.rb @@ -1,3 +1,5 @@ +require 'monitor' + module Puppet::Util::Cacher module Expirer attr_reader :timestamp @@ -49,7 +51,7 @@ module Puppet::Util::Cacher define_method(name.to_s + "=") do |value| # Make sure the cache timestamp is set cache_timestamp - value_cache[name] = value + value_cache.synchronize { value_cache[name] = value } end if ttl = options[:ttl] @@ -70,6 +72,7 @@ module Puppet::Util::Cacher # Methods that get added to instances. module InstanceMethods + def expire # Only expire if we have an expirer. This is # mostly so that we can comfortably handle cases @@ -92,15 +95,17 @@ module Puppet::Util::Cacher end def cached_value(name) - # Allow a nil expirer, in which case we regenerate the value every time. - if expired_by_expirer?(name) - value_cache.clear - @cache_timestamp = Time.now - elsif expired_by_ttl?(name) - value_cache.delete(name) + value_cache.synchronize do + # Allow a nil expirer, in which case we regenerate the value every time. + if expired_by_expirer?(name) + value_cache.clear + @cache_timestamp = Time.now + elsif expired_by_ttl?(name) + value_cache.delete(name) + end + value_cache[name] = send("init_#{name}") unless value_cache.include?(name) + value_cache[name] end - value_cache[name] = send("init_#{name}") unless value_cache.include?(name) - value_cache[name] end def expired_by_expirer?(name) @@ -121,7 +126,7 @@ module Puppet::Util::Cacher end def value_cache - @value_cache ||= {} + @value_cache ||= {}.extend(MonitorMixin) end end end -- cgit From 4065e81f76020c29ed8ccfc1cf6c481aa202cb23 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 18:09:33 +0200 Subject: Fix race condition in rack autoloading of request/response Ruby autoloader seems to not be thread-safe. Since rack uses it to lazily load the Rack::Request and Rack::Response classes, on jruby it fails if the first compilation is done with multiple concurrent threads. Signed-off-by: Brice Figureau --- lib/puppet/network/http/rack.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/puppet/network/http/rack.rb b/lib/puppet/network/http/rack.rb index c18a07559..5b4ef7e1c 100644 --- a/lib/puppet/network/http/rack.rb +++ b/lib/puppet/network/http/rack.rb @@ -1,3 +1,6 @@ +require 'rack' +require 'rack/request' +require 'rack/response' require 'puppet/network/http' require 'puppet/network/http/rack/rest' -- cgit From ef9a4a6df163c370a524d9ab76453c96bb99e8a4 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 25 Jul 2010 19:19:04 +0200 Subject: Fix #4245 - default insertion of ACL is not thread safe This can happen under jruby with native threads. Signed-off-by: Brice Figureau --- lib/puppet/network/rest_authconfig.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/puppet/network/rest_authconfig.rb b/lib/puppet/network/rest_authconfig.rb index 38a8f9a09..82d5a9de2 100644 --- a/lib/puppet/network/rest_authconfig.rb +++ b/lib/puppet/network/rest_authconfig.rb @@ -3,6 +3,7 @@ require 'puppet/network/authconfig' module Puppet class Network::RestAuthConfig < Network::AuthConfig + extend MonitorMixin attr_accessor :rights DEFAULT_ACL = [ @@ -20,9 +21,11 @@ module Puppet ] def self.main - add_acl = @main.nil? - super - @main.insert_default_acl if add_acl and !@main.exists? + synchronize do + add_acl = @main.nil? + super + @main.insert_default_acl if add_acl and !@main.exists? + end @main end -- cgit From 9f08e7c1e49282754c839e631bf52102707336c0 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Wed, 21 Jul 2010 22:51:45 +0200 Subject: Feature: puppet-load - a tool to stress-test master compilation This commit introduce a new executable (ext/puppet-load) which aims to simulate concurrent clients to stress-test load a puppet master. At the end of a run, it produces some statistics. This tool is very lightweight: * it runs under Event Machine (and thus is event-driven) * it doesn't do anything with the received catalog This tool, to run, needs access to: * a certificate/private_key pair from a node known by the master * a fact file like those persisted on the master * obviously a puppet-master running somewhere Refer to the embedded help for options and run examples. TODO: * fetch different nodes catalog * exercise the file server Signed-off-by: Brice Figureau --- ext/puppet-load.rb | 357 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 ext/puppet-load.rb diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb new file mode 100644 index 000000000..110282d01 --- /dev/null +++ b/ext/puppet-load.rb @@ -0,0 +1,357 @@ +#!/usr/bin/env ruby +# == Synopsis +# +# This tool can exercize a puppetmaster by simulating an arbitraty number of concurrent clients +# in a lightweight way. +# +# = Prerequisites +# +# This tool requires Event Machine and em-http-request, and an installation of Puppet. +# Event Machine can be installed from gem. +# em-http-request can be installed from gem. +# +# = Usage +# +# puppet-load [-d|--debug] [--concurrency ] [--repeat ] [-V|--version] [-v|--verbose] +# [--node ] [--facts ] [--cert ] [--key ] +# [--server ] +# +# = Description +# +# This is a simple script meant for doing performance tests of puppet masters. It does this +# by simulating concurrent connections to a puppet master and asking for catalog compilation. +# +# = Options +# +# Unlike other puppet executables, puppet-load doesn't parse puppet.conf nor use puppet options +# +# debug:: +# Enable full debugging. +# +# concurreny:: +# Number of simulated concurrent clients. +# +# server:: +# Set the puppet master hostname or IP address.. +# +# node:: +# Set the fully-qualified domain name of the client. This is only used for +# certificate purposes, but can be used to override the discovered hostname. +# +# help:: +# Print this help message +# +# facts:: +# This can be used to provide facts for the compilation, directly from a YAML +# file as found in the clientyaml directory. If none are provided, puppet-load +# will look by itself using Puppet facts indirector. +# +# cert:: +# This option is mandatory. It should be set to the cert PEM file that will be used +# to quthenticate the client connections. +# +# key:: +# This option is mandatory. It should be set to the private key PEM file that will be used +# to quthenticate the client connections. +# +# timeout:: +# The number of seconds after which a simulated client is declared in error if it didn't get +# a catalog. The default is 180s. +# +# repeat:: +# How many times to perform the test. This means puppet-load will ask for +# concurrency * repeat catalogs. +# +# verbose:: +# Turn on verbose reporting. +# +# version:: +# Print the puppet version number and exit. +# +# = Example usage +# +# 1) On the master host, generate a new certificate and private key for our test host: +# puppet ca --generate puppet-load.domain.com [*] +# +# 2) Copy the cert and key to the puppet-load host (which can be the same as the master one) +# +# 3) On the master host edit or create the auth.conf so that the catalog ACL match: +# path ~ ^/catalog/([^/]+)$ +# method find +# allow $1 +# allow puppet-load.domain.com +# +# 4) launch the master +# +# 5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host +# you want to simulate. +# +# 5) launch puppet-load +# puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 +# +# [*]: unfortunately at this stage Puppet trusts the certname of the connecting node more than +# than the node name request paramater. It means that the master will compile +# the puppet-load node and not the --node given. +# +# = TODO +# * Allow to simulate any different nodes +# * More output stats for error connections (ie report errors, HTTP code...) +# +# + +# Do an initial trap, so that cancels don't get a stack trace. +trap(:INT) do + $stderr.puts "Cancelling startup" + exit(1) +end + +require 'rubygems' +require 'eventmachine' +require 'em-http' +require 'getoptlong' +require 'puppet' + +$cmdargs = [ + [ "--concurrency", "-c", GetoptLong::REQUIRED_ARGUMENT ], + [ "--node", "-n", GetoptLong::REQUIRED_ARGUMENT ], + [ "--facts", GetoptLong::REQUIRED_ARGUMENT ], + [ "--repeat", "-r", GetoptLong::REQUIRED_ARGUMENT ], + [ "--cert", "-C", GetoptLong::REQUIRED_ARGUMENT ], + [ "--key", "-k", GetoptLong::REQUIRED_ARGUMENT ], + [ "--timeout", "-t", GetoptLong::REQUIRED_ARGUMENT ], + [ "--server", "-s", GetoptLong::REQUIRED_ARGUMENT ], + [ "--debug", "-d", GetoptLong::NO_ARGUMENT ], + [ "--help", "-h", GetoptLong::NO_ARGUMENT ], + [ "--verbose", "-v", GetoptLong::NO_ARGUMENT ], + [ "--version", "-V", GetoptLong::NO_ARGUMENT ], +] + +Puppet::Util::Log.newdestination(:console) + +times = {} + +def read_facts(file) + YAML.load(File.read(file)) +end + + +result = GetoptLong.new(*$cmdargs) + +$args = {} +$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140} + +begin + result.each { |opt,arg| + case opt + when "--concurrency" + begin + $options[:concurrency] = Integer(arg) + rescue => detail + $stderr.puts "The argument to 'fork' must be an integer" + exit(14) + end + when "--node" + $options[:node] = arg + when "--server" + $options[:server] = arg + when "--masterport" + $options[:masterport] = arg + when "--facts" + $options[:facts] = arg + when "--repeat" + $options[:repeat] = Integer(arg) + when "--help" + if Puppet.features.usage? + RDoc::usage && exit + else + puts "No help available unless you have RDoc::usage installed" + exit + end + when "--version" + puts "%s" % Puppet.version + exit + when "--verbose" + Puppet::Util::Log.level = :info + Puppet::Util::Log.newdestination(:console) + when "--debug" + Puppet::Util::Log.level = :debug + Puppet::Util::Log.newdestination(:console) + when "--cert" + $options[:cert] = arg + when "--key" + $options[:key] = arg + end + } +rescue GetoptLong::InvalidOption => detail + $stderr.puts detail + $stderr.puts "Try '#{$0} --help'" + exit(1) +end + +unless $options[:cert] and $options[:key] + raise "--cert and --key are mandatory to authenticate the client" +end + +unless $options[:facts] and facts = read_facts($options[:facts]) + unless facts = Puppet::Node::Facts.find($options[:node]) + raise "Could not find facts for %s" % $options[:node] + end +end + +unless $options[:node] + raise "--node is a mandatory argument. It tells to the master what node to compile" +end + +facts.values["fqdn"] = $options[:node] +facts.values["hostname"] = $options[:node].sub(/\..+/, '') +facts.values["domain"] = $options[:node].sub(/^[^.]+\./, '') + +parameters = {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(facts.render(:b64_zlib_yaml))} + +class RequestPool + include EventMachine::Deferrable + + attr_reader :requests, :responses, :times, :sizes + attr_reader :repeat, :concurrency, :max_request + + def initialize(concurrency, repeat, parameters) + @parameters = parameters + @current_request = 0 + @max_request = repeat * concurrency + @repeat = repeat + @concurrency = concurrency + @requests = [] + @responses = {:succeeded => [], :failed => []} + @times = {} + @sizes = {} + + # initial spawn + (1..concurrency).each do |i| + spawn + end + + end + + def spawn_request(index) + EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{$options[:node]}").get( + :port => $options[:masterport], + :query => @parameters, + :timeout => $options[:timeout], + :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" }, + :ssl => { :private_key_file => $options[:key], + :cert_chain_file => $options[:cert], + :verify_peer => false } ) do + Puppet.debug("starting client #{index}") + @times[index] = Time.now + @sizes[index] = 0 + end + end + + def add(index, conn) + @requests.push(conn) + + conn.stream { |data| + @sizes[index] += data.length + } + + conn.callback { + @times[index] = Time.now - @times[index] + code = conn.response_header.status + if code >= 200 && code < 300 + Puppet.debug("Client #{index} finished successfully") + @responses[:succeeded].push(conn) + else + Puppet.debug("Client #{index} finished with HTTP code #{code}") + @responses[:failed].push(conn) + end + check_progress + } + + conn.errback { + Puppet.debug("Client #{index} finished with an error: #{conn.response.error}") + @times[index] = Time.now - @times[index] + @responses[:failed].push(conn) + check_progress + } + end + + def all_responses + @responses[:succeeded] + @responses[:failed] + end + + protected + + def check_progress + spawn unless all_spawned? + succeed if all_finished? + end + + def all_spawned? + @requests.size >= max_request + end + + def all_finished? + @responses[:failed].size + @responses[:succeeded].size >= max_request + end + + def spawn + add(@current_request, spawn_request(@current_request)) + @current_request += 1 + end +end + + +def mean(array) + array.inject(0) { |sum, x| sum += x } / array.size.to_f +end + +def median(array) + array = array.sort + m_pos = array.size / 2 + return array.size % 2 == 1 ? array[m_pos] : mean(array[m_pos-1..m_pos]) +end + +def format_bytes(bytes) + if bytes < 1024 + "%.2f B" % bytes + elsif bytes < 1024 * 1024 + "%.2f KiB" % (bytes/1024.0) + else + "%.2f MiB" % (bytes/(1024.0*1024.0)) + end +end + +EM::run { + + start = Time.now + multi = RequestPool.new($options[:concurrency], $options[:repeat], parameters) + + multi.callback do + duration = Time.now - start + puts "#{multi.max_request} requests finished in #{duration} s" + puts "#{multi.responses[:failed].size} requests failed" + puts "Availability: %3.2f %%" % (100.0*multi.responses[:succeeded].size/(multi.responses[:succeeded].size+multi.responses[:failed].size)) + + minmax = multi.times.values.minmax + all_time = multi.times.values.reduce(:+) + + puts "\nTime (s):" + puts "\tmin: #{minmax[0]} s" + puts "\tmax: #{minmax[1]} s" + puts "\taverage: #{mean(multi.times.values)} s" + puts "\tmedian: #{median(multi.times.values)} s" + + puts "\nConcurrency: %.2f" % (all_time/duration) + puts "Transaction Rate (tps): %.2f t/s" % (multi.max_request / duration) + + transferred = multi.sizes.values.reduce(:+) + + puts "\nReceived bytes: #{format_bytes(transferred)}" + puts "Throughput: %.5f MiB/s" % (transferred/duration/(1024.0*1024.0)) + + # this is the end + EventMachine.stop + end +} + + -- cgit From bdfcac5c9c44cdbb4a13e282720ff29fed79642c Mon Sep 17 00:00:00 2001 From: Todd Zullinger Date: Tue, 20 Jul 2010 09:55:06 -0400 Subject: Update Red Hat spec file for 2.6.0 --- conf/redhat/puppet.spec | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/conf/redhat/puppet.spec b/conf/redhat/puppet.spec index 6791cea77..e51af0a6b 100644 --- a/conf/redhat/puppet.spec +++ b/conf/redhat/puppet.spec @@ -5,14 +5,14 @@ %global confdir conf/redhat Name: puppet -Version: 0.25.5 +Version: 2.6.0 Release: 1%{?dist} Summary: A network tool for managing many disparate systems License: GPLv2+ URL: http://puppetlabs.com Source0: http://puppetlabs.com/downloads/%{name}/%{name}-%{version}.tar.gz Source1: http://puppetlabs.com/downloads/%{name}/%{name}-%{version}.tar.gz.sign -Patch0: rundir-perms.patch + Group: System Environment/Base BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) @@ -68,6 +68,7 @@ The server can also function as a certificate authority and file server. %setup -q patch -p1 < conf/redhat/rundir-perms.patch + %build # Fix some rpmlint complaints for f in mac_dscl.pp mac_dscl_revert.pp \ @@ -91,6 +92,7 @@ rm -rf %{buildroot} ruby install.rb --destdir=%{buildroot} --quick --no-rdoc install -d -m0755 %{buildroot}%{_sysconfdir}/puppet/manifests +install -d -m0755 %{buildroot}%{_datadir}/%{name}/modules install -d -m0755 %{buildroot}%{_localstatedir}/lib/puppet install -d -m0755 %{buildroot}%{_localstatedir}/run/puppet install -d -m0750 %{buildroot}%{_localstatedir}/log/puppet @@ -221,6 +223,10 @@ fi rm -rf %{buildroot} %changelog +* Tue Jul 20 2010 Todd Zullinger - 2.6.0-1 +- Update to 2.6.0 +- Create and own /usr/share/puppet/modules (#615432) + * Mon May 03 2010 Todd Zullinger - 0.25.5-1 - Update to 0.25.5 - Adjust selinux conditional for EL-6 -- cgit From ecf44e4408c168893d74af58a4c7c8606634a844 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Tue, 27 Jul 2010 00:36:13 +1000 Subject: Updated CHANGELOG for 2.6.1rc1 --- CHANGELOG | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 53124a13b..421b5136b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,38 @@ +2.6.1rc1 +======== +bdfcac5 Update Red Hat spec file for 2.6.0 +9f08e7c Feature: puppet-load - a tool to stress-test master compilation +ef9a4a6 Fix #4245 - default insertion of ACL is not thread safe +4065e81 Fix race condition in rack autoloading of request/response +3163932 Fix #4244 - Cached Attributes is not thread safe +7d42f77 JRuby doesn't implement Process.maxgroups +760e418 Fix #4349 - Parsing with ignoreimport=true was always loading site.pp +67bdf89 Fix #4348 - Puppet doc single manifest broken +13c71b9 extlookup() is a builtin +d38e522 [#4333] old optparse doesn't support default_argv= +86b0882 Fixed #4326 - Updated SUSE packaging +03313b8 Fix #4319 - source file url sent to the master is invalid +ac3a0d2 vim: highlight default parameters in definition/classes +be2141a vim: match collected resources. +c047c8d vim: added elsif +9569136 Fix for 4314 -- Need to allow '-' in class name for refs +636079f Fixed #4304 - Changed logging level for auto import message +000fd1e Fix for #4303 -- reverting to old escaping in '-strings +1d494a3 Tweak to fix for #4302--dangling ref to known_resource_types +2383050 Fix #4302 - Compilation speed regression compared to 2.6 +63ec207 Minimal fix for #4297, with notes for follow-up +7ad7eb1 Fix #4286 - rename puppetdoc global module to __site__ +28bb195 Fixed yumrepo type deprecation wanring ` +067a46d Temporary tweak to tests for #4242 +9778f2a [#4242] Fixed recursion due to parents including their children +59a23d6 Fix for #3382 -- Empty classes as graph placeholders +865282a Fixed example config.ru +a0a63c3 Fixed network and indirection reference +64386cf Fixed Indirection reference + 2.6.0 ===== +db0b30d Updated CHANGELOG for 2.6.0 42a475e Fixing #4268 - manifests always imported 06fc40c [#4269] Undef variables interpolate to empty string 1288f8c [#4270] Force inherited classes to load into the correct environment -- cgit