From ce1865f3d0bdcdf89d18e746806645b2b84b5252 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 17 Oct 2010 16:21:12 +0200 Subject: Fix #5023 - puppet-load multiple nodes support This patch allows puppet-load to compile multiple nodes catalog. This is done by using multiple --node. Puppet-load will use round-robin to chose which nodes catalog to ask for a given simulated client. It is also possible to pass a directory of facts yaml file in which puppet-load will load given --node facts file. This can work only if #5020 is applied to the puppetmaster first. Signed-off-by: Brice Figureau --- ext/puppet-load.rb | 84 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb index 110282d01..24437c10b 100644 --- a/ext/puppet-load.rb +++ b/ext/puppet-load.rb @@ -14,7 +14,7 @@ # # puppet-load [-d|--debug] [--concurrency ] [--repeat ] [-V|--version] [-v|--verbose] # [--node ] [--facts ] [--cert ] [--key ] -# [--server ] +# [--factsdir ] [--server ] # # = Description # @@ -35,8 +35,9 @@ # Set the puppet master hostname or IP address.. # # node:: -# Set the fully-qualified domain name of the client. This is only used for -# certificate purposes, but can be used to override the discovered hostname. +# Set the fully-qualified domain name of the client. This option can be given multiple +# times. In this case puppet-load will ask for catalog compilation of all the given nodes +# on a round robin way. # # help:: # Print this help message @@ -46,6 +47,11 @@ # file as found in the clientyaml directory. If none are provided, puppet-load # will look by itself using Puppet facts indirector. # +# factsdir:: +# Specify a directory where the yaml facts files can be found. If provided puppet-load +# will look up facts in this directory. If not found it will resort to using Puppet Facts +# indirector. +# # cert:: # This option is mandatory. It should be set to the cert PEM file that will be used # to quthenticate the client connections. @@ -70,8 +76,9 @@ # # = Example usage # +# SINGLE NODE: # 1) On the master host, generate a new certificate and private key for our test host: -# puppet ca --generate puppet-load.domain.com [*] +# puppet ca --generate puppet-load.domain.com # # 2) Copy the cert and key to the puppet-load host (which can be the same as the master one) # @@ -81,7 +88,7 @@ # allow $1 # allow puppet-load.domain.com # -# 4) launch the master +# 4) launch the master(s) # # 5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host # you want to simulate. @@ -89,12 +96,30 @@ # 5) launch puppet-load # puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 # -# [*]: unfortunately at this stage Puppet trusts the certname of the connecting node more than -# than the node name request paramater. It means that the master will compile -# the puppet-load node and not the --node given. +# MULTIPLE NODES: +# 1) On the master host, generate a new certificate and private key for our test host: +# puppet ca --generate puppet-load.domain.com +# +# 2) Copy the cert and key to the puppet-load host (which can be the same as the master one) +# +# 3) On the master host edit or create the auth.conf so that the catalog ACL match: +# path ~ ^/catalog/([^/]+)$ +# method find +# allow $1 +# allow puppet-load.domain.com +# +# 4) launch the master(s) +# +# 5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host +# you want to simulate. +# +# 5) launch puppet-load +# puppet-load -debug --node server1.domain.com --node server2.domain.com --node server3.domain.com \ +# --server master.domain.com --factsdir /var/lib/puppet/yaml/facts --concurrency 2 --repeat 20 +# +# puppet-load will load facts file in the --factsdir directory based on the node name. # # = TODO -# * Allow to simulate any different nodes # * More output stats for error connections (ie report errors, HTTP code...) # # @@ -115,6 +140,7 @@ $cmdargs = [ [ "--concurrency", "-c", GetoptLong::REQUIRED_ARGUMENT ], [ "--node", "-n", GetoptLong::REQUIRED_ARGUMENT ], [ "--facts", GetoptLong::REQUIRED_ARGUMENT ], + [ "--factsdir", GetoptLong::REQUIRED_ARGUMENT ], [ "--repeat", "-r", GetoptLong::REQUIRED_ARGUMENT ], [ "--cert", "-C", GetoptLong::REQUIRED_ARGUMENT ], [ "--key", "-k", GetoptLong::REQUIRED_ARGUMENT ], @@ -131,14 +157,15 @@ Puppet::Util::Log.newdestination(:console) times = {} def read_facts(file) - YAML.load(File.read(file)) + Puppet.debug("reading facts from: #{file}") + fact = YAML.load(File.read(file)) end result = GetoptLong.new(*$cmdargs) $args = {} -$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140} +$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140, :node => [], :factsdir => nil} begin result.each { |opt,arg| @@ -151,7 +178,9 @@ begin exit(14) end when "--node" - $options[:node] = arg + $options[:node] << arg + when "--factsdir" + $options[:factsdir] = arg when "--server" $options[:server] = arg when "--masterport" @@ -192,21 +221,24 @@ unless $options[:cert] and $options[:key] raise "--cert and --key are mandatory to authenticate the client" end -unless $options[:facts] and facts = read_facts($options[:facts]) - unless facts = Puppet::Node::Facts.find($options[:node]) - raise "Could not find facts for %s" % $options[:node] - end -end +parameters = [] -unless $options[:node] +unless $options[:node].size > 0 raise "--node is a mandatory argument. It tells to the master what node to compile" end -facts.values["fqdn"] = $options[:node] -facts.values["hostname"] = $options[:node].sub(/\..+/, '') -facts.values["domain"] = $options[:node].sub(/^[^.]+\./, '') +$options[:node].each do |node| + factfile = $options[:factsdir] ? File.join($options[:factsdir], node + ".yaml") : $options[:facts] + unless fact = read_facts(factfile) or fact = Puppet::Node::Facts.find(node) + raise "Could not find facts for %s" % node + end + fact.values["fqdn"] = node + fact.values["hostname"] = node.sub(/\..+/, '') + fact.values["domain"] = node.sub(/^[^.]+\./, '') + + parameters << {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(fact.render(:b64_zlib_yaml))} +end -parameters = {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(facts.render(:b64_zlib_yaml))} class RequestPool include EventMachine::Deferrable @@ -233,17 +265,19 @@ class RequestPool end def spawn_request(index) - EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{$options[:node]}").get( + nodeidx = index % $options[:node].size + node = $options[:node][nodeidx] + EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{node}").get( :port => $options[:masterport], - :query => @parameters, + :query => @parameters[nodeidx], :timeout => $options[:timeout], :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" }, :ssl => { :private_key_file => $options[:key], :cert_chain_file => $options[:cert], :verify_peer => false } ) do - Puppet.debug("starting client #{index}") @times[index] = Time.now @sizes[index] = 0 + Puppet.debug("starting client #{index} for #{node}") end end -- cgit From f2537d8f913ff78f5198dabd43727660010a4480 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 17 Oct 2010 16:06:50 +0200 Subject: Puppet-load: better and safer error reporting In case of connection, dns or timeout error, puppet-load would not report the error correctly or could crash. Signed-off-by: Brice Figureau --- ext/puppet-load.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb index 24437c10b..35bee6ef8 100644 --- a/ext/puppet-load.rb +++ b/ext/puppet-load.rb @@ -265,6 +265,8 @@ class RequestPool end def spawn_request(index) + @times[index] = Time.now + @sizes[index] = 0 nodeidx = index % $options[:node].size node = $options[:node][nodeidx] EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{node}").get( @@ -302,7 +304,7 @@ class RequestPool } conn.errback { - Puppet.debug("Client #{index} finished with an error: #{conn.response.error}") + Puppet.debug("Client #{index} finished with an error: #{conn.error}") @times[index] = Time.now - @times[index] @responses[:failed].push(conn) check_progress -- cgit From 3b53bfcd0dd20a43c751b2b0d5dbb0e3463ef47b Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Mon, 18 Oct 2010 14:25:17 -0700 Subject: Fix for #5022 -- Escaped newlines should be elided This was a regression, not covered by a test; previously the string "foo\ bar" would be interpreded as "foobar" but this was changed to "foo\\\nbar" in 2.6.x with my string interpolation refactor. This change restores the behaviour. --- lib/puppet/parser/lexer.rb | 3 ++- spec/unit/parser/lexer_spec.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb index 9036d652e..31d39ae2f 100644 --- a/lib/puppet/parser/lexer.rb +++ b/lib/puppet/parser/lexer.rb @@ -522,13 +522,14 @@ class Puppet::Parser::Lexer # backslash; the caret is there to match empty strings str = @scanner.scan_until(/([^\\]|^|[^\\])([\\]{2})*[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'" @line += str.count("\n") # literal carriage returns add to the line count. - str.gsub!(/\\(.)/) { + str.gsub!(/\\(.)/m) { ch = $1 if escapes.include? ch case ch when 'n'; "\n" when 't'; "\t" when 's'; " " + when "\n": '' else ch end else diff --git a/spec/unit/parser/lexer_spec.rb b/spec/unit/parser/lexer_spec.rb index 2d67bf357..d52add399 100755 --- a/spec/unit/parser/lexer_spec.rb +++ b/spec/unit/parser/lexer_spec.rb @@ -424,6 +424,7 @@ describe Puppet::Parser::Lexer,"when lexing strings" do %q{'single quoted string with an escaped "\\\\"'} => [[:STRING,'single quoted string with an escaped "\\\\"']], %q{"string with an escaped '\\"'"} => [[:STRING,"string with an escaped '\"'"]], %q{"string with an escaped '\\$'"} => [[:STRING,"string with an escaped '$'"]], + %Q{"string with a line ending with a backslash: \\\nfoo"} => [[:STRING,"string with a line ending with a backslash: foo"]], %q{"string with $v (but no braces)"} => [[:DQPRE,"string with "],[:VARIABLE,'v'],[:DQPOST,' (but no braces)']], %q["string with ${v} in braces"] => [[:DQPRE,"string with "],[:VARIABLE,'v'],[:DQPOST,' in braces']], %q["string with ${qualified::var} in braces"] => [[:DQPRE,"string with "],[:VARIABLE,'qualified::var'],[:DQPOST,' in braces']], -- cgit From 4d1681e05a187e55422bf00de5c932736548be15 Mon Sep 17 00:00:00 2001 From: Jeff McCune Date: Wed, 20 Oct 2010 15:22:06 -0700 Subject: (#5062) Add envpuppet helper script to ext/ This script helps people quickly test different branches of various puppet related projects like facter, puppet-scaffold, puppet-dashboard, etc... It allows the user to simply clone puppet and start running different puppet version without installing puppet or any other setup costs. --- ext/envpuppet | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 ext/envpuppet diff --git a/ext/envpuppet b/ext/envpuppet new file mode 100755 index 000000000..d921a19b8 --- /dev/null +++ b/ext/envpuppet @@ -0,0 +1,80 @@ +#! /bin/bash +# +# Jeff McCune +# 2010-10-20 +# +# Copyright (c) 2010, Puppet Labs +# License: BSD 3-clause license +# +# This script provides a simple way to execute puppet and related tools +# directly from a git clone of the upstream repositories. This allows you to +# quickly switch branches and test different versions of code without much +# friction. +# +# NOTE: There may be issues if puppet, facter, etc... are already installed +# into RUBY's site_ruby directory. If you run into strange problems, make sure +# the correct ruby libraries are being loaded... +# +# Sample Usage: +# ============= +# cd ~/src +# git clone git://github.com/puppetlabs/puppet.git +# git clone git://github.com/puppetlabs/facter.git +# pushd puppet +# git checkout tags/2.6.1 +# popd +# pushd facter +# git checkout tags/1.5.8 +# export ENVPUPPET_BASEDIR=/home/jeff/src +# envpuppet puppet --version +# 2.6.1 +# envpuppet facter --version +# 1.5.8 + +set -e +set -u + +if test -d puppet -o -d facter; then + echo " WARNING!" + echo " Strange things happen if puppet or facter are in the" + echo " current working directory" + echo " (import errors from ruby are a prime example)" + echo " WARNING!" + echo "" + echo "I suggest changing to ~ or /tmp or something..." + echo "" + echo "Sleeping 2 seconds." + echo "" + sleep 2 +fi + +# Set this to where you check out puppet and facter +: ${ENVPUPPET_BASEDIR:="${HOME}/src"} + +# git://github.com/reductivelabs/puppet.git +mypath="${ENVPUPPET_BASEDIR}/puppet/sbin:${ENVPUPPET_BASEDIR}/puppet/bin" +myrubylib="${ENVPUPPET_BASEDIR}/puppet/lib" + +# git://github.com/reductivelabs/facter.git +mypath="${mypath}:${ENVPUPPET_BASEDIR}/facter/bin" +myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/facter/lib" + +# http://github.com/jamtur01/puppet-scaffold.git +mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-scaffold/bin" +myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-scaffold/lib" + +# http://github.com/puppetlabs/puppet-module-tool.git +# Also known as "pmt" Will become "puppet module" +mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-module-tool/bin" +myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-module-tool/lib" + +# Use the existing environment, if present. +# Default to no value to prevent unbound variable issues +mypath="${mypath}:${PATH:-}" +myrubylib="${myrubylib}:${RUBYLIB:-}" + +# Trim any trailing colons from the path list. +export PATH="${mypath%%:}" +export RUBYLIB="${myrubylib%%:}" + +exec "$@" -- cgit From 7d35a479b760e382638a2efe1881b8bd94704a45 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Wed, 27 Oct 2010 09:23:00 +1100 Subject: Fixed to #5108 - Change default of service hasstatus property to true --- lib/puppet/type/service.rb | 2 ++ spec/unit/type/service_spec.rb | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/lib/puppet/type/service.rb b/lib/puppet/type/service.rb index c00f02789..786a50448 100644 --- a/lib/puppet/type/service.rb +++ b/lib/puppet/type/service.rb @@ -100,6 +100,8 @@ module Puppet looked for in the process table." newvalues(:true, :false) + + defaultto :true end newparam(:name) do desc "The name of the service to run. This name is used to find diff --git a/spec/unit/type/service_spec.rb b/spec/unit/type/service_spec.rb index 0958a69fa..77628670a 100755 --- a/spec/unit/type/service_spec.rb +++ b/spec/unit/type/service_spec.rb @@ -66,6 +66,10 @@ describe Puppet::Type.type(:service), "when validating attribute values" do Puppet::Type.type(:service).new(:name => "yay", :hasstatus => :false) end + it "should specify :true as the default value of hasstatus" do + Puppet::Type.type(:service).new(:name => "yay")[:hasstatus].should == :true + end + it "should support :true as a value to :hasrestart" do Puppet::Type.type(:service).new(:name => "yay", :hasrestart => :true) end -- cgit From 65ef24e5c1c33b7d42012891d368917fd6aaf68c Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 8 Oct 2010 15:26:28 -0700 Subject: (#4534/#4778) -- Normalize parameterized classes This is a reconciliation/melding of Paul's (#4534) Class inheritance with parameterized classes is no longer ignored and Markus's Fix for #4778 -- evaluate parameterized classes when they are instantiated Extracted the code from Resource::Type#mk_plain_resource that evaluates parents and tags the catalog, and moved that into a new method called instantiate_resource. Instantiate_resource is now also called from Parser::Ast::Resource#evaluate, so that the notation "class { classname: }" now executes this code too. Likewise adds class evaluation so that it behaves the same (with regard to lazy / strict evaluation) as include classname --- lib/puppet/parser/ast/resource.rb | 7 +- lib/puppet/parser/compiler.rb | 4 +- lib/puppet/resource/type.rb | 31 +++-- spec/unit/parser/ast/resource_spec.rb | 221 ++++++++++++++++++++-------------- spec/unit/parser/compiler_spec.rb | 16 +-- spec/unit/resource/type_spec.rb | 26 ++-- 6 files changed, 178 insertions(+), 127 deletions(-) diff --git a/lib/puppet/parser/ast/resource.rb b/lib/puppet/parser/ast/resource.rb index 6909c85c2..b019e6aac 100644 --- a/lib/puppet/parser/ast/resource.rb +++ b/lib/puppet/parser/ast/resource.rb @@ -49,10 +49,11 @@ class Resource < AST::ResourceReference :strict => true ) - # And then store the resource in the compiler. - # At some point, we need to switch all of this to return - # resources instead of storing them like this. + if resource.resource_type.is_a? Puppet::Resource::Type + resource.resource_type.instantiate_resource(scope, resource) + end scope.compiler.add_resource(scope, resource) + scope.compiler.evaluate_classes([resource_title],scope,false) if fully_qualified_type == 'class' resource end }.reject { |resource| resource.nil? } diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index e1227e753..c60e1d4fb 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -144,7 +144,7 @@ class Puppet::Parser::Compiler if klass = scope.find_hostclass(name) found << name and next if scope.class_scope(klass) - resource = klass.mk_plain_resource(scope) + resource = klass.ensure_in_catalog(scope) # If they've disabled lazy evaluation (which the :include function does), # then evaluate our resource immediately. @@ -220,7 +220,7 @@ class Puppet::Parser::Compiler # Create a resource to model this node, and then add it to the list # of resources. - resource = astnode.mk_plain_resource(topscope) + resource = astnode.ensure_in_catalog(topscope) resource.evaluate diff --git a/lib/puppet/resource/type.rb b/lib/puppet/resource/type.rb index 7b21e55dc..d40adc145 100644 --- a/lib/puppet/resource/type.rb +++ b/lib/puppet/resource/type.rb @@ -138,21 +138,15 @@ class Puppet::Resource::Type end end - # Make an instance of our resource type. This is only possible - # for those classes and nodes that don't have any arguments, and is - # only useful for things like the 'include' function. - def mk_plain_resource(scope) + # Make an instance of the resource type, and place it in the catalog + # if it isn't in the catalog already. This is only possible for + # classes and nodes. No parameters are be supplied--if this is a + # parameterized class, then all parameters take on their default + # values. + def ensure_in_catalog(scope) type == :definition and raise ArgumentError, "Cannot create resources for defined resource types" resource_type = type == :hostclass ? :class : :node - # Make sure our parent class has been evaluated, if we have one. - if parent - parent_resource = scope.catalog.resource(resource_type, parent) - unless parent_resource - parent_type(scope).mk_plain_resource(scope) - end - end - # Do nothing if the resource already exists; this makes sure we don't # get multiple copies of the class resource, which helps provide the # singleton nature of classes. @@ -161,11 +155,22 @@ class Puppet::Resource::Type end resource = Puppet::Parser::Resource.new(resource_type, name, :scope => scope, :source => self) + instantiate_resource(scope, resource) scope.compiler.add_resource(scope, resource) - scope.catalog.tag(*resource.tags) resource end + def instantiate_resource(scope, resource) + # Make sure our parent class has been evaluated, if we have one. + if parent && !scope.catalog.resource(resource.type, parent) + parent_type(scope).ensure_in_catalog(scope) + end + + if ['Class', 'Node'].include? resource.type + scope.catalog.tag(*resource.tags) + end + end + def name return @name unless @name.is_a?(Regexp) @name.source.downcase.gsub(/[^-\w:.]/,'').sub(/^\.+/,'') diff --git a/spec/unit/parser/ast/resource_spec.rb b/spec/unit/parser/ast/resource_spec.rb index 5c94ac0e9..e1f73e161 100755 --- a/spec/unit/parser/ast/resource_spec.rb +++ b/spec/unit/parser/ast/resource_spec.rb @@ -5,123 +5,168 @@ require File.dirname(__FILE__) + '/../../../spec_helper' describe Puppet::Parser::AST::Resource do ast = Puppet::Parser::AST - before :each do - @title = Puppet::Parser::AST::String.new(:value => "mytitle") - @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("mynode")) - @scope = Puppet::Parser::Scope.new(:compiler => @compiler) - @scope.stubs(:resource).returns(stub_everything) - @resource = ast::Resource.new(:title => @title, :type => "file", :parameters => ast::ASTArray.new(:children => []) ) - @resource.stubs(:qualified_type).returns("Resource") - end - - it "should evaluate all its parameters" do - param = stub 'param' - param.expects(:safeevaluate).with(@scope).returns Puppet::Parser::Resource::Param.new(:name => "myparam", :value => "myvalue", :source => stub("source")) - @resource.stubs(:parameters).returns [param] - - @resource.evaluate(@scope) - end - - it "should evaluate its title" do - @resource.evaluate(@scope)[0].title.should == "mytitle" - end - - it "should flatten the titles array" do - titles = [] - %w{one two}.each do |title| - titles << Puppet::Parser::AST::String.new(:value => title) + describe "for builtin types" do + before :each do + @title = Puppet::Parser::AST::String.new(:value => "mytitle") + @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("mynode")) + @scope = Puppet::Parser::Scope.new(:compiler => @compiler) + @scope.stubs(:resource).returns(stub_everything) + @resource = ast::Resource.new(:title => @title, :type => "file", :parameters => ast::ASTArray.new(:children => []) ) + @resource.stubs(:qualified_type).returns("Resource") end - array = Puppet::Parser::AST::ASTArray.new(:children => titles) + it "should evaluate all its parameters" do + param = stub 'param' + param.expects(:safeevaluate).with(@scope).returns Puppet::Parser::Resource::Param.new(:name => "myparam", :value => "myvalue", :source => stub("source")) + @resource.stubs(:parameters).returns [param] - @resource.title = array - result = @resource.evaluate(@scope).collect { |r| r.title } - result.should be_include("one") - result.should be_include("two") - end + @resource.evaluate(@scope) + end - it "should create and return one resource objects per title" do - titles = [] - %w{one two}.each do |title| - titles << Puppet::Parser::AST::String.new(:value => title) + it "should evaluate its title" do + @resource.evaluate(@scope)[0].title.should == "mytitle" end - array = Puppet::Parser::AST::ASTArray.new(:children => titles) + it "should flatten the titles array" do + titles = [] + %w{one two}.each do |title| + titles << Puppet::Parser::AST::String.new(:value => title) + end - @resource.title = array - result = @resource.evaluate(@scope).collect { |r| r.title } - result.should be_include("one") - result.should be_include("two") - end + array = Puppet::Parser::AST::ASTArray.new(:children => titles) - it "should handover resources to the compiler" do - titles = [] - %w{one two}.each do |title| - titles << Puppet::Parser::AST::String.new(:value => title) + @resource.title = array + result = @resource.evaluate(@scope).collect { |r| r.title } + result.should be_include("one") + result.should be_include("two") end - array = Puppet::Parser::AST::ASTArray.new(:children => titles) + it "should create and return one resource objects per title" do + titles = [] + %w{one two}.each do |title| + titles << Puppet::Parser::AST::String.new(:value => title) + end - @resource.title = array - result = @resource.evaluate(@scope) + array = Puppet::Parser::AST::ASTArray.new(:children => titles) - result.each do |res| - @compiler.catalog.resource(res.ref).should be_instance_of(Puppet::Parser::Resource) + @resource.title = array + result = @resource.evaluate(@scope).collect { |r| r.title } + result.should be_include("one") + result.should be_include("two") end - end - it "should generate virtual resources if it is virtual" do - @resource.virtual = true - result = @resource.evaluate(@scope) - result[0].should be_virtual - end + it "should handover resources to the compiler" do + titles = [] + %w{one two}.each do |title| + titles << Puppet::Parser::AST::String.new(:value => title) + end - it "should generate virtual and exported resources if it is exported" do - @resource.exported = true + array = Puppet::Parser::AST::ASTArray.new(:children => titles) - result = @resource.evaluate(@scope) - result[0].should be_virtual - result[0].should be_exported - end + @resource.title = array + result = @resource.evaluate(@scope) - # Related to #806, make sure resources always look up the full path to the resource. - describe "when generating qualified resources" do - before do - @scope = Puppet::Parser::Scope.new :compiler => Puppet::Parser::Compiler.new(Puppet::Node.new("mynode")) - @parser = Puppet::Parser::Parser.new(Puppet::Node::Environment.new) - @parser.newdefine "one" - @parser.newdefine "one::two" - @parser.newdefine "three" - @twoscope = @scope.newscope(:namespace => "one") - @twoscope.resource = @scope.resource + result.each do |res| + @compiler.catalog.resource(res.ref).should be_instance_of(Puppet::Parser::Resource) + end end - def resource(type, params = nil) - params ||= Puppet::Parser::AST::ASTArray.new(:children => []) - Puppet::Parser::AST::Resource.new(:type => type, :title => Puppet::Parser::AST::String.new(:value => "myresource"), :parameters => params) + it "should generate virtual resources if it is virtual" do + @resource.virtual = true + + result = @resource.evaluate(@scope) + result[0].should be_virtual end - it "should be able to generate resources with fully qualified type information" do - resource("two").evaluate(@twoscope)[0].type.should == "One::Two" + it "should generate virtual and exported resources if it is exported" do + @resource.exported = true + + result = @resource.evaluate(@scope) + result[0].should be_virtual + result[0].should be_exported end - it "should be able to generate resources with unqualified type information" do - resource("one").evaluate(@twoscope)[0].type.should == "One" + # Related to #806, make sure resources always look up the full path to the resource. + describe "when generating qualified resources" do + before do + @scope = Puppet::Parser::Scope.new :compiler => Puppet::Parser::Compiler.new(Puppet::Node.new("mynode")) + @parser = Puppet::Parser::Parser.new(Puppet::Node::Environment.new) + @parser.newdefine "one" + @parser.newdefine "one::two" + @parser.newdefine "three" + @twoscope = @scope.newscope(:namespace => "one") + @twoscope.resource = @scope.resource + end + + def resource(type, params = nil) + params ||= Puppet::Parser::AST::ASTArray.new(:children => []) + Puppet::Parser::AST::Resource.new(:type => type, :title => Puppet::Parser::AST::String.new(:value => "myresource"), :parameters => params) + end + + it "should be able to generate resources with fully qualified type information" do + resource("two").evaluate(@twoscope)[0].type.should == "One::Two" + end + + it "should be able to generate resources with unqualified type information" do + resource("one").evaluate(@twoscope)[0].type.should == "One" + end + + it "should correctly generate resources that can look up builtin types" do + resource("file").evaluate(@twoscope)[0].type.should == "File" + end + + it "should correctly generate resources that can look up defined classes by title" do + @scope.known_resource_types.add_hostclass Puppet::Resource::Type.new(:hostclass, "Myresource", {}) + @scope.compiler.stubs(:evaluate_classes) + res = resource("class").evaluate(@twoscope)[0] + res.type.should == "Class" + res.title.should == "Myresource" + end + + it "should evaluate parameterized classes when they are instantiated" do + @scope.known_resource_types.add_hostclass Puppet::Resource::Type.new(:hostclass, "Myresource", {}) + @scope.compiler.expects(:evaluate_classes).with(['myresource'],@twoscope,false) + resource("class").evaluate(@twoscope)[0] + end + + it "should fail for resource types that do not exist" do + lambda { resource("nosuchtype").evaluate(@twoscope) }.should raise_error(Puppet::ParseError) + end end + end - it "should correctly generate resources that can look up builtin types" do - resource("file").evaluate(@twoscope)[0].type.should == "File" + describe "for class resources" do + before do + @title = Puppet::Parser::AST::String.new(:value => "classname") + @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("mynode")) + @scope = Puppet::Parser::Scope.new(:compiler => @compiler) + @scope.stubs(:resource).returns(stub_everything) + @resource = ast::Resource.new(:title => @title, :type => "Class", :parameters => ast::ASTArray.new(:children => []) ) + @resource.stubs(:qualified_type).returns("Resource") + @type = Puppet::Resource::Type.new(:hostclass, "classname") + @compiler.known_resource_types.add(@type) end - it "should correctly generate resources that can look up defined classes by title" do - @scope.known_resource_types.add_hostclass Puppet::Resource::Type.new(:hostclass, "Myresource", {}) - res = resource("class").evaluate(@twoscope)[0] - res.type.should == "Class" - res.title.should == "Myresource" + it "should instantiate the class" do + @compiler.stubs(:evaluate_classes) + result = @resource.evaluate(@scope) + result.length.should == 1 + result.first.ref.should == "Class[Classname]" + @compiler.catalog.resource("Class[Classname]").should equal result.first end - it "should fail for resource types that do not exist" do - lambda { resource("nosuchtype").evaluate(@twoscope) }.should raise_error(Puppet::ParseError) + it "should cause its parent to be evaluated" do + parent_type = Puppet::Resource::Type.new(:hostclass, "parentname") + @compiler.stubs(:evaluate_classes) + @compiler.known_resource_types.add(parent_type) + @type.parent = "parentname" + result = @resource.evaluate(@scope) + result.length.should == 1 + result.first.ref.should == "Class[Classname]" + @compiler.catalog.resource("Class[Classname]").should equal result.first + @compiler.catalog.resource("Class[Parentname]").should be_instance_of(Puppet::Parser::Resource) end + end + end diff --git a/spec/unit/parser/compiler_spec.rb b/spec/unit/parser/compiler_spec.rb index 22d52f257..95f3853e2 100755 --- a/spec/unit/parser/compiler_spec.rb +++ b/spec/unit/parser/compiler_spec.rb @@ -580,7 +580,7 @@ describe Puppet::Parser::Compiler do it "should evaluate each class" do @compiler.catalog.stubs(:tag) - @class.expects(:mk_plain_resource).with(@scope) + @class.expects(:ensure_in_catalog).with(@scope) @scope.stubs(:class_scope).with(@class) @compiler.evaluate_classes(%w{myclass}, @scope) @@ -591,7 +591,7 @@ describe Puppet::Parser::Compiler do @resource.expects(:evaluate).never - @class.expects(:mk_plain_resource).returns(@resource) + @class.expects(:ensure_in_catalog).returns(@resource) @scope.stubs(:class_scope).with(@class) @compiler.evaluate_classes(%w{myclass}, @scope) @@ -601,7 +601,7 @@ describe Puppet::Parser::Compiler do @compiler.catalog.stubs(:tag) @resource.expects(:evaluate) - @class.expects(:mk_plain_resource).returns(@resource) + @class.expects(:ensure_in_catalog).returns(@resource) @scope.stubs(:class_scope).with(@class) @compiler.evaluate_classes(%w{myclass}, @scope, false) @@ -638,7 +638,7 @@ describe Puppet::Parser::Compiler do @scope.stubs(:class_scope).with(@class) Puppet::Parser::Resource.stubs(:new).returns(@resource) - @class.stubs :mk_plain_resource + @class.stubs :ensure_in_catalog @compiler.evaluate_classes(%w{myclass notfound}, @scope).should == %w{myclass} end end @@ -678,7 +678,7 @@ describe Puppet::Parser::Compiler do @compiler.known_resource_types.stubs(:node).with("c").returns(node_class) node_resource = stub 'node resource', :ref => "Node[c]", :evaluate => nil, :type => "node" - node_class.expects(:mk_plain_resource).returns(node_resource) + node_class.expects(:ensure_in_catalog).returns(node_resource) @compiler.compile end @@ -688,7 +688,7 @@ describe Puppet::Parser::Compiler do @compiler.known_resource_types.stubs(:node).with("default").returns(node_class) node_resource = stub 'node resource', :ref => "Node[default]", :evaluate => nil, :type => "node" - node_class.expects(:mk_plain_resource).returns(node_resource) + node_class.expects(:ensure_in_catalog).returns(node_resource) @compiler.compile end @@ -698,7 +698,7 @@ describe Puppet::Parser::Compiler do @compiler.known_resource_types.stubs(:node).with("c").returns(node_class) node_resource = stub 'node resource', :ref => "Node[c]", :type => "node" - node_class.expects(:mk_plain_resource).returns(node_resource) + node_class.expects(:ensure_in_catalog).returns(node_resource) node_resource.expects(:evaluate) @@ -707,7 +707,7 @@ describe Puppet::Parser::Compiler do it "should set the node's scope as the top scope" do node_resource = stub 'node resource', :ref => "Node[c]", :evaluate => nil, :type => "node" - node_class = stub 'node', :name => "c", :mk_plain_resource => node_resource + node_class = stub 'node', :name => "c", :ensure_in_catalog => node_resource @compiler.known_resource_types.stubs(:node).with("c").returns(node_class) diff --git a/spec/unit/resource/type_spec.rb b/spec/unit/resource/type_spec.rb index f58092ec5..7b240bb82 100755 --- a/spec/unit/resource/type_spec.rb +++ b/spec/unit/resource/type_spec.rb @@ -580,29 +580,29 @@ describe Puppet::Resource::Type do end it "should create a resource instance" do - @top.mk_plain_resource(@scope).should be_instance_of(Puppet::Parser::Resource) + @top.ensure_in_catalog(@scope).should be_instance_of(Puppet::Parser::Resource) end it "should set its resource type to 'class' when it is a hostclass" do - Puppet::Resource::Type.new(:hostclass, "top").mk_plain_resource(@scope).type.should == "Class" + Puppet::Resource::Type.new(:hostclass, "top").ensure_in_catalog(@scope).type.should == "Class" end it "should set its resource type to 'node' when it is a node" do - Puppet::Resource::Type.new(:node, "top").mk_plain_resource(@scope).type.should == "Node" + Puppet::Resource::Type.new(:node, "top").ensure_in_catalog(@scope).type.should == "Node" end it "should fail when it is a definition" do - lambda { Puppet::Resource::Type.new(:definition, "top").mk_plain_resource(@scope) }.should raise_error(ArgumentError) + lambda { Puppet::Resource::Type.new(:definition, "top").ensure_in_catalog(@scope) }.should raise_error(ArgumentError) end it "should add the created resource to the scope's catalog" do - @top.mk_plain_resource(@scope) + @top.ensure_in_catalog(@scope) @compiler.catalog.resource(:class, "top").should be_instance_of(Puppet::Parser::Resource) end it "should evaluate the parent class if one exists" do - @middle.mk_plain_resource(@scope) + @middle.ensure_in_catalog(@scope) @compiler.catalog.resource(:class, "top").should be_instance_of(Puppet::Parser::Resource) end @@ -610,40 +610,40 @@ describe Puppet::Resource::Type do it "should fail to evaluate if a parent class is defined but cannot be found" do othertop = Puppet::Resource::Type.new :hostclass, "something", :parent => "yay" @code.add othertop - lambda { othertop.mk_plain_resource(@scope) }.should raise_error(Puppet::ParseError) + lambda { othertop.ensure_in_catalog(@scope) }.should raise_error(Puppet::ParseError) end it "should not create a new resource if one already exists" do @compiler.catalog.expects(:resource).with(:class, "top").returns("something") @compiler.catalog.expects(:add_resource).never - @top.mk_plain_resource(@scope) + @top.ensure_in_catalog(@scope) end it "should return the existing resource when not creating a new one" do @compiler.catalog.expects(:resource).with(:class, "top").returns("something") @compiler.catalog.expects(:add_resource).never - @top.mk_plain_resource(@scope).should == "something" + @top.ensure_in_catalog(@scope).should == "something" end it "should not create a new parent resource if one already exists and it has a parent class" do - @top.mk_plain_resource(@scope) + @top.ensure_in_catalog(@scope) top_resource = @compiler.catalog.resource(:class, "top") - @middle.mk_plain_resource(@scope) + @middle.ensure_in_catalog(@scope) @compiler.catalog.resource(:class, "top").should equal(top_resource) end # #795 - tag before evaluation. it "should tag the catalog with the resource tags when it is evaluated" do - @middle.mk_plain_resource(@scope) + @middle.ensure_in_catalog(@scope) @compiler.catalog.should be_tagged("middle") end it "should tag the catalog with the parent class tags when it is evaluated" do - @middle.mk_plain_resource(@scope) + @middle.ensure_in_catalog(@scope) @compiler.catalog.should be_tagged("top") end -- cgit From 31118fe85aca4ee46903b17a3eb7aee07b8c0d69 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sat, 23 Oct 2010 08:26:24 -0700 Subject: Kludge for #5048 -- serialization compatibility with 0.25.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 0.25.x the type & title of a resource were wrapped in a Puppet::Resource::Reference object whereas in 2.6.x they are attributes of the resource itself without the additional indirection (see 7089446697ad550c22012bc2b5572030727d67e1). When pson serialization is used this isn’t a problem but with formats in which we just blindly emit the structure either because we have no choice (marshal) or because we just use the default (yaml) it is a compatibility-breaking change. This patch resoloves the problem by adding a dummy reference object to cause the "correct" serialization; it is intended as a stop-gap for 2.6.x and should NOT be merged into next. --- lib/puppet/resource.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/puppet/resource.rb b/lib/puppet/resource.rb index 39803b077..7dea270e8 100644 --- a/lib/puppet/resource.rb +++ b/lib/puppet/resource.rb @@ -154,6 +154,14 @@ class Puppet::Resource end end + # This stub class is only needed for serialization compatibility with 0.25.x + class Reference + attr_accessor :type,:title + def initialize(type,title) + @type,@title = type,title + end + end + # Create our resource. def initialize(type, title = nil, attributes = {}) @parameters = {} @@ -180,6 +188,8 @@ class Puppet::Resource tag(self.type) tag(self.title) if valid_tag?(self.title) + @reference = Reference.new(@type,@title) # for serialization compatibility with 0.25.x + raise ArgumentError, "Invalid resource type #{type}" if strict? and ! resource_type end -- cgit From 776ea2a17de7834ecdaded9fcaabc48446d2f29d Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Thu, 28 Oct 2010 20:10:59 +1100 Subject: Fixed #5137 - Removed no longer required TOC references --- lib/puppet/util/reference.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/puppet/util/reference.rb b/lib/puppet/util/reference.rb index 99458aa57..ab201cde4 100644 --- a/lib/puppet/util/reference.rb +++ b/lib/puppet/util/reference.rb @@ -32,7 +32,6 @@ class Puppet::Util::Reference section = reference(name) or raise "Could not find section #{name}" depth = section.depth if section.depth < depth end - text = "* TOC text.\n{:toc}\n\n" end def self.pdf(text) @@ -141,7 +140,6 @@ class Puppet::Util::Reference # First the header text = h(@title, 1) text += "\n\n**This page is autogenerated; any changes will get overwritten** *(last generated on #{Time.now.to_s})*\n\n" - text += "* TOC Text.\n{:toc}\n\n" if withcontents text += @header -- cgit From 76ac1f88ce2a13cabbda38eb56ed21a43e6923a7 Mon Sep 17 00:00:00 2001 From: donavan Date: Wed, 27 Oct 2010 17:57:03 -0700 Subject: Fixed #5112 - Launchd Service broke in 2.6.2 with OS X 10.4 Clients. Just to follow up on 5112 I have a dirty patch that appears to work. Nominally tested it on 10.4, 10.5, & 10.6. 10.4 now applies catalogs instead of failing. All versions successfully manage a test services state as well. Does anyone have a better suggestion than '-o /dev/stdout'? Seems a mite hacky to me. Also I think that the 10.4 machines are going to a have a \ ( slash ) file in whatever puppets working dir was. plutil seems to have been interpreting as literal file name. --- lib/puppet/provider/service/launchd.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/provider/service/launchd.rb b/lib/puppet/provider/service/launchd.rb index b296e0a38..1632edabf 100644 --- a/lib/puppet/provider/service/launchd.rb +++ b/lib/puppet/provider/service/launchd.rb @@ -56,7 +56,7 @@ Puppet::Type.type(:service).provide :launchd, :parent => :base do # Read a plist, whether its format is XML or in Apple's "binary1" # format. def self.read_plist(path) - Plist::parse_xml(plutil('-convert', 'xml1', '-o', '-', path)) + Plist::parse_xml(plutil('-convert', 'xml1', '-o', '/dev/stdout', path)) end # returns a label => path map for either all jobs, or just a single -- cgit From 244213c856f3753dd3e06c5f01003f6a0d3f77ef Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 29 Oct 2010 08:57:24 +1100 Subject: Updated CHANGELOG for 2.6.3rc2 --- CHANGELOG | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 1a28fd8e6..e708b2b6b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,14 @@ +2.6.3rc2 +======== +76ac1f8 Fixed #5112 - Launchd Service broke in 2.6.2 with OS X 10.4 Clients. +776ea2a Fixed #5137 - Removed no longer required TOC references +31118fe Kludge for #5048 -- serialization compatibility with 0.25.x +65ef24e (#4534/#4778) -- Normalize parameterized classes +3b53bfc Fix for #5022 -- Escaped newlines should be elided + 2.6.3rc1 ======== +e3fc5b9 Updated CHANGELOG and version for 2.6.3rc1 3c56705 Fix for #4832 -- Making PSON handle arbitrary binary data e232770 Minimal fix for #4975 -- only call chage when managing password age rules a090e86 Fix for #4963 -- Use correct commands for password expiry on solaris -- cgit From 91ac1629dbefa70382636d6e08768038aeb7f298 Mon Sep 17 00:00:00 2001 From: Garrett Honeycutt Date: Wed, 3 Nov 2010 16:36:43 -0700 Subject: (#5198) add gigabyte reference to docs for tidy type's size parameter --- lib/puppet/type/tidy.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/puppet/type/tidy.rb b/lib/puppet/type/tidy.rb index 65cc077cf..897ee90da 100755 --- a/lib/puppet/type/tidy.rb +++ b/lib/puppet/type/tidy.rb @@ -141,9 +141,10 @@ Puppet::Type.newtype(:tidy) do newparam(:size) do desc "Tidy files whose size is equal to or greater than the specified size. Unqualified values are in kilobytes, but - *b*, *k*, and *m* can be appended to specify *bytes*, *kilobytes*, - and *megabytes*, respectively. Only the first character is - significant, so the full word can also be used." + *b*, *k*, *m*, and *g* can be appended to specify *bytes*, + *kilobytes*, *megabytes*, and *gigabytes*, respectively. + Only the first character is significant, so the full word can also + be used." @@sizeconvertors = { :b => 0, -- cgit From 71a0ceadffc816c08dd979e139fbbf2fa94023a0 Mon Sep 17 00:00:00 2001 From: Garrett Honeycutt Date: Wed, 3 Nov 2010 16:38:06 -0700 Subject: (#5198) add terabyte support to tidy type's size parameter --- lib/puppet/type/tidy.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/puppet/type/tidy.rb b/lib/puppet/type/tidy.rb index 897ee90da..93a7e96cf 100755 --- a/lib/puppet/type/tidy.rb +++ b/lib/puppet/type/tidy.rb @@ -141,8 +141,8 @@ Puppet::Type.newtype(:tidy) do newparam(:size) do desc "Tidy files whose size is equal to or greater than the specified size. Unqualified values are in kilobytes, but - *b*, *k*, *m*, and *g* can be appended to specify *bytes*, - *kilobytes*, *megabytes*, and *gigabytes*, respectively. + *b*, *k*, *m*, *g*, and *t* can be appended to specify *bytes*, + *kilobytes*, *megabytes*, *gigabytes*, and *terabytes*, respectively. Only the first character is significant, so the full word can also be used." @@ -150,7 +150,8 @@ Puppet::Type.newtype(:tidy) do :b => 0, :k => 1, :m => 2, - :g => 3 + :g => 3, + :t => 4 } def convert(unit, multi) -- cgit From 4506dfeab3eead32f44f4baa461ebba88fe098ab Mon Sep 17 00:00:00 2001 From: Matt Robinson Date: Fri, 5 Nov 2010 11:37:27 -0700 Subject: (#5150) Make fact REST terminus configurable to connect to inventory service Puppet masters can now set the inventory_server and inventory_port option to point to another puppet master that will function as the central inventory service. When agents connect to the puppet master, this will send fact data from the puppet master over REST to the inventory service. The puppet master itself will still store the client fact data in the local yaml dir by setting the cache class to yaml. Getting puppet masters to talk to each other using certs is difficult. Paired-with: Jesse Wolfe --- lib/puppet/defaults.rb | 19 +++++++++++++++++-- lib/puppet/indirector/facts/rest.rb | 2 ++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/puppet/defaults.rb b/lib/puppet/defaults.rb index c7bebf8f5..7ae553827 100644 --- a/lib/puppet/defaults.rb +++ b/lib/puppet/defaults.rb @@ -115,7 +115,16 @@ module Puppet :node_terminus => ["plain", "Where to find information about nodes."], :catalog_terminus => ["compiler", "Where to get node catalogs. This is useful to change if, for instance, you'd like to pre-compile catalogs and store them in memcached or some other easily-accessed store."], - :facts_terminus => [Puppet.application_name.to_s == "master" ? 'yaml' : 'facter', "The node facts terminus."], + :facts_terminus => { + :default => Puppet.application_name.to_s == "master" ? 'yaml' : 'facter', + :desc => "The node facts terminus.", + :hook => proc do |value| + require 'puppet/node/facts' + if value.to_s == "rest" + Puppet::Node::Facts.cache_class = :yaml + end + end + }, :inventory_terminus => [ "$facts_terminus", "Should usually be the same as the facts terminus" ], :httplog => { :default => "$logdir/http.log", :owner => "root", @@ -579,11 +588,17 @@ module Puppet end }, :report_server => ["$server", - "The server to which to send transaction reports." + "The server to send transaction reports to." ], :report_port => ["$masterport", "The port to communicate with the report_server." ], + :inventory_server => ["$server", + "The server to send facts to." + ], + :inventory_port => ["$masterport", + "The port to communicate with the inventory_server." + ], :report => [false, "Whether to send reports after every transaction." ], diff --git a/lib/puppet/indirector/facts/rest.rb b/lib/puppet/indirector/facts/rest.rb index 07491fc77..e2afa14b2 100644 --- a/lib/puppet/indirector/facts/rest.rb +++ b/lib/puppet/indirector/facts/rest.rb @@ -3,4 +3,6 @@ require 'puppet/indirector/rest' class Puppet::Node::Facts::Rest < Puppet::Indirector::REST desc "Find and save facts about nodes over HTTP via REST." + use_server_setting(:inventory_server) + use_port_setting(:inventory_port) end -- cgit From f8d1427379f16d116c161e7231a26ba5451bc45f Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 5 Nov 2010 11:45:31 -0700 Subject: maint: First draft of cert inspector This script, ext/cert_inspector, takes directory names as command line arguments, searches those directories for SSL certificates, determines the type and contents of each cert, maps relationships between certificates, and prints a summary to standard out. --- ext/cert_inspector | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 ext/cert_inspector diff --git a/ext/cert_inspector b/ext/cert_inspector new file mode 100755 index 000000000..1effcaa04 --- /dev/null +++ b/ext/cert_inspector @@ -0,0 +1,140 @@ +#!/usr/bin/env ruby +require 'openssl' + +class X509Collector + include Enumerable + + def initialize + @collected_data = {} + end + + def interpret_contents(contents) + cls = case contents.split("\n")[0] + when /BEGIN X509 CRL/ then OpenSSL::X509::CRL + when /BEGIN CERTIFICATE REQUEST/ then OpenSSL::X509::Request + when /BEGIN CERTIFICATE/ then OpenSSL::X509::Certificate + when /BEGIN RSA (PRIVATE|PUBLIC) KEY/ then OpenSSL::PKey::RSA + else return nil + end + cls.new(contents) + rescue + nil + end + + def expected_non_x509_files + ['inventory.txt', 'ca.pass', 'serial'] + end + + def investigate_path(path) + if File.directory?(path) + Dir.foreach(path) do |x| + next if ['.', '..'].include? x + investigate_path File.join(path, x) + end + else + contents = File.read path + meaning = interpret_contents contents + unless meaning || expected_non_x509_files.include?(File.basename(path)) + puts "WARNING: file #{path.inspect} could not be interpreted" + end + @collected_data[path] = meaning if meaning + end + end + + def each(&block) + @collected_data.each(&block) + end + + def extract_public_key_info(path, meaning) + case meaning + when OpenSSL::PKey::RSA + if meaning.private? + [meaning.public_key, 2, path] + else + [meaning, 3, path] + end + when OpenSSL::X509::Certificate + [meaning.public_key, 0, meaning.subject.to_s] + when OpenSSL::X509::Request + [meaning.public_key, 1, meaning.subject.to_s] + end + end + + def who_signed(meaning, key_names, keys) + signing_key = keys.find { |key| meaning.verify(key) } + if signing_key then "#{key_names[signing_key.to_s]}" else "???" end + end + + def explain(meaning, key_names, keys) + case meaning + when OpenSSL::PKey::RSA + if meaning.private? + "Private key for #{key_names[meaning.public_key.to_s]}" + else + "Public key for #{key_names[meaning.public_key.to_s]}" + end + when OpenSSL::X509::Certificate + signature_desc = who_signed(meaning, key_names, keys) + "Certificate assigning name #{meaning.subject.to_s} to #{key_names[meaning.public_key.to_s]}\n serial number #{meaning.serial}\n issued by #{meaning.issuer.to_s}\n signed by #{signature_desc}" + when OpenSSL::X509::Request + signature_desc = who_signed(meaning, key_names, keys) + "Certificate request for #{meaning.subject.to_s} having key #{key_names[meaning.public_key.to_s]}\n signed by #{signature_desc}" + when OpenSSL::X509::CRL + signature_desc = who_signed(meaning, key_names, keys) + revoked_serial_numbers = meaning.revoked.map { |r| r.serial } + revoked_desc = if revoked_serial_numbers.count > 0 then "serial numbers #{revoked_serial_numbers.inspect}" else "nothing" end + "Certificate revocation list revoking #{revoked_desc}\n issued by #{meaning.issuer.to_s}\n signed by #{signature_desc}" + else + "Unknown" + end + end + + # Yield unique public keys, with a canonical name for each. + def collect_public_keys + key_data = {} # pem => (priority, name, public_key) + @collected_data.collect do |path, meaning| + begin + next unless public_key_info = extract_public_key_info(path, meaning) + public_key, priority, name = public_key_info + pem = public_key.to_s + existing_priority, existing_name, existing_public_key = key_data[pem] + next if existing_priority and existing_priority < priority + key_data[pem] = priority, name, public_key + rescue + puts "exception!" + end + end + name_to_key_hash = {} + key_data.each do |pem, data| + priority, name, public_key = data + if name_to_key_hash[name] + suffix_num = 2 + while name_to_key_hash[name + " (#{suffix_num})"] + suffix_num += 1 + end + name = name + " (#{suffix_num})" + end + name_to_key_hash[name] = public_key + end + key_names = {} + keys = [] + name_to_key_hash.each do |name, public_key| + key_names[public_key.to_s] = "key<#{name}>" + keys << public_key + end + [key_names, keys] + end +end + +collector = X509Collector.new +ARGV.each do |path| + collector.investigate_path(path) +end +key_names, keys = collector.collect_public_keys +collector.map do |path, meaning| + [collector.explain(meaning, key_names, keys), path] +end.sort.each do |description, path| + puts "#{path}:" + puts " #{description}" + puts +end -- cgit From 5c2457952660e3e531e085757fd85c382676a96e Mon Sep 17 00:00:00 2001 From: Jesse Wolfe Date: Thu, 4 Nov 2010 17:32:11 -0700 Subject: maint: prevent fork bombs by disabling ActiveSupport's Kernel.daemonize ActiveSupport provides a "daemonize" method on all objects that causes the ruby process to fork to the background. This is extremely surprising and dangerous, and some of our spec tests could trigger this accidentally. This patch adds a "daemonize" method to Object which shadows the ActiveSupport version, preventing it from ever being called. --- lib/puppet/util/monkey_patches.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/puppet/util/monkey_patches.rb b/lib/puppet/util/monkey_patches.rb index 6b5af8350..bdce5ec1d 100644 --- a/lib/puppet/util/monkey_patches.rb +++ b/lib/puppet/util/monkey_patches.rb @@ -48,3 +48,11 @@ if RUBY_VERSION == '1.8.7' end end +class Object + # ActiveSupport 2.3.x mixes in a dangerous method + # that can cause rspec to fork bomb + # and other strange things like that. + def daemonize + raise NotImplementedError, "Kernel.daemonize is too dangerous, please don't try to use it." + end +end -- cgit From f0a146766a8425513e3c0cb3e3a1b642e59820b9 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Tue, 9 Nov 2010 11:56:43 -0800 Subject: Maint: remove unnecessary stubbing from agent_spec The agent spec tests were stubbing out all methods related to Puppet settings, making it difficult to keep these tests maintained. The tests now function by setting the settings in question. --- spec/unit/application/agent_spec.rb | 41 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/spec/unit/application/agent_spec.rb b/spec/unit/application/agent_spec.rb index 54726c185..8fc98b8c1 100755 --- a/spec/unit/application/agent_spec.rb +++ b/spec/unit/application/agent_spec.rb @@ -6,6 +6,7 @@ require 'puppet/agent' require 'puppet/application/agent' require 'puppet/network/server' require 'puppet/daemon' +require 'puppet/network/handler' describe Puppet::Application::Agent do before :each do @@ -13,6 +14,7 @@ describe Puppet::Application::Agent do @puppetd.stubs(:puts) @daemon = stub_everything 'daemon' Puppet::Daemon.stubs(:new).returns(@daemon) + Puppet[:daemonize] = false @agent = stub_everything 'agent' Puppet::Agent.stubs(:new).returns(@agent) @puppetd.preinit @@ -175,11 +177,7 @@ describe Puppet::Application::Agent do @puppetd.options.stubs(:[]) Puppet.stubs(:info) FileTest.stubs(:exists?).returns(true) - Puppet.stubs(:[]) - Puppet.stubs(:[]=) - Puppet.stubs(:[]).with(:libdir).returns("/dev/null/lib") - Puppet.settings.stubs(:print_config?) - Puppet.settings.stubs(:print_config) + Puppet[:libdir] = "/dev/null/lib" Puppet::SSL::Host.stubs(:ca_location=) Puppet::Transaction::Report.stubs(:terminus_class=) Puppet::Resource::Catalog.stubs(:terminus_class=) @@ -192,7 +190,7 @@ describe Puppet::Application::Agent do describe "with --test" do before :each do - Puppet.settings.stubs(:handlearg) + #Puppet.settings.stubs(:handlearg) @puppetd.options.stubs(:[]=) end @@ -207,8 +205,9 @@ describe Puppet::Application::Agent do @puppetd.setup_test end it "should set options[:onetime] to true" do - Puppet.expects(:[]=).with(:onetime,true) + Puppet[:onetime] = false @puppetd.setup_test + Puppet[:onetime].should == true end it "should set options[:detailed_exitcodes] to true" do @puppetd.options.expects(:[]=).with(:detailed_exitcodes,true) @@ -264,7 +263,7 @@ describe Puppet::Application::Agent do it "should print puppet config if asked to in Puppet config" do @puppetd.stubs(:exit) - Puppet.settings.stubs(:print_configs?).returns(true) + Puppet[:configprint] = "pluginsync" Puppet.settings.expects(:print_configs) @@ -272,14 +271,14 @@ describe Puppet::Application::Agent do end it "should exit after printing puppet config if asked to in Puppet config" do - Puppet.settings.stubs(:print_configs?).returns(true) + Puppet[:configprint] = "pluginsync" lambda { @puppetd.setup }.should raise_error(SystemExit) end it "should set a central log destination with --centrallogs" do @puppetd.options.stubs(:[]).with(:centrallogs).returns(true) - Puppet.stubs(:[]).with(:server).returns("puppet.reductivelabs.com") + Puppet[:server] = "puppet.reductivelabs.com" Puppet::Util::Log.stubs(:newdestination).with(:syslog) Puppet::Util::Log.expects(:newdestination).with("puppet.reductivelabs.com") @@ -313,8 +312,9 @@ describe Puppet::Application::Agent do end it "should change the catalog_terminus setting to 'rest'" do - Puppet.expects(:[]=).with(:catalog_terminus, :rest) + Puppet[:catalog_terminus] = :foo @puppetd.setup + Puppet[:catalog_terminus].should == :rest end it "should tell the catalog handler to use cache" do @@ -324,9 +324,10 @@ describe Puppet::Application::Agent do end it "should change the facts_terminus setting to 'facter'" do - Puppet.expects(:[]=).with(:facts_terminus, :facter) + Puppet[:facts_terminus] = :foo @puppetd.setup + Puppet[:facts_terminus].should == :facter end it "should create an agent" do @@ -374,7 +375,7 @@ describe Puppet::Application::Agent do end it "should daemonize if needed" do - Puppet.stubs(:[]).with(:daemonize).returns(true) + Puppet[:daemonize] = true @daemon.expects(:daemonize) @@ -397,7 +398,7 @@ describe Puppet::Application::Agent do end it "should setup listen if told to and not onetime" do - Puppet.stubs(:[]).with(:listen).returns(true) + Puppet[:listen] = true @puppetd.options.stubs(:[]).with(:onetime).returns(false) @puppetd.expects(:setup_listen) @@ -407,7 +408,7 @@ describe Puppet::Application::Agent do describe "when setting up listen" do before :each do - Puppet.stubs(:[]).with(:authconfig).returns('auth') + Puppet[:authconfig] = 'auth' FileTest.stubs(:exists?).with('auth').returns(true) File.stubs(:exist?).returns(true) @puppetd.options.stubs(:[]).with(:serve).returns([]) @@ -419,7 +420,7 @@ describe Puppet::Application::Agent do it "should exit if no authorization file" do Puppet.stubs(:err) - FileTest.stubs(:exists?).with('auth').returns(false) + FileTest.stubs(:exists?).with(Puppet[:authconfig]).returns(false) @puppetd.expects(:exit) @@ -440,9 +441,9 @@ describe Puppet::Application::Agent do end it "should use puppet default port" do - Puppet.stubs(:[]).with(:puppetport).returns(:port) + Puppet[:puppetport] = 32768 - Puppet::Network::Server.expects(:new).with { |args| args[:port] == :port } + Puppet::Network::Server.expects(:new).with { |args| args[:port] == 32768 } @puppetd.setup_listen end @@ -521,7 +522,7 @@ describe Puppet::Application::Agent do end it "should exit with report's computed exit status" do - Puppet.stubs(:[]).with(:noop).returns(false) + Puppet[:noop] = false report = stub 'report', :exit_status => 666 @agent.stubs(:run).returns(report) @puppetd.expects(:exit).with(666) @@ -530,7 +531,7 @@ describe Puppet::Application::Agent do end it "should always exit with 0 if --noop" do - Puppet.stubs(:[]).with(:noop).returns(true) + Puppet[:noop] = true report = stub 'report', :exit_status => 666 @agent.stubs(:run).returns(report) @puppetd.expects(:exit).with(0) -- cgit From 2ec1b552e24d3b42391a6471e477422f67ea00bd Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Tue, 9 Nov 2010 12:05:49 -0800 Subject: Maint: Added missing requires to fileserver.rb. --- lib/puppet/network/handler/fileserver.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/puppet/network/handler/fileserver.rb b/lib/puppet/network/handler/fileserver.rb index 27b913ab9..8844ab990 100755 --- a/lib/puppet/network/handler/fileserver.rb +++ b/lib/puppet/network/handler/fileserver.rb @@ -4,9 +4,11 @@ require 'webrick/httpstatus' require 'cgi' require 'delegate' require 'sync' +require 'xmlrpc/server' require 'puppet/file_serving' require 'puppet/file_serving/metadata' +require 'puppet/network/handler' class Puppet::Network::Handler AuthStoreError = Puppet::AuthStoreError -- cgit From 631c5a8e455808dbc919211b5151fd501f6475c9 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Tue, 9 Nov 2010 13:35:59 -0800 Subject: Maint: Add "Local-branch:" info to mails sent by "rake mail_patches" --- tasks/rake/git_workflow.rake | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tasks/rake/git_workflow.rake b/tasks/rake/git_workflow.rake index b2f96c603..c275bba3f 100644 --- a/tasks/rake/git_workflow.rake +++ b/tasks/rake/git_workflow.rake @@ -103,10 +103,21 @@ task :mail_patches do # Create all of the patches sh "git format-patch -C -M -s -n --subject-prefix='PATCH/puppet' #{parent}..HEAD" + # Add info to the patches + additional_info = "Local-branch: #{branch}\n" + files = Dir.glob("00*.patch") + files.each do |file| + contents = File.read(file) + contents.sub!(/^---$/, "#{additional_info}---") + File.open(file, 'w') do |file_handle| + file_handle.print contents + end + end + # And then mail them out. # If we've got more than one patch, add --compose - if Dir.glob("00*.patch").length > 1 + if files.length > 1 compose = "--compose" else compose = "" -- cgit From 04515cfb1230d6bf0fd88910c91715d1d94faeae Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Tue, 9 Nov 2010 15:42:48 -0800 Subject: (#5198) Added a spec test for new TB unit --- spec/unit/type/tidy_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/unit/type/tidy_spec.rb b/spec/unit/type/tidy_spec.rb index 11edbfbf3..1573ead1b 100755 --- a/spec/unit/type/tidy_spec.rb +++ b/spec/unit/type/tidy_spec.rb @@ -110,7 +110,8 @@ describe tidy do :b => 0, :kb => 1, :mb => 2, - :gb => 3 + :gb => 3, + :tb => 4 } convertors.each do |unit, multiple| -- cgit