From ce1865f3d0bdcdf89d18e746806645b2b84b5252 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 17 Oct 2010 16:21:12 +0200 Subject: Fix #5023 - puppet-load multiple nodes support This patch allows puppet-load to compile multiple nodes catalog. This is done by using multiple --node. Puppet-load will use round-robin to chose which nodes catalog to ask for a given simulated client. It is also possible to pass a directory of facts yaml file in which puppet-load will load given --node facts file. This can work only if #5020 is applied to the puppetmaster first. Signed-off-by: Brice Figureau --- ext/puppet-load.rb | 84 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 25 deletions(-) (limited to 'ext') diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb index 110282d01..24437c10b 100644 --- a/ext/puppet-load.rb +++ b/ext/puppet-load.rb @@ -14,7 +14,7 @@ # # puppet-load [-d|--debug] [--concurrency ] [--repeat ] [-V|--version] [-v|--verbose] # [--node ] [--facts ] [--cert ] [--key ] -# [--server ] +# [--factsdir ] [--server ] # # = Description # @@ -35,8 +35,9 @@ # Set the puppet master hostname or IP address.. # # node:: -# Set the fully-qualified domain name of the client. This is only used for -# certificate purposes, but can be used to override the discovered hostname. +# Set the fully-qualified domain name of the client. This option can be given multiple +# times. In this case puppet-load will ask for catalog compilation of all the given nodes +# on a round robin way. # # help:: # Print this help message @@ -46,6 +47,11 @@ # file as found in the clientyaml directory. If none are provided, puppet-load # will look by itself using Puppet facts indirector. # +# factsdir:: +# Specify a directory where the yaml facts files can be found. If provided puppet-load +# will look up facts in this directory. If not found it will resort to using Puppet Facts +# indirector. +# # cert:: # This option is mandatory. It should be set to the cert PEM file that will be used # to quthenticate the client connections. @@ -70,8 +76,9 @@ # # = Example usage # +# SINGLE NODE: # 1) On the master host, generate a new certificate and private key for our test host: -# puppet ca --generate puppet-load.domain.com [*] +# puppet ca --generate puppet-load.domain.com # # 2) Copy the cert and key to the puppet-load host (which can be the same as the master one) # @@ -81,7 +88,7 @@ # allow $1 # allow puppet-load.domain.com # -# 4) launch the master +# 4) launch the master(s) # # 5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host # you want to simulate. @@ -89,12 +96,30 @@ # 5) launch puppet-load # puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 # -# [*]: unfortunately at this stage Puppet trusts the certname of the connecting node more than -# than the node name request paramater. It means that the master will compile -# the puppet-load node and not the --node given. +# MULTIPLE NODES: +# 1) On the master host, generate a new certificate and private key for our test host: +# puppet ca --generate puppet-load.domain.com +# +# 2) Copy the cert and key to the puppet-load host (which can be the same as the master one) +# +# 3) On the master host edit or create the auth.conf so that the catalog ACL match: +# path ~ ^/catalog/([^/]+)$ +# method find +# allow $1 +# allow puppet-load.domain.com +# +# 4) launch the master(s) +# +# 5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host +# you want to simulate. +# +# 5) launch puppet-load +# puppet-load -debug --node server1.domain.com --node server2.domain.com --node server3.domain.com \ +# --server master.domain.com --factsdir /var/lib/puppet/yaml/facts --concurrency 2 --repeat 20 +# +# puppet-load will load facts file in the --factsdir directory based on the node name. # # = TODO -# * Allow to simulate any different nodes # * More output stats for error connections (ie report errors, HTTP code...) # # @@ -115,6 +140,7 @@ $cmdargs = [ [ "--concurrency", "-c", GetoptLong::REQUIRED_ARGUMENT ], [ "--node", "-n", GetoptLong::REQUIRED_ARGUMENT ], [ "--facts", GetoptLong::REQUIRED_ARGUMENT ], + [ "--factsdir", GetoptLong::REQUIRED_ARGUMENT ], [ "--repeat", "-r", GetoptLong::REQUIRED_ARGUMENT ], [ "--cert", "-C", GetoptLong::REQUIRED_ARGUMENT ], [ "--key", "-k", GetoptLong::REQUIRED_ARGUMENT ], @@ -131,14 +157,15 @@ Puppet::Util::Log.newdestination(:console) times = {} def read_facts(file) - YAML.load(File.read(file)) + Puppet.debug("reading facts from: #{file}") + fact = YAML.load(File.read(file)) end result = GetoptLong.new(*$cmdargs) $args = {} -$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140} +$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140, :node => [], :factsdir => nil} begin result.each { |opt,arg| @@ -151,7 +178,9 @@ begin exit(14) end when "--node" - $options[:node] = arg + $options[:node] << arg + when "--factsdir" + $options[:factsdir] = arg when "--server" $options[:server] = arg when "--masterport" @@ -192,21 +221,24 @@ unless $options[:cert] and $options[:key] raise "--cert and --key are mandatory to authenticate the client" end -unless $options[:facts] and facts = read_facts($options[:facts]) - unless facts = Puppet::Node::Facts.find($options[:node]) - raise "Could not find facts for %s" % $options[:node] - end -end +parameters = [] -unless $options[:node] +unless $options[:node].size > 0 raise "--node is a mandatory argument. It tells to the master what node to compile" end -facts.values["fqdn"] = $options[:node] -facts.values["hostname"] = $options[:node].sub(/\..+/, '') -facts.values["domain"] = $options[:node].sub(/^[^.]+\./, '') +$options[:node].each do |node| + factfile = $options[:factsdir] ? File.join($options[:factsdir], node + ".yaml") : $options[:facts] + unless fact = read_facts(factfile) or fact = Puppet::Node::Facts.find(node) + raise "Could not find facts for %s" % node + end + fact.values["fqdn"] = node + fact.values["hostname"] = node.sub(/\..+/, '') + fact.values["domain"] = node.sub(/^[^.]+\./, '') + + parameters << {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(fact.render(:b64_zlib_yaml))} +end -parameters = {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(facts.render(:b64_zlib_yaml))} class RequestPool include EventMachine::Deferrable @@ -233,17 +265,19 @@ class RequestPool end def spawn_request(index) - EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{$options[:node]}").get( + nodeidx = index % $options[:node].size + node = $options[:node][nodeidx] + EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{node}").get( :port => $options[:masterport], - :query => @parameters, + :query => @parameters[nodeidx], :timeout => $options[:timeout], :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" }, :ssl => { :private_key_file => $options[:key], :cert_chain_file => $options[:cert], :verify_peer => false } ) do - Puppet.debug("starting client #{index}") @times[index] = Time.now @sizes[index] = 0 + Puppet.debug("starting client #{index} for #{node}") end end -- cgit From f2537d8f913ff78f5198dabd43727660010a4480 Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Sun, 17 Oct 2010 16:06:50 +0200 Subject: Puppet-load: better and safer error reporting In case of connection, dns or timeout error, puppet-load would not report the error correctly or could crash. Signed-off-by: Brice Figureau --- ext/puppet-load.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'ext') diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb index 24437c10b..35bee6ef8 100644 --- a/ext/puppet-load.rb +++ b/ext/puppet-load.rb @@ -265,6 +265,8 @@ class RequestPool end def spawn_request(index) + @times[index] = Time.now + @sizes[index] = 0 nodeidx = index % $options[:node].size node = $options[:node][nodeidx] EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{node}").get( @@ -302,7 +304,7 @@ class RequestPool } conn.errback { - Puppet.debug("Client #{index} finished with an error: #{conn.response.error}") + Puppet.debug("Client #{index} finished with an error: #{conn.error}") @times[index] = Time.now - @times[index] @responses[:failed].push(conn) check_progress -- cgit From 4d1681e05a187e55422bf00de5c932736548be15 Mon Sep 17 00:00:00 2001 From: Jeff McCune Date: Wed, 20 Oct 2010 15:22:06 -0700 Subject: (#5062) Add envpuppet helper script to ext/ This script helps people quickly test different branches of various puppet related projects like facter, puppet-scaffold, puppet-dashboard, etc... It allows the user to simply clone puppet and start running different puppet version without installing puppet or any other setup costs. --- ext/envpuppet | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 ext/envpuppet (limited to 'ext') diff --git a/ext/envpuppet b/ext/envpuppet new file mode 100755 index 000000000..d921a19b8 --- /dev/null +++ b/ext/envpuppet @@ -0,0 +1,80 @@ +#! /bin/bash +# +# Jeff McCune +# 2010-10-20 +# +# Copyright (c) 2010, Puppet Labs +# License: BSD 3-clause license +# +# This script provides a simple way to execute puppet and related tools +# directly from a git clone of the upstream repositories. This allows you to +# quickly switch branches and test different versions of code without much +# friction. +# +# NOTE: There may be issues if puppet, facter, etc... are already installed +# into RUBY's site_ruby directory. If you run into strange problems, make sure +# the correct ruby libraries are being loaded... +# +# Sample Usage: +# ============= +# cd ~/src +# git clone git://github.com/puppetlabs/puppet.git +# git clone git://github.com/puppetlabs/facter.git +# pushd puppet +# git checkout tags/2.6.1 +# popd +# pushd facter +# git checkout tags/1.5.8 +# export ENVPUPPET_BASEDIR=/home/jeff/src +# envpuppet puppet --version +# 2.6.1 +# envpuppet facter --version +# 1.5.8 + +set -e +set -u + +if test -d puppet -o -d facter; then + echo " WARNING!" + echo " Strange things happen if puppet or facter are in the" + echo " current working directory" + echo " (import errors from ruby are a prime example)" + echo " WARNING!" + echo "" + echo "I suggest changing to ~ or /tmp or something..." + echo "" + echo "Sleeping 2 seconds." + echo "" + sleep 2 +fi + +# Set this to where you check out puppet and facter +: ${ENVPUPPET_BASEDIR:="${HOME}/src"} + +# git://github.com/reductivelabs/puppet.git +mypath="${ENVPUPPET_BASEDIR}/puppet/sbin:${ENVPUPPET_BASEDIR}/puppet/bin" +myrubylib="${ENVPUPPET_BASEDIR}/puppet/lib" + +# git://github.com/reductivelabs/facter.git +mypath="${mypath}:${ENVPUPPET_BASEDIR}/facter/bin" +myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/facter/lib" + +# http://github.com/jamtur01/puppet-scaffold.git +mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-scaffold/bin" +myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-scaffold/lib" + +# http://github.com/puppetlabs/puppet-module-tool.git +# Also known as "pmt" Will become "puppet module" +mypath="${mypath}:${ENVPUPPET_BASEDIR}/puppet-module-tool/bin" +myrubylib="${myrubylib}:${ENVPUPPET_BASEDIR}/puppet-module-tool/lib" + +# Use the existing environment, if present. +# Default to no value to prevent unbound variable issues +mypath="${mypath}:${PATH:-}" +myrubylib="${myrubylib}:${RUBYLIB:-}" + +# Trim any trailing colons from the path list. +export PATH="${mypath%%:}" +export RUBYLIB="${myrubylib%%:}" + +exec "$@" -- cgit From f8d1427379f16d116c161e7231a26ba5451bc45f Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 5 Nov 2010 11:45:31 -0700 Subject: maint: First draft of cert inspector This script, ext/cert_inspector, takes directory names as command line arguments, searches those directories for SSL certificates, determines the type and contents of each cert, maps relationships between certificates, and prints a summary to standard out. --- ext/cert_inspector | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 ext/cert_inspector (limited to 'ext') diff --git a/ext/cert_inspector b/ext/cert_inspector new file mode 100755 index 000000000..1effcaa04 --- /dev/null +++ b/ext/cert_inspector @@ -0,0 +1,140 @@ +#!/usr/bin/env ruby +require 'openssl' + +class X509Collector + include Enumerable + + def initialize + @collected_data = {} + end + + def interpret_contents(contents) + cls = case contents.split("\n")[0] + when /BEGIN X509 CRL/ then OpenSSL::X509::CRL + when /BEGIN CERTIFICATE REQUEST/ then OpenSSL::X509::Request + when /BEGIN CERTIFICATE/ then OpenSSL::X509::Certificate + when /BEGIN RSA (PRIVATE|PUBLIC) KEY/ then OpenSSL::PKey::RSA + else return nil + end + cls.new(contents) + rescue + nil + end + + def expected_non_x509_files + ['inventory.txt', 'ca.pass', 'serial'] + end + + def investigate_path(path) + if File.directory?(path) + Dir.foreach(path) do |x| + next if ['.', '..'].include? x + investigate_path File.join(path, x) + end + else + contents = File.read path + meaning = interpret_contents contents + unless meaning || expected_non_x509_files.include?(File.basename(path)) + puts "WARNING: file #{path.inspect} could not be interpreted" + end + @collected_data[path] = meaning if meaning + end + end + + def each(&block) + @collected_data.each(&block) + end + + def extract_public_key_info(path, meaning) + case meaning + when OpenSSL::PKey::RSA + if meaning.private? + [meaning.public_key, 2, path] + else + [meaning, 3, path] + end + when OpenSSL::X509::Certificate + [meaning.public_key, 0, meaning.subject.to_s] + when OpenSSL::X509::Request + [meaning.public_key, 1, meaning.subject.to_s] + end + end + + def who_signed(meaning, key_names, keys) + signing_key = keys.find { |key| meaning.verify(key) } + if signing_key then "#{key_names[signing_key.to_s]}" else "???" end + end + + def explain(meaning, key_names, keys) + case meaning + when OpenSSL::PKey::RSA + if meaning.private? + "Private key for #{key_names[meaning.public_key.to_s]}" + else + "Public key for #{key_names[meaning.public_key.to_s]}" + end + when OpenSSL::X509::Certificate + signature_desc = who_signed(meaning, key_names, keys) + "Certificate assigning name #{meaning.subject.to_s} to #{key_names[meaning.public_key.to_s]}\n serial number #{meaning.serial}\n issued by #{meaning.issuer.to_s}\n signed by #{signature_desc}" + when OpenSSL::X509::Request + signature_desc = who_signed(meaning, key_names, keys) + "Certificate request for #{meaning.subject.to_s} having key #{key_names[meaning.public_key.to_s]}\n signed by #{signature_desc}" + when OpenSSL::X509::CRL + signature_desc = who_signed(meaning, key_names, keys) + revoked_serial_numbers = meaning.revoked.map { |r| r.serial } + revoked_desc = if revoked_serial_numbers.count > 0 then "serial numbers #{revoked_serial_numbers.inspect}" else "nothing" end + "Certificate revocation list revoking #{revoked_desc}\n issued by #{meaning.issuer.to_s}\n signed by #{signature_desc}" + else + "Unknown" + end + end + + # Yield unique public keys, with a canonical name for each. + def collect_public_keys + key_data = {} # pem => (priority, name, public_key) + @collected_data.collect do |path, meaning| + begin + next unless public_key_info = extract_public_key_info(path, meaning) + public_key, priority, name = public_key_info + pem = public_key.to_s + existing_priority, existing_name, existing_public_key = key_data[pem] + next if existing_priority and existing_priority < priority + key_data[pem] = priority, name, public_key + rescue + puts "exception!" + end + end + name_to_key_hash = {} + key_data.each do |pem, data| + priority, name, public_key = data + if name_to_key_hash[name] + suffix_num = 2 + while name_to_key_hash[name + " (#{suffix_num})"] + suffix_num += 1 + end + name = name + " (#{suffix_num})" + end + name_to_key_hash[name] = public_key + end + key_names = {} + keys = [] + name_to_key_hash.each do |name, public_key| + key_names[public_key.to_s] = "key<#{name}>" + keys << public_key + end + [key_names, keys] + end +end + +collector = X509Collector.new +ARGV.each do |path| + collector.investigate_path(path) +end +key_names, keys = collector.collect_public_keys +collector.map do |path, meaning| + [collector.explain(meaning, key_names, keys), path] +end.sort.each do |description, path| + puts "#{path}:" + puts " #{description}" + puts +end -- cgit