diff options
author | Jesse Wolfe <jes5199@gmail.com> | 2010-07-27 10:17:44 -0700 |
---|---|---|
committer | Jesse Wolfe <jes5199@gmail.com> | 2010-07-27 10:17:44 -0700 |
commit | b53e7d78e2e87571ae53170e9716b9ccd75da6e2 (patch) | |
tree | 58489a37bd0ee14b3d053a0e23f9638b4f6205e7 | |
parent | 94edd404130b4236f0c65a579857e3a25c5ee17f (diff) | |
parent | ecf44e4408c168893d74af58a4c7c8606634a844 (diff) | |
download | puppet-b53e7d78e2e87571ae53170e9716b9ccd75da6e2.tar.gz puppet-b53e7d78e2e87571ae53170e9716b9ccd75da6e2.tar.xz puppet-b53e7d78e2e87571ae53170e9716b9ccd75da6e2.zip |
Merge commit '2.6.1rc1' into next
46 files changed, 909 insertions, 161 deletions
@@ -1,5 +1,38 @@ +2.6.1rc1 +======== +bdfcac5 Update Red Hat spec file for 2.6.0 +9f08e7c Feature: puppet-load - a tool to stress-test master compilation +ef9a4a6 Fix #4245 - default insertion of ACL is not thread safe +4065e81 Fix race condition in rack autoloading of request/response +3163932 Fix #4244 - Cached Attributes is not thread safe +7d42f77 JRuby doesn't implement Process.maxgroups +760e418 Fix #4349 - Parsing with ignoreimport=true was always loading site.pp +67bdf89 Fix #4348 - Puppet doc single manifest broken +13c71b9 extlookup() is a builtin +d38e522 [#4333] old optparse doesn't support default_argv= +86b0882 Fixed #4326 - Updated SUSE packaging +03313b8 Fix #4319 - source file url sent to the master is invalid +ac3a0d2 vim: highlight default parameters in definition/classes +be2141a vim: match collected resources. +c047c8d vim: added elsif +9569136 Fix for 4314 -- Need to allow '-' in class name for refs +636079f Fixed #4304 - Changed logging level for auto import message +000fd1e Fix for #4303 -- reverting to old escaping in '-strings +1d494a3 Tweak to fix for #4302--dangling ref to known_resource_types +2383050 Fix #4302 - Compilation speed regression compared to 2.6 +63ec207 Minimal fix for #4297, with notes for follow-up +7ad7eb1 Fix #4286 - rename puppetdoc global module <site> to __site__ +28bb195 Fixed yumrepo type deprecation wanring ` +067a46d Temporary tweak to tests for #4242 +9778f2a [#4242] Fixed recursion due to parents including their children +59a23d6 Fix for #3382 -- Empty classes as graph placeholders +865282a Fixed example config.ru +a0a63c3 Fixed network and indirection reference +64386cf Fixed Indirection reference + 2.6.0 ===== +db0b30d Updated CHANGELOG for 2.6.0 42a475e Fixing #4268 - manifests always imported 06fc40c [#4269] Undef variables interpolate to empty string 1288f8c [#4270] Force inherited classes to load into the correct environment diff --git a/conf/redhat/puppet.spec b/conf/redhat/puppet.spec index 6791cea77..e51af0a6b 100644 --- a/conf/redhat/puppet.spec +++ b/conf/redhat/puppet.spec @@ -5,14 +5,14 @@ %global confdir conf/redhat Name: puppet -Version: 0.25.5 +Version: 2.6.0 Release: 1%{?dist} Summary: A network tool for managing many disparate systems License: GPLv2+ URL: http://puppetlabs.com Source0: http://puppetlabs.com/downloads/%{name}/%{name}-%{version}.tar.gz Source1: http://puppetlabs.com/downloads/%{name}/%{name}-%{version}.tar.gz.sign -Patch0: rundir-perms.patch + Group: System Environment/Base BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) @@ -68,6 +68,7 @@ The server can also function as a certificate authority and file server. %setup -q patch -p1 < conf/redhat/rundir-perms.patch + %build # Fix some rpmlint complaints for f in mac_dscl.pp mac_dscl_revert.pp \ @@ -91,6 +92,7 @@ rm -rf %{buildroot} ruby install.rb --destdir=%{buildroot} --quick --no-rdoc install -d -m0755 %{buildroot}%{_sysconfdir}/puppet/manifests +install -d -m0755 %{buildroot}%{_datadir}/%{name}/modules install -d -m0755 %{buildroot}%{_localstatedir}/lib/puppet install -d -m0755 %{buildroot}%{_localstatedir}/run/puppet install -d -m0750 %{buildroot}%{_localstatedir}/log/puppet @@ -221,6 +223,10 @@ fi rm -rf %{buildroot} %changelog +* Tue Jul 20 2010 Todd Zullinger <tmz@pobox.com> - 2.6.0-1 +- Update to 2.6.0 +- Create and own /usr/share/puppet/modules (#615432) + * Mon May 03 2010 Todd Zullinger <tmz@pobox.com> - 0.25.5-1 - Update to 0.25.5 - Adjust selinux conditional for EL-6 diff --git a/conf/suse/client.init b/conf/suse/client.init index d72b09b2c..71585cd7b 100644 --- a/conf/suse/client.init +++ b/conf/suse/client.init @@ -13,10 +13,10 @@ ### BEGIN INIT INFO # Provides: puppet -# Required-Start: $local_fs $network $syslog -# Should-Start: $null -# Required-Stop: $null -# Should-Stop: $null +# Required-Start: $local_fs $remote_fs $network $syslog +# Should-Start: puppet +# Required-Stop: $local_fs $remote_fs $network $syslog +# Should-Stop: puppet # Default-Start: 3 4 5 # Default-Stop: 0 1 2 6 # Short-Description: puppet diff --git a/conf/suse/fileserver.conf b/conf/suse/fileserver.conf new file mode 100644 index 000000000..67e387ca0 --- /dev/null +++ b/conf/suse/fileserver.conf @@ -0,0 +1,12 @@ +# This file consists of arbitrarily named sections/modules +# defining where files are served from and to whom + +# Define a section 'files' +# Adapt the allow/deny settings to your needs. Order +# for allow/deny does not matter, allow always takes precedence +# over deny +# [files] +# path /var/lib/puppet/files +# allow *.example.com +# deny *.evil.example.com +# allow 192.168.0.0/24 diff --git a/conf/suse/logrotate b/conf/suse/logrotate new file mode 100644 index 000000000..c3a4d437a --- /dev/null +++ b/conf/suse/logrotate @@ -0,0 +1,10 @@ +/var/log/puppet/*log { + missingok + notifempty + create 0644 puppet puppet + sharedscripts + postrotate + [ -e /etc/init.d/puppetmaster ] && /etc/init.d/puppetmaster condrestart >/dev/null 2>&1 || true + [ -e /etc/init.d/puppet ] && /etc/init.d/puppet reload > /dev/null 2>&1 || true + endscript +} diff --git a/conf/suse/puppet.conf b/conf/suse/puppet.conf new file mode 100644 index 000000000..47501a388 --- /dev/null +++ b/conf/suse/puppet.conf @@ -0,0 +1,25 @@ +[main] + # The Puppet log directory. + # The default value is '$vardir/log'. + logdir = /var/log/puppet + + # Where Puppet PID files are kept. + # The default value is '$vardir/run'. + rundir = /var/run/puppet + + # Where SSL certificates are kept. + # The default value is '$confdir/ssl'. + ssldir = $vardir/ssl + +[agent] + # The file in which puppetd stores a list of the classes + # associated with the retrieved configuratiion. Can be loaded in + # the separate ``puppet`` executable using the ``--loadclasses`` + # option. + # The default value is '$confdir/classes.txt'. + classfile = $vardir/classes.txt + + # Where puppetd caches the local configuration. An + # extension indicating the cache format is added automatically. + # The default value is '$confdir/localconfig'. + localconfig = $vardir/localconfig diff --git a/conf/suse/puppet.spec b/conf/suse/puppet.spec index 2a0e6e359..bd4b35885 100644 --- a/conf/suse/puppet.spec +++ b/conf/suse/puppet.spec @@ -1,31 +1,35 @@ %{!?ruby_sitelibdir: %define ruby_sitelibdir %(ruby -rrbconfig -e 'puts Config::CONFIG["sitelibdir"]')} %define pbuild %{_builddir}/%{name}-%{version} -%define suseconfdir conf/suse -%define confdir conf/redhat +%define confdir conf/suse Summary: A network tool for managing many disparate systems Name: puppet -Version: 0.25.4 +Version: 2.6.0 Release: 1%{?dist} License: GPL -Group: System Environment/Base +Group: Productivity/Networking/System -URL: http://reductivelabs.com/projects/puppet/ -Source: http://reductivelabs.com/downloads/puppet/%{name}-%{version}.tar.gz -Patch0: puppet.suse.patch +URL: http://puppetlabs.com/projects/puppet/ +Source0: http://puppetlabs.com/downloads/puppet/%{name}-%{version}.tar.gz +Source1: client.init +Source2: server.init +Patch0: ruby-env.patch + +PreReq: %{insserv_prereq} %{fillup_prereq} Requires: ruby >= 1.8.2 -Requires: facter >= 1.3.7 +Requires: facter >= 1.5 +Requires: cron BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildRequires: ruby >= 1.8.2 +BuildRequires: ruby >= 1.8.7 %description -Puppet lets you centrally manage every important aspect of your system using a -cross-platform specification language that manages all the separate elements -normally aggregated in different files, like users, cron jobs, and hosts, +Puppet lets you centrally manage every important aspect of your system using a +cross-platform specification language that manages all the separate elements +normally aggregated in different files, like users, cron jobs, and hosts, along with obviously discrete elements like packages, services, and files. %package server -Group: System Environment/Base +Group: Productivity/Networking/System Summary: Server for the puppet system management tool Requires: puppet = %{version}-%{release} @@ -45,45 +49,47 @@ done %install %{__install} -d -m0755 %{buildroot}%{_sbindir} %{__install} -d -m0755 %{buildroot}%{_bindir} +%{__install} -d -m0755 %{buildroot}%{_confdir} %{__install} -d -m0755 %{buildroot}%{ruby_sitelibdir} %{__install} -d -m0755 %{buildroot}%{_sysconfdir}/puppet/manifests %{__install} -d -m0755 %{buildroot}%{_docdir}/%{name}-%{version} %{__install} -d -m0755 %{buildroot}%{_localstatedir}/lib/puppet %{__install} -d -m0755 %{buildroot}%{_localstatedir}/run/puppet %{__install} -d -m0755 %{buildroot}%{_localstatedir}/log/puppet -%{__install} -Dp -m0755 %{pbuild}/bin/* %{buildroot}%{_sbindir} +%{__install} -Dp -m0755 %{pbuild}/bin/* %{buildroot}%{_bindir} %{__install} -Dp -m0755 %{pbuild}/sbin/* %{buildroot}%{_sbindir} -%{__mv} %{buildroot}%{_sbindir}/puppet %{buildroot}%{_bindir}/puppet -%{__mv} %{buildroot}%{_sbindir}/puppetrun %{buildroot}%{_bindir}/puppetrun %{__install} -Dp -m0644 %{pbuild}/lib/puppet.rb %{buildroot}%{ruby_sitelibdir}/puppet.rb %{__cp} -a %{pbuild}/lib/puppet %{buildroot}%{ruby_sitelibdir} find %{buildroot}%{ruby_sitelibdir} -type f -perm +ugo+x -exec chmod a-x '{}' \; -%{__install} -Dp -m0644 %{confdir}/client.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppet +%{__cp} -a %{pbuild}/conf/redhat/client.sysconfig %{buildroot}%{_confdir}/client.sysconfig +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/client.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppet %{__install} -Dp -m0755 %SOURCE1 %{buildroot}%{_initrddir}/puppet -%{__install} -Dp -m0644 %{confdir}/server.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppetmaster +%{__cp} -a %{pbuild}/conf/redhat/server.sysconfig %{buildroot}%{_confdir}/server.sysconfig +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/server.sysconfig %{buildroot}/var/adm/fillup-templates/sysconfig.puppetmaster %{__install} -Dp -m0755 %SOURCE2 %{buildroot}%{_initrddir}/puppetmaster -%{__install} -Dp -m0644 %{confdir}/fileserver.conf %{buildroot}%{_sysconfdir}/puppet/fileserver.conf -%{__install} -Dp -m0644 %{confdir}/puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppet.conf -#%{__ln_s} puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppetmasterd.conf -#%{__ln_s} puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppetca.conf -%{__install} -Dp -m0644 %{confdir}/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/puppet +%{__cp} -a %{pbuild}/conf/redhat/fileserver.conf %{buildroot}%{_confdir}/fileserver.conf +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/fileserver.conf %{buildroot}%{_sysconfdir}/puppet/fileserver.conf +%{__cp} -a %{pbuild}/conf/redhat/puppet.conf %{buildroot}%{_confdir}/puppet.conf +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/puppet.conf %{buildroot}%{_sysconfdir}/puppet/puppet.conf +%{__cp} -a %{pbuild}/conf/redhat/logrotate %{buildroot}%{_confdir}/logrotate +%{__install} -Dp -m0644 %{buildroot}%{_confdir}/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/puppet %{__ln_s} %{_initrddir}/puppet %{buildroot}%{_sbindir}/rcpuppet %{__ln_s} %{_initrddir}/puppetmaster %{buildroot}%{_sbindir}/rcpuppetmaster %files %defattr(-, root, root, 0755) %{_bindir}/puppet -%{_sbindir}/filebucket -%{_sbindir}/ralsh +%{_bindir}/puppetdoc +%{_bindir}/filebucket +%{_bindir}/ralsh +%{_bindir}/pi %{_sbindir}/puppetd %{_sbindir}/rcpuppet -%{_sbindir}/pi %{ruby_sitelibdir}/* %{_initrddir}/puppet /var/adm/fillup-templates/sysconfig.puppet %config(noreplace) %{_sysconfdir}/puppet/puppet.conf %doc CHANGELOG COPYING LICENSE README examples -%exclude %{_sbindir}/puppetdoc %config(noreplace) %{_sysconfdir}/logrotate.d/puppet %dir %{_sysconfdir}/puppet # These need to be owned by puppet so the server can @@ -95,56 +101,115 @@ find %{buildroot}%{ruby_sitelibdir} -type f -perm +ugo+x -exec chmod a-x '{}' \; %files server %defattr(-, root, root, 0755) %{_sbindir}/puppetmasterd +%{_sbindir}/rcpuppetmaster %{_sbindir}/puppetqd -%{_bindir}/puppetrun +%{_sbindir}/puppetrun +%{_sbindir}/puppetca %{_initrddir}/puppetmaster %config(noreplace) %{_sysconfdir}/puppet/* -%config(noreplace) %{_sysconfdir}/sysconfig/puppetmaster -%{_sbindir}/puppetca +%exclude %{_sysconfdir}/puppet/puppet.conf +/var/adm/fillup-templates/sysconfig.puppetmaster +%dir %{_sysconfdir}/puppet %pre /usr/sbin/groupadd -r puppet 2>/dev/null || : /usr/sbin/useradd -g puppet -c "Puppet" \ - -s /sbin/nologin -r -d /var/lib/puppet puppet 2> /dev/null || : + -s /sbin/nologin -r -d /var/puppet puppet 2> /dev/null || : %post -/sbin/chkconfig --add puppet -exit 0 +%{fillup_and_insserv -y puppet} %post server -/sbin/chkconfig --add puppetmaster +%{fillup_and_insserv -n -y puppetmaster} %preun -if [ "$1" = 0 ] ; then - /sbin/service puppet stop > /dev/null 2>&1 - /sbin/chkconfig --del puppet -fi +%stop_on_removal puppet %preun server -if [ "$1" = 0 ] ; then - /sbin/service puppetmaster stop > /dev/null 2>&1 - /sbin/chkconfig --del puppetmaster -fi +%stop_on_removal puppetmaster + +%postun +%restart_on_update puppet +%{insserv_cleanup} %postun server -if [ "$1" -ge 1 ]; then - /sbin/service puppetmaster try-restart > /dev/null 2>&1 -fi +%restart_on_update puppetmaster +%{insserv_cleanup} %clean %{__rm} -rf %{buildroot} %changelog -* Sat Feb 16 2008 James Turnbull <james@lovedthanlost.net> - 0.24.1-1 -- Fixed puppet configuation file references to match single puppet.conf file -- Update versions for 0.24.1 release +* Wed Jul 21 2010 Ben Kevan <ben.kevan@gmail.com> - 2.6.0 +- New version and ruby version bump +- Add puppetdoc to %_bindir (unknown why original suse package, excluded or forgot to add) +- Corrected patch for ruby environment +- Move binaries back to the correct directories + +* Wed Jul 14 2010 Ben Kevan <ben.kevan@gmail.com> - 0.25.5 +- New version. +- Use original client, server.init names +- Revert to puppetmaster +- Fixed client.init and server.init and included $null and Should-Stop for both + +* Tue Mar 2 2010 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.25.4 +- New version. + +* Sun Aug 9 2009 Noah Fontes <nfontes@transtruct.org> +- Fix build on SLES 9. +- Enable puppet and puppet-server services by default. + +* Sat Aug 8 2009 Noah Fontes <nfontes@transtruct.org> +- Fix a lot of relevant warnings from rpmlint. +- Build on OpenSUSE 11.1 correctly. +- Rename puppetmaster init scripts to puppet-server to correspond to the package name. + +* Wed Apr 22 2009 Leo Eraly <leo@unstable.be> - 0.24.8 +- New version. + +* Tue Dec 9 2008 Leo Eraly <leo@unstable.be> - 0.24.6 +- New version. + +* Fri Sep 5 2008 Leo Eraly <leo@unstable.be> - 0.24.5 +- New version. + +* Fri Jun 20 2008 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.24.4 +- Removed symlinks to old configuration files + +* Fri Dec 14 2007 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.24.0 +- New version. + +* Fri Jun 29 2007 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.23.0 +- New version. + +* Wed May 2 2007 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.22.4 +- New version. Includes provider for rug package manager. + +* Wed Apr 25 2007 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.22.3 +- New version. Added links /sbin/rcpuppet and /sbin/rcpuppetmaster + +* Sun Jan 7 2007 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.22.0 +- version bump + +* Tue Oct 3 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.19.3-3 +- Made package arch dependant. + +* Sat Sep 23 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.19.3-1 +- New version + +* Sun Sep 17 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.19.1-1 +- New version + +* Tue Aug 30 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.19.0-1 +- New version +- No need to patch anymore :-), since my changes went into official release. * Tue Aug 3 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.18.4-3 - Replaced puppet-bin.patch with %build section from David's spec * Tue Aug 1 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.18.4-2 - Added supprot for enabling services in SuSE - + * Tue Aug 1 2006 Martin Vuk <martin.vuk@fri.uni-lj.si> - 0.18.4-1 - New version and support for SuSE diff --git a/conf/suse/puppet.suse.patch b/conf/suse/puppet.suse.patch deleted file mode 100644 index 41974679f..000000000 --- a/conf/suse/puppet.suse.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- sbin/puppetmasterd.orig 2009-11-23 10:38:16.632264990 -0500 -+++ sbin/puppetmasterd 2009-11-23 10:38:28.884265155 -0500 -@@ -1,4 +1,4 @@ --#!/usr/bin/env ruby -+#!/usr/bin/ruby - - # - # = Synopsis ---- sbin/puppetd.orig 2009-11-23 10:38:23.240265893 -0500 -+++ sbin/puppetd 2009-11-23 10:38:35.044265271 -0500 -@@ -1,4 +1,4 @@ --#!/usr/bin/env ruby -+#!/usr/bin/ruby - # == Synopsis - # - # Retrieve the client configuration from the central puppet server and apply diff --git a/conf/suse/ruby-env.patch b/conf/suse/ruby-env.patch new file mode 100644 index 000000000..0a7a8a748 --- /dev/null +++ b/conf/suse/ruby-env.patch @@ -0,0 +1,17 @@ +--- sbin/puppetd 2010-07-18 14:17:38.000000000 -0700 ++++ sbin/puppetd 2010-07-21 17:31:57.057138615 -0700 +@@ -1,4 +1,4 @@ +-#!/usr/bin/env ruby ++#!/usr/bin/ruby + + require 'puppet/application/agent' + Puppet::Application[:agent].run + +--- sbin/puppetmasterd 2010-07-18 14:17:38.000000000 -0700 ++++ sbin/puppetmasterd 2010-07-21 17:32:10.178513969 -0700 +@@ -1,4 +1,4 @@ +-#!/usr/bin/env ruby ++#!/usr/bin/ruby + + require 'puppet/application/master' + Puppet::Application[:master].run diff --git a/conf/suse/server.init b/conf/suse/server.init index e3a1d902a..50b9fef81 100644 --- a/conf/suse/server.init +++ b/conf/suse/server.init @@ -11,10 +11,10 @@ ### BEGIN INIT INFO # Provides: puppetmaster -# Required-Start: $local_fs $network $syslog -# Should-Start: $null -# Required-Stop: $null -# Should-Stop: $null +# Required-Start: $local_fs $remote_fs $network $syslog +# Should-Start: puppetmaster +# Required-Stop: $local_fs $remote_fs $network $syslog +# Should-Stop: puppetmaster # Default-Start: 3 4 5 # Default-Stop: 0 1 2 6 # Short-Description: puppetmaster diff --git a/ext/puppet-load.rb b/ext/puppet-load.rb new file mode 100644 index 000000000..110282d01 --- /dev/null +++ b/ext/puppet-load.rb @@ -0,0 +1,357 @@ +#!/usr/bin/env ruby +# == Synopsis +# +# This tool can exercize a puppetmaster by simulating an arbitraty number of concurrent clients +# in a lightweight way. +# +# = Prerequisites +# +# This tool requires Event Machine and em-http-request, and an installation of Puppet. +# Event Machine can be installed from gem. +# em-http-request can be installed from gem. +# +# = Usage +# +# puppet-load [-d|--debug] [--concurrency <num>] [--repeat <num>] [-V|--version] [-v|--verbose] +# [--node <host.domain.com>] [--facts <factfile>] [--cert <certfile>] [--key <keyfile>] +# [--server <server.domain.com>] +# +# = Description +# +# This is a simple script meant for doing performance tests of puppet masters. It does this +# by simulating concurrent connections to a puppet master and asking for catalog compilation. +# +# = Options +# +# Unlike other puppet executables, puppet-load doesn't parse puppet.conf nor use puppet options +# +# debug:: +# Enable full debugging. +# +# concurreny:: +# Number of simulated concurrent clients. +# +# server:: +# Set the puppet master hostname or IP address.. +# +# node:: +# Set the fully-qualified domain name of the client. This is only used for +# certificate purposes, but can be used to override the discovered hostname. +# +# help:: +# Print this help message +# +# facts:: +# This can be used to provide facts for the compilation, directly from a YAML +# file as found in the clientyaml directory. If none are provided, puppet-load +# will look by itself using Puppet facts indirector. +# +# cert:: +# This option is mandatory. It should be set to the cert PEM file that will be used +# to quthenticate the client connections. +# +# key:: +# This option is mandatory. It should be set to the private key PEM file that will be used +# to quthenticate the client connections. +# +# timeout:: +# The number of seconds after which a simulated client is declared in error if it didn't get +# a catalog. The default is 180s. +# +# repeat:: +# How many times to perform the test. This means puppet-load will ask for +# concurrency * repeat catalogs. +# +# verbose:: +# Turn on verbose reporting. +# +# version:: +# Print the puppet version number and exit. +# +# = Example usage +# +# 1) On the master host, generate a new certificate and private key for our test host: +# puppet ca --generate puppet-load.domain.com [*] +# +# 2) Copy the cert and key to the puppet-load host (which can be the same as the master one) +# +# 3) On the master host edit or create the auth.conf so that the catalog ACL match: +# path ~ ^/catalog/([^/]+)$ +# method find +# allow $1 +# allow puppet-load.domain.com +# +# 4) launch the master +# +# 5) Prepare or get a fact file. One way to get one is to look on the master in $vardir/yaml/ for the host +# you want to simulate. +# +# 5) launch puppet-load +# puppet-load -debug --node server.domain.com --server master.domain.com --facts server.domain.com.yaml --concurrency 2 --repeat 20 +# +# [*]: unfortunately at this stage Puppet trusts the certname of the connecting node more than +# than the node name request paramater. It means that the master will compile +# the puppet-load node and not the --node given. +# +# = TODO +# * Allow to simulate any different nodes +# * More output stats for error connections (ie report errors, HTTP code...) +# +# + +# Do an initial trap, so that cancels don't get a stack trace. +trap(:INT) do + $stderr.puts "Cancelling startup" + exit(1) +end + +require 'rubygems' +require 'eventmachine' +require 'em-http' +require 'getoptlong' +require 'puppet' + +$cmdargs = [ + [ "--concurrency", "-c", GetoptLong::REQUIRED_ARGUMENT ], + [ "--node", "-n", GetoptLong::REQUIRED_ARGUMENT ], + [ "--facts", GetoptLong::REQUIRED_ARGUMENT ], + [ "--repeat", "-r", GetoptLong::REQUIRED_ARGUMENT ], + [ "--cert", "-C", GetoptLong::REQUIRED_ARGUMENT ], + [ "--key", "-k", GetoptLong::REQUIRED_ARGUMENT ], + [ "--timeout", "-t", GetoptLong::REQUIRED_ARGUMENT ], + [ "--server", "-s", GetoptLong::REQUIRED_ARGUMENT ], + [ "--debug", "-d", GetoptLong::NO_ARGUMENT ], + [ "--help", "-h", GetoptLong::NO_ARGUMENT ], + [ "--verbose", "-v", GetoptLong::NO_ARGUMENT ], + [ "--version", "-V", GetoptLong::NO_ARGUMENT ], +] + +Puppet::Util::Log.newdestination(:console) + +times = {} + +def read_facts(file) + YAML.load(File.read(file)) +end + + +result = GetoptLong.new(*$cmdargs) + +$args = {} +$options = {:repeat => 1, :concurrency => 1, :pause => false, :cert => nil, :key => nil, :timeout => 180, :masterport => 8140} + +begin + result.each { |opt,arg| + case opt + when "--concurrency" + begin + $options[:concurrency] = Integer(arg) + rescue => detail + $stderr.puts "The argument to 'fork' must be an integer" + exit(14) + end + when "--node" + $options[:node] = arg + when "--server" + $options[:server] = arg + when "--masterport" + $options[:masterport] = arg + when "--facts" + $options[:facts] = arg + when "--repeat" + $options[:repeat] = Integer(arg) + when "--help" + if Puppet.features.usage? + RDoc::usage && exit + else + puts "No help available unless you have RDoc::usage installed" + exit + end + when "--version" + puts "%s" % Puppet.version + exit + when "--verbose" + Puppet::Util::Log.level = :info + Puppet::Util::Log.newdestination(:console) + when "--debug" + Puppet::Util::Log.level = :debug + Puppet::Util::Log.newdestination(:console) + when "--cert" + $options[:cert] = arg + when "--key" + $options[:key] = arg + end + } +rescue GetoptLong::InvalidOption => detail + $stderr.puts detail + $stderr.puts "Try '#{$0} --help'" + exit(1) +end + +unless $options[:cert] and $options[:key] + raise "--cert and --key are mandatory to authenticate the client" +end + +unless $options[:facts] and facts = read_facts($options[:facts]) + unless facts = Puppet::Node::Facts.find($options[:node]) + raise "Could not find facts for %s" % $options[:node] + end +end + +unless $options[:node] + raise "--node is a mandatory argument. It tells to the master what node to compile" +end + +facts.values["fqdn"] = $options[:node] +facts.values["hostname"] = $options[:node].sub(/\..+/, '') +facts.values["domain"] = $options[:node].sub(/^[^.]+\./, '') + +parameters = {:facts_format => "b64_zlib_yaml", :facts => CGI.escape(facts.render(:b64_zlib_yaml))} + +class RequestPool + include EventMachine::Deferrable + + attr_reader :requests, :responses, :times, :sizes + attr_reader :repeat, :concurrency, :max_request + + def initialize(concurrency, repeat, parameters) + @parameters = parameters + @current_request = 0 + @max_request = repeat * concurrency + @repeat = repeat + @concurrency = concurrency + @requests = [] + @responses = {:succeeded => [], :failed => []} + @times = {} + @sizes = {} + + # initial spawn + (1..concurrency).each do |i| + spawn + end + + end + + def spawn_request(index) + EventMachine::HttpRequest.new("https://#{$options[:server]}:#{$options[:masterport]}/production/catalog/#{$options[:node]}").get( + :port => $options[:masterport], + :query => @parameters, + :timeout => $options[:timeout], + :head => { "Accept" => "pson, yaml, b64_zlib_yaml, marshal, dot, raw", "Accept-Encoding" => "gzip, deflate" }, + :ssl => { :private_key_file => $options[:key], + :cert_chain_file => $options[:cert], + :verify_peer => false } ) do + Puppet.debug("starting client #{index}") + @times[index] = Time.now + @sizes[index] = 0 + end + end + + def add(index, conn) + @requests.push(conn) + + conn.stream { |data| + @sizes[index] += data.length + } + + conn.callback { + @times[index] = Time.now - @times[index] + code = conn.response_header.status + if code >= 200 && code < 300 + Puppet.debug("Client #{index} finished successfully") + @responses[:succeeded].push(conn) + else + Puppet.debug("Client #{index} finished with HTTP code #{code}") + @responses[:failed].push(conn) + end + check_progress + } + + conn.errback { + Puppet.debug("Client #{index} finished with an error: #{conn.response.error}") + @times[index] = Time.now - @times[index] + @responses[:failed].push(conn) + check_progress + } + end + + def all_responses + @responses[:succeeded] + @responses[:failed] + end + + protected + + def check_progress + spawn unless all_spawned? + succeed if all_finished? + end + + def all_spawned? + @requests.size >= max_request + end + + def all_finished? + @responses[:failed].size + @responses[:succeeded].size >= max_request + end + + def spawn + add(@current_request, spawn_request(@current_request)) + @current_request += 1 + end +end + + +def mean(array) + array.inject(0) { |sum, x| sum += x } / array.size.to_f +end + +def median(array) + array = array.sort + m_pos = array.size / 2 + return array.size % 2 == 1 ? array[m_pos] : mean(array[m_pos-1..m_pos]) +end + +def format_bytes(bytes) + if bytes < 1024 + "%.2f B" % bytes + elsif bytes < 1024 * 1024 + "%.2f KiB" % (bytes/1024.0) + else + "%.2f MiB" % (bytes/(1024.0*1024.0)) + end +end + +EM::run { + + start = Time.now + multi = RequestPool.new($options[:concurrency], $options[:repeat], parameters) + + multi.callback do + duration = Time.now - start + puts "#{multi.max_request} requests finished in #{duration} s" + puts "#{multi.responses[:failed].size} requests failed" + puts "Availability: %3.2f %%" % (100.0*multi.responses[:succeeded].size/(multi.responses[:succeeded].size+multi.responses[:failed].size)) + + minmax = multi.times.values.minmax + all_time = multi.times.values.reduce(:+) + + puts "\nTime (s):" + puts "\tmin: #{minmax[0]} s" + puts "\tmax: #{minmax[1]} s" + puts "\taverage: #{mean(multi.times.values)} s" + puts "\tmedian: #{median(multi.times.values)} s" + + puts "\nConcurrency: %.2f" % (all_time/duration) + puts "Transaction Rate (tps): %.2f t/s" % (multi.max_request / duration) + + transferred = multi.sizes.values.reduce(:+) + + puts "\nReceived bytes: #{format_bytes(transferred)}" + puts "Throughput: %.5f MiB/s" % (transferred/duration/(1024.0*1024.0)) + + # this is the end + EventMachine.stop + end +} + + diff --git a/ext/rack/files/config.ru b/ext/rack/files/config.ru index 5f0834a7d..995a9f8c5 100644 --- a/ext/rack/files/config.ru +++ b/ext/rack/files/config.ru @@ -11,8 +11,7 @@ require 'puppet' # ARGV << "--debug" ARGV << "--rack" -require 'puppet/application/puppetmasterd' +require 'puppet/application/master' # we're usually running inside a Rack::Builder.new {} block, # therefore we need to call run *here*. -run Puppet::Application[:puppetmasterd].run - +run Puppet::Application[:master].run diff --git a/ext/vim/syntax/puppet.vim b/ext/vim/syntax/puppet.vim index 80cd91c6c..0025e2d1c 100644 --- a/ext/vim/syntax/puppet.vim +++ b/ext/vim/syntax/puppet.vim @@ -19,7 +19,7 @@ endif " match class/definition/node declarations syn region puppetDefine start="^\s*\(class\|define\|node\)\s" end="{" contains=puppetDefType,puppetDefName,puppetDefArguments,puppetNodeRe syn keyword puppetDefType class define node inherits contained -syn region puppetDefArguments start="(" end=")" contained contains=puppetArgument +syn region puppetDefArguments start="(" end=")" contained contains=puppetArgument,puppetString syn match puppetArgument "\w\+" contained syn match puppetArgument "\$\w\+" contained syn match puppetArgument "'[^']+'" contained @@ -33,6 +33,7 @@ syn match puppetNodeRe "/.*/" contained "FIXME: "Foo-bar" doesn't get highlighted as expected, although "foo-bar" does. syn match puppetInstance "[A-Za-z0-9_-]\+\(::[A-Za-z0-9_-]\+\)*\s*{" contains=puppetTypeName,puppetTypeDefault syn match puppetInstance "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*[[{]" contains=puppetTypeName,puppetTypeDefault +syn match puppetInstance "[A-Z][a-z_-]\+\(::[A-Z][a-z_-]\+\)*\s*<\?<|" contains=puppetTypeName,puppetTypeDefault syn match puppetTypeName "[a-z]\w*" contained syn match puppetTypeDefault "[A-Z]\w*" contained @@ -68,7 +69,7 @@ syn match puppetNotVariable "\\$\w\+" contained syn match puppetNotVariable "\\${\w\+}" contained syn keyword puppetKeyword import inherits include -syn keyword puppetControl case default if else +syn keyword puppetControl case default if else elsif syn keyword puppetSpecial true false undef " comments last overriding everything else diff --git a/lib/puppet/application.rb b/lib/puppet/application.rb index 05b7d466f..0a8fbc155 100644 --- a/lib/puppet/application.rb +++ b/lib/puppet/application.rb @@ -264,7 +264,6 @@ class Application self.send(fname, value) end end - @option_parser.default_argv = self.command_line.args @option_parser end @@ -337,7 +336,7 @@ class Application # scan command line argument begin - self.option_parser.parse! + self.option_parser.parse!(self.command_line.args) rescue OptionParser::ParseError => detail $stderr.puts detail $stderr.puts "Try 'puppet #{command_line.subcommand_name} --help'" diff --git a/lib/puppet/network/http/rack.rb b/lib/puppet/network/http/rack.rb index c18a07559..5b4ef7e1c 100644 --- a/lib/puppet/network/http/rack.rb +++ b/lib/puppet/network/http/rack.rb @@ -1,3 +1,6 @@ +require 'rack' +require 'rack/request' +require 'rack/response' require 'puppet/network/http' require 'puppet/network/http/rack/rest' diff --git a/lib/puppet/network/rest_authconfig.rb b/lib/puppet/network/rest_authconfig.rb index 38a8f9a09..82d5a9de2 100644 --- a/lib/puppet/network/rest_authconfig.rb +++ b/lib/puppet/network/rest_authconfig.rb @@ -3,6 +3,7 @@ require 'puppet/network/authconfig' module Puppet class Network::RestAuthConfig < Network::AuthConfig + extend MonitorMixin attr_accessor :rights DEFAULT_ACL = [ @@ -20,9 +21,11 @@ module Puppet ] def self.main - add_acl = @main.nil? - super - @main.insert_default_acl if add_acl and !@main.exists? + synchronize do + add_acl = @main.nil? + super + @main.insert_default_acl if add_acl and !@main.exists? + end @main end diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 762599cff..3f67474f9 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -1,4 +1,5 @@ require 'puppet/util/cacher' +require 'monitor' # Just define it, so this class has fewer load dependencies. class Puppet::Node @@ -67,14 +68,23 @@ class Puppet::Node::Environment def initialize(name) @name = name + extend MonitorMixin end def known_resource_types - if @known_resource_types.nil? or @known_resource_types.stale? - @known_resource_types = Puppet::Resource::TypeCollection.new(self) - @known_resource_types.perform_initial_import - end - @known_resource_types + # This makes use of short circuit evaluation to get the right thread-safe + # per environment semantics with an efficient most common cases; we almost + # always just return our thread's known-resource types. Only at the start + # of a compilation (after our thread var has been set to nil) or when the + # environment has changed do we delve deeper. + Thread.current[:known_resource_types] = nil if (krt = Thread.current[:known_resource_types]) && krt.environment != self + Thread.current[:known_resource_types] ||= synchronize { + if @known_resource_types.nil? or @known_resource_types.stale? + @known_resource_types = Puppet::Resource::TypeCollection.new(self) + @known_resource_types.perform_initial_import + end + @known_resource_types + } end def module(name) diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index a901c0dd6..61bb13cb6 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -19,7 +19,11 @@ class Puppet::Parser::Compiler rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "#{detail} on node #{node.name}" - end + ensure + # We get these from the environment and only cache them in a thread + # variable for the duration of the compilation. + Thread.current[:known_resource_types] = nil + end attr_reader :node, :facts, :collections, :catalog, :node_scope, :resources, :relationships diff --git a/ext/extlookup.rb b/lib/puppet/parser/functions/extlookup.rb index d87583ba7..ee230e7ce 100644 --- a/ext/extlookup.rb +++ b/lib/puppet/parser/functions/extlookup.rb @@ -82,11 +82,11 @@ require 'csv' module Puppet::Parser::Functions newfunction(:extlookup, :type => :rvalue) do |args| key = args[0] - default = "_ExtUNSET_" - datafile = "_ExtUNSET_" - default = args[1] if args[1] - datafile = args[2] if args[2] + default = args[1] + datafile = args[2] + + raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3 extlookup_datadir = lookupvar('extlookup_datadir') extlookup_precedence = Array.new @@ -123,12 +123,13 @@ module Puppet::Parser::Functions datafiles << extlookup_datadir + "/#{d}.csv" end - desired = "_ExtUNSET_" + desired = nil datafiles.each do |file| + parser = Puppet::Parser::Parser.new(environment) parser.watch_file(file) if File.exists?(file) - if desired == "_ExtUNSET_" + if desired.nil? if File.exists?(file) result = CSV.read(file).find_all do |r| r[0] == key @@ -167,15 +168,6 @@ module Puppet::Parser::Functions end end - # don't accidently return nil's and such rather throw a parse error - if desired == "_ExtUNSET_" && default == "_ExtUNSET_" - raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" - else - desired = default if desired == "_ExtUNSET_" - end - - desired + desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" end end - -# vi:tabstop=4:expandtab:ai diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb index 1e10ff96c..aa04f17a0 100644 --- a/lib/puppet/parser/lexer.rb +++ b/lib/puppet/parser/lexer.rb @@ -221,7 +221,7 @@ class Puppet::Parser::Lexer TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true TOKENS.add_token :SQUOTE, "'" do |lexer, value| - [TOKENS[:STRING], lexer.slurpstring(value).first ] + [TOKENS[:STRING], lexer.slurpstring(value,["'"],:ignore_invalid_esapes).first ] end DQ_initial_token_types = {'$' => :DQPRE,'"' => :STRING} @@ -517,8 +517,7 @@ class Puppet::Parser::Lexer # we've encountered the start of a string... # slurp in the rest of the string and return it - Valid_escapes_in_strings = %w{ \\ $ ' " n t s }+["\n"] - def slurpstring(terminators) + def slurpstring(terminators,escapes=%w{ \\ $ ' " n t s }+["\n"],ignore_invalid_escapes=false) # we search for the next quote that isn't preceded by a # backslash; the caret is there to match empty strings str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'" @@ -529,10 +528,10 @@ class Puppet::Parser::Lexer when 't'; "\t" when 's'; " " else - if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'") + if escapes.include? ch ch else - Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" + Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" unless ignore_invalid_escapes "\\#{ch}" end end diff --git a/lib/puppet/parser/resource.rb b/lib/puppet/parser/resource.rb index c956a1106..3c451929e 100644 --- a/lib/puppet/parser/resource.rb +++ b/lib/puppet/parser/resource.rb @@ -64,6 +64,7 @@ class Puppet::Parser::Resource < Puppet::Resource # Retrieve the associated definition and evaluate it. def evaluate + @evaluated = true if klass = resource_type and ! builtin_type? finish return klass.evaluate_code(self) @@ -72,8 +73,6 @@ class Puppet::Parser::Resource < Puppet::Resource else self.fail "Cannot find definition #{type}" end - ensure - @evaluated = true end # Mark this resource as both exported and virtual, diff --git a/lib/puppet/parser/type_loader.rb b/lib/puppet/parser/type_loader.rb index 35ad49593..09aa636e1 100644 --- a/lib/puppet/parser/type_loader.rb +++ b/lib/puppet/parser/type_loader.rb @@ -88,7 +88,7 @@ class Puppet::Parser::TypeLoader nil end if result = yield(filename) - Puppet.info "Automatically imported #{name} from #{filename} into #{environment}" + Puppet.debug "Automatically imported #{name} from #{filename} into #{environment}" result.module_name = modname if modname and result.respond_to?(:module_name=) return result end diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb index 6c008bccf..986cebd0a 100644 --- a/lib/puppet/rails/host.rb +++ b/lib/puppet/rails/host.rb @@ -1,3 +1,4 @@ +require 'puppet/rails' require 'puppet/rails/resource' require 'puppet/rails/fact_name' require 'puppet/rails/source_file' diff --git a/lib/puppet/reference/indirection.rb b/lib/puppet/reference/indirection.rb index 549299b36..d14510c16 100644 --- a/lib/puppet/reference/indirection.rb +++ b/lib/puppet/reference/indirection.rb @@ -1,5 +1,5 @@ require 'puppet/indirector/indirection' -require 'puppet/checksum' +require 'puppet/util/checksums' require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' diff --git a/lib/puppet/reference/network.rb b/lib/puppet/reference/network.rb index a3bff656b..15e4f6769 100644 --- a/lib/puppet/reference/network.rb +++ b/lib/puppet/reference/network.rb @@ -1,3 +1,5 @@ +require 'puppet/network/handler' + network = Puppet::Util::Reference.newreference :network, :depth => 2, :doc => "Available network handlers and clients" do ret = "" Puppet::Network::Handler.subclasses.sort { |a,b| a.to_s <=> b.to_s }.each do |name| diff --git a/lib/puppet/resource/catalog.rb b/lib/puppet/resource/catalog.rb index 4b4342d11..a8668d844 100644 --- a/lib/puppet/resource/catalog.rb +++ b/lib/puppet/resource/catalog.rb @@ -57,7 +57,7 @@ class Puppet::Resource::Catalog < Puppet::SimpleGraph end def title_key_for_ref( ref ) - ref =~ /^([\w:]+)\[(.*)\]$/m + ref =~ /^([-\w:]+)\[(.*)\]$/m [$1, $2] end diff --git a/lib/puppet/resource/type.rb b/lib/puppet/resource/type.rb index 85c0979c1..6296d26e5 100644 --- a/lib/puppet/resource/type.rb +++ b/lib/puppet/resource/type.rb @@ -174,12 +174,29 @@ class Puppet::Resource::Type @name.is_a?(Regexp) end + # MQR TODO: + # + # The change(s) introduced by the fix for #4270 are mostly silly & should be + # removed, though we didn't realize it at the time. If it can be established/ + # ensured that nodes never call parent_type and that resource_types are always + # (as they should be) members of exactly one resource_type_collection the + # following method could / should be replaced with: + # + # def parent_type + # @parent_type ||= parent && ( + # resource_type_collection.find_or_load([name],parent,type.to_sym) || + # fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{resource_type_collection.environment}" + # ) + # end + # + # ...and then the rest of the changes around passing in scope reverted. + # def parent_type(scope = nil) return nil unless parent unless @parent_type raise "Must pass scope to parent_type when called first time" unless scope - unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", scope.namespaces, parent) + unless @parent_type = scope.environment.known_resource_types.send("find_#{type}", [name], parent) fail Puppet::ParseError, "Could not find parent resource type '#{parent}' of type #{type} in #{scope.environment}" end end diff --git a/lib/puppet/resource/type_collection.rb b/lib/puppet/resource/type_collection.rb index 6a933362f..90b6df9c1 100644 --- a/lib/puppet/resource/type_collection.rb +++ b/lib/puppet/resource/type_collection.rb @@ -153,6 +153,7 @@ class Puppet::Resource::TypeCollection end def perform_initial_import + return if Puppet.settings[:ignoreimport] parser = Puppet::Parser::Parser.new(environment) if code = Puppet.settings.uninterpolated_value(:code, environment.to_s) and code != "" parser.string = code diff --git a/lib/puppet/simple_graph.rb b/lib/puppet/simple_graph.rb index 945b1beb3..55b39fadf 100644 --- a/lib/puppet/simple_graph.rb +++ b/lib/puppet/simple_graph.rb @@ -318,16 +318,15 @@ class Puppet::SimpleGraph # to container leaves, but that would result in many more # relationships. stage_class = Puppet::Type.type(:stage) + whit_class = Puppet::Type.type(:whit) containers = other.topsort.find_all { |v| (v.is_a?(type) or v.is_a?(stage_class)) and vertex?(v) } containers.each do |container| # Get the list of children from the other graph. children = other.adjacent(container, :direction => :out) - # Just remove the container if it's empty. - if children.empty? - remove_vertex!(container) - next - end + # MQR TODO: Luke suggests that it should be possible to refactor the system so that + # container nodes are retained, thus obviating the need for the whit. + children = [whit_class.new(:name => container.name, :catalog => other)] if children.empty? # First create new edges for each of the :in edges [:in, :out].each do |dir| diff --git a/lib/puppet/type/file/content.rb b/lib/puppet/type/file/content.rb index 74b380f55..0d0bb5571 100755 --- a/lib/puppet/type/file/content.rb +++ b/lib/puppet/type/file/content.rb @@ -184,7 +184,7 @@ module Puppet end def chunk_file_from_source(source_or_content) - request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path) + request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path.sub(/^\//,'')) connection = Puppet::Network::HttpPool.http_instance(source_or_content.server, source_or_content.port) connection.request_get(indirection2uri(request), add_accept_encoding({"Accept" => "raw"})) do |response| case response.code diff --git a/lib/puppet/type/whit.rb b/lib/puppet/type/whit.rb new file mode 100644 index 000000000..6e5ba9eab --- /dev/null +++ b/lib/puppet/type/whit.rb @@ -0,0 +1,7 @@ +Puppet::Type.newtype(:whit) do + desc "The smallest possible resource type, for when you need a resource and naught else." + + newparam :name do + desc "The name of the whit, because it must have one." + end +end diff --git a/lib/puppet/type/yumrepo.rb b/lib/puppet/type/yumrepo.rb index e0ed3ad37..5d86b1c75 100644 --- a/lib/puppet/type/yumrepo.rb +++ b/lib/puppet/type/yumrepo.rb @@ -85,7 +85,7 @@ module Puppet clear inifile.each_section do |s| next if s.name == "main" - obj = create(:name => s.name, :audit => check) + obj = new(:name => s.name, :audit => check) current_values = obj.retrieve obj.eachproperty do |property| if current_values[property].nil? diff --git a/lib/puppet/util/cacher.rb b/lib/puppet/util/cacher.rb index 8785c694f..3dddec0d4 100644 --- a/lib/puppet/util/cacher.rb +++ b/lib/puppet/util/cacher.rb @@ -1,3 +1,5 @@ +require 'monitor' + module Puppet::Util::Cacher module Expirer attr_reader :timestamp @@ -49,7 +51,7 @@ module Puppet::Util::Cacher define_method(name.to_s + "=") do |value| # Make sure the cache timestamp is set cache_timestamp - value_cache[name] = value + value_cache.synchronize { value_cache[name] = value } end if ttl = options[:ttl] @@ -70,6 +72,7 @@ module Puppet::Util::Cacher # Methods that get added to instances. module InstanceMethods + def expire # Only expire if we have an expirer. This is # mostly so that we can comfortably handle cases @@ -92,15 +95,17 @@ module Puppet::Util::Cacher end def cached_value(name) - # Allow a nil expirer, in which case we regenerate the value every time. - if expired_by_expirer?(name) - value_cache.clear - @cache_timestamp = Time.now - elsif expired_by_ttl?(name) - value_cache.delete(name) + value_cache.synchronize do + # Allow a nil expirer, in which case we regenerate the value every time. + if expired_by_expirer?(name) + value_cache.clear + @cache_timestamp = Time.now + elsif expired_by_ttl?(name) + value_cache.delete(name) + end + value_cache[name] = send("init_#{name}") unless value_cache.include?(name) + value_cache[name] end - value_cache[name] = send("init_#{name}") unless value_cache.include?(name) - value_cache[name] end def expired_by_expirer?(name) @@ -121,7 +126,7 @@ module Puppet::Util::Cacher end def value_cache - @value_cache ||= {} + @value_cache ||= {}.extend(MonitorMixin) end end end diff --git a/lib/puppet/util/monkey_patches.rb b/lib/puppet/util/monkey_patches.rb index e035afd9f..9664ff310 100644 --- a/lib/puppet/util/monkey_patches.rb +++ b/lib/puppet/util/monkey_patches.rb @@ -1,4 +1,7 @@ -Process.maxgroups = 1024 + +unless defined? JRUBY_VERSION + Process.maxgroups = 1024 +end module RDoc def self.caller(skip=nil) diff --git a/lib/puppet/util/rdoc.rb b/lib/puppet/util/rdoc.rb index 4a80b069b..085d8ec93 100644 --- a/lib/puppet/util/rdoc.rb +++ b/lib/puppet/util/rdoc.rb @@ -41,7 +41,7 @@ module Puppet::Util::RDoc def manifestdoc(files) Puppet[:ignoreimport] = true files.select { |f| FileTest.file?(f) }.each do |f| - parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment]) + parser = Puppet::Parser::Parser.new(Puppet::Node::Environment.new(Puppet[:environment])) parser.file = f ast = parser.parse output(f, ast) diff --git a/lib/puppet/util/rdoc/parser.rb b/lib/puppet/util/rdoc/parser.rb index 573d1766f..63df38ab9 100644 --- a/lib/puppet/util/rdoc/parser.rb +++ b/lib/puppet/util/rdoc/parser.rb @@ -15,6 +15,8 @@ module RDoc class Parser extend ParserFactory + SITE = "__site__" + attr_accessor :ast, :input_file_name, :top_level # parser registration into RDoc @@ -74,7 +76,7 @@ class Parser # split_module tries to find if +path+ belongs to the module path # if it does, it returns the module name, otherwise if we are sure - # it is part of the global manifest path, "<site>" is returned. + # it is part of the global manifest path, "__site__" is returned. # And finally if this path couldn't be mapped anywhere, nil is returned. def split_module(path) # find a module @@ -105,7 +107,7 @@ class Parser end # we are under a global manifests Puppet.debug "rdoc: global manifests" - "<site>" + SITE end # create documentation for the top level +container+ @@ -128,7 +130,7 @@ class Parser Puppet.debug "rdoc: scanning for #{name}" container.module_name = name - container.global=true if name == "<site>" + container.global=true if name == SITE @stats.num_modules += 1 container, name = get_class_or_module(container,name) diff --git a/spec/unit/application_spec.rb b/spec/unit/application_spec.rb index 3c354cce9..433809172 100755 --- a/spec/unit/application_spec.rb +++ b/spec/unit/application_spec.rb @@ -194,7 +194,6 @@ describe Puppet::Application do it "should create a new option parser when needed" do option_parser = stub "option parser" option_parser.stubs(:on) - option_parser.stubs(:default_argv=) OptionParser.expects(:new).returns(option_parser).once @app.option_parser.should == option_parser @app.option_parser.should == option_parser @@ -203,7 +202,6 @@ describe Puppet::Application do it "should pass the banner to the option parser" do option_parser = stub "option parser" option_parser.stubs(:on) - option_parser.stubs(:default_argv=) @app.class.instance_eval do banner "banner" end @@ -213,15 +211,6 @@ describe Puppet::Application do @app.option_parser end - it "should set the optionparser's args to the command line args" do - option_parser = stub "option parser" - option_parser.stubs(:on) - option_parser.expects(:default_argv=).with(%w{ fake args }) - @app.command_line.stubs(:args).returns(%w{ fake args }) - OptionParser.expects(:new).returns(option_parser) - @app.option_parser - end - it "should get options from Puppet.settings.optparse_addargs" do Puppet.settings.expects(:optparse_addargs).returns([]) @@ -237,7 +226,8 @@ describe Puppet::Application do end it "should ask OptionParser to parse the command-line argument" do - @app.option_parser.expects(:parse!) + @app.command_line.stubs(:args).returns(%w{ fake args }) + @app.option_parser.expects(:parse!).with(%w{ fake args }) @app.parse_options end diff --git a/spec/unit/node/environment_spec.rb b/spec/unit/node/environment_spec.rb index b400865a2..6edcce56c 100755 --- a/spec/unit/node/environment_spec.rb +++ b/spec/unit/node/environment_spec.rb @@ -53,6 +53,7 @@ describe Puppet::Node::Environment do @env = Puppet::Node::Environment.new("dev") @collection = Puppet::Resource::TypeCollection.new(@env) @collection.stubs(:perform_initial_import) + Thread.current[:known_resource_types] = nil end it "should create a resource type collection if none exists" do @@ -71,13 +72,41 @@ describe Puppet::Node::Environment do @env.known_resource_types end - it "should create and return a new collection rather than returning a stale collection" do - @env.known_resource_types.expects(:stale?).returns true + it "should return the same collection even if stale if it's the same thread" do + Puppet::Resource::TypeCollection.stubs(:new).returns @collection + @env.known_resource_types.stubs(:stale?).returns true - Puppet::Resource::TypeCollection.expects(:new).returns @collection + @env.known_resource_types.should equal(@collection) + end + + it "should return the current thread associated collection if there is one" do + Thread.current[:known_resource_types] = @collection @env.known_resource_types.should equal(@collection) end + + it "should give to all threads the same collection if it didn't change" do + Puppet::Resource::TypeCollection.expects(:new).with(@env).returns @collection + @env.known_resource_types + + t = Thread.new { + @env.known_resource_types.should equal(@collection) + } + t.join + end + + it "should give to new threads a new collection if it isn't stale" do + Puppet::Resource::TypeCollection.expects(:new).with(@env).returns @collection + @env.known_resource_types.expects(:stale?).returns(true) + + Puppet::Resource::TypeCollection.expects(:new).returns @collection + + t = Thread.new { + @env.known_resource_types.should equal(@collection) + } + t.join + end + end [:modulepath, :manifestdir].each do |setting| diff --git a/spec/unit/parser/functions/extlookup_spec.rb b/spec/unit/parser/functions/extlookup_spec.rb new file mode 100755 index 000000000..bf2880345 --- /dev/null +++ b/spec/unit/parser/functions/extlookup_spec.rb @@ -0,0 +1,85 @@ +#! /usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../../spec_helper' +require 'tempfile' + +describe "the extlookup function" do + + before :each do + @scope = Puppet::Parser::Scope.new + + @scope.stubs(:environment).returns(Puppet::Node::Environment.new('production')) + end + + it "should exist" do + Puppet::Parser::Functions.function("extlookup").should == "function_extlookup" + end + + it "should raise a ParseError if there is less than 1 arguments" do + lambda { @scope.function_extlookup([]) }.should( raise_error(Puppet::ParseError)) + end + + it "should raise a ParseError if there is more than 3 arguments" do + lambda { @scope.function_extlookup(["foo", "bar", "baz", "gazonk"]) }.should( raise_error(Puppet::ParseError)) + end + + it "should return the default" do + result = @scope.function_extlookup([ "key", "default"]) + result.should == "default" + end + + it "should lookup the key in a supplied datafile" do + t = Tempfile.new('extlookup.csv') do + t.puts 'key,value' + t.puts 'nonkey,nonvalue' + t.close + + result = @scope.function_extlookup([ "key", "default", t.path]) + result.should == "value" + end + end + + it "should return an array if the datafile contains more than two columns" do + t = Tempfile.new('extlookup.csv') do + t.puts 'key,value1,value2' + t.puts 'nonkey,nonvalue,nonvalue' + t.close + + result = @scope.function_extlookup([ "key", "default", t.path]) + result.should == ["value1", "value2"] + end + end + + it "should raise an error if there's no matching key and no default" do + t = Tempfile.new('extlookup.csv') do + t.puts 'key,value' + t.puts 'nonkey,nonvalue' + t.close + + result = @scope.function_extlookup([ "key", nil, t.path]) + result.should == "value" + end + end + + describe "should look in $extlookup_datadir for data files listed by $extlookup_precedence" do + before do + @scope.stubs(:lookupvar).with('extlookup_datadir').returns("/tmp") + @scope.stubs(:lookupvar).with('extlookup_precedence').returns(["one","two"]) + File.open("/tmp/one.csv","w"){|one| one.puts "key,value1" } + File.open("/tmp/two.csv","w") do |two| + two.puts "key,value2" + two.puts "key2,value_two" + end + end + + it "when the key is in the first file" do + result = @scope.function_extlookup([ "key" ]) + result.should == "value1" + end + + it "when the key is in the second file" do + result = @scope.function_extlookup([ "key2" ]) + result.should == "value_two" + end + end +end diff --git a/spec/unit/parser/functions/include_spec.rb b/spec/unit/parser/functions/include_spec.rb new file mode 100644 index 000000000..e5f051906 --- /dev/null +++ b/spec/unit/parser/functions/include_spec.rb @@ -0,0 +1,55 @@ +#!/usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../../spec_helper' + +describe "the 'include' function" do + + before :each do + Puppet::Node::Environment.stubs(:current).returns(nil) + @compiler = Puppet::Parser::Compiler.new(Puppet::Node.new("foo")) + @scope = Puppet::Parser::Scope.new(:compiler => @compiler) + # MQR TODO: Without the following stub these tests cause hundreds of spurious errors in + # subsequent tests. With it, there are no spurious failures and all but one + # of the tests (marked pending, bellow) fail. This needs a better solution. + Puppet::Parser::Resource.stubs(:new).with('stage', :main, :scope => @scope).returns 'foo' + end + + it "should exist" do + Puppet::Parser::Functions.function("include").should == "function_include" + end + + it "should include a single class" do + inc = "foo" + @compiler.expects(:evaluate_classes).with {|klasses,parser,lazy| klasses == [inc]}.returns([inc]) + @scope.function_include("foo") + end + + it "should include multiple classes" do + inc = ["foo","bar"] + @compiler.expects(:evaluate_classes).with {|klasses,parser,lazy| klasses == inc}.returns(inc) + @scope.function_include(["foo","bar"]) + end + + it "should not lazily evaluate the included class" do + @compiler.expects(:evaluate_classes).with {|klasses,parser,lazy| lazy == false}.returns("foo") + @scope.function_include("foo") + end + + it "should allow a parent to include its child" do + pending "Resolution of MQR TODO item, above" + @parent_type = Puppet::Resource::Type.new(:hostclass, "parent") + @parent_resource = Puppet::Parser::Resource.new(:hostclass, "parent", :scope => @scope) + @subscope = @parent_type.subscope(@scope,@parent_resource) + @scope.environment.known_resource_types.stubs(:find_hostclass).with{|nses,name| name.downcase == "parent"}.returns(@parent_type) + + @type = Puppet::Resource::Type.new(:hostclass, "foo") + @type.stubs(:parent_scope).returns(@subscope) + @type.parent = "parent" + @resource = Puppet::Parser::Resource.new(:hostclass, "foo", :scope => @subscope) + @resource.stubs(:resource_type).returns(@type) + @scope.environment.known_resource_types.stubs(:find_hostclass).with{|nses,name| name.downcase == "foo"}.returns(@parent_type) + Puppet::Resource.stubs(:new).returns(@resource) + Puppet::Parser::Resource.stubs(:new).returns(@resource) + lambda { @subscope.function_include("foo") }.should_not raise_error + end +end diff --git a/spec/unit/parser/lexer_spec.rb b/spec/unit/parser/lexer_spec.rb index d3d2a0a31..a85d1b257 100755 --- a/spec/unit/parser/lexer_spec.rb +++ b/spec/unit/parser/lexer_spec.rb @@ -410,6 +410,9 @@ describe Puppet::Parser::Lexer,"when lexing strings" do %q{'single quoted string')} => [[:STRING,'single quoted string']], %q{"double quoted string"} => [[:STRING,'double quoted string']], %q{'single quoted string with an escaped "\\'"'} => [[:STRING,'single quoted string with an escaped "\'"']], + %q{'single quoted string with an escaped "\$"'} => [[:STRING,'single quoted string with an escaped "\$"']], + %q{'single quoted string with an escaped "\."'} => [[:STRING,'single quoted string with an escaped "\."']], + %q{'single quoted string with an escaped "\\\\"'} => [[:STRING,'single quoted string with an escaped "\\\\"']], %q{"string with an escaped '\\"'"} => [[:STRING,"string with an escaped '\"'"]], %q{"string with an escaped '\\$'"} => [[:STRING,"string with an escaped '$'"]], %q{"string with $v (but no braces)"} => [[:DQPRE,"string with "],[:VARIABLE,'v'],[:DQPOST,' (but no braces)']], diff --git a/spec/unit/resource/type_collection_spec.rb b/spec/unit/resource/type_collection_spec.rb index 09643cd24..45fc05da0 100644 --- a/spec/unit/resource/type_collection_spec.rb +++ b/spec/unit/resource/type_collection_spec.rb @@ -400,6 +400,14 @@ describe Puppet::Resource::TypeCollection do @parser.expects(:parse).raises ArgumentError lambda { @code.perform_initial_import }.should raise_error(Puppet::Error) end + + it "should not do anything if the ignore_import settings is set" do + Puppet.settings[:ignoreimport] = true + @parser.expects(:string=).never + @parser.expects(:file=).never + @parser.expects(:parse).never + @code.perform_initial_import + end end describe "when determining the configuration version" do diff --git a/spec/unit/simple_graph_spec.rb b/spec/unit/simple_graph_spec.rb index 83e674139..2ca8888c5 100755 --- a/spec/unit/simple_graph_spec.rb +++ b/spec/unit/simple_graph_spec.rb @@ -439,7 +439,8 @@ describe Puppet::SimpleGraph do @middle = Container.new("middle", ["e", "f", @two]) @top = Container.new("top", ["g", "h", @middle, @one, @three]) @empty = Container.new("empty", []) - + + @whit = Puppet::Type.type(:whit) @stage = Puppet::Type.type(:stage).new(:name => "foo") @contgraph = @top.to_graph @@ -499,8 +500,17 @@ describe Puppet::SimpleGraph do end end + it "should contain a whit-resource to mark the place held by the empty container" do + @depgraph.vertices.find_all { |v| v.is_a?(@whit) }.length.should == 1 + end + + it "should replace edges to empty containers with edges to their residual whit" do + emptys_whit = @depgraph.vertices.find_all { |v| v.is_a?(@whit) }.first + @depgraph.should be_edge("c", emptys_whit) + end + it "should no longer contain anything but the non-container objects" do - @depgraph.vertices.find_all { |v| ! v.is_a?(String) }.should be_empty + @depgraph.vertices.find_all { |v| ! v.is_a?(String) and ! v.is_a?(@whit)}.should be_empty end it "should copy labels" do diff --git a/spec/unit/type/file/content_spec.rb b/spec/unit/type/file/content_spec.rb index e4b9f9bc6..a01a14a59 100755 --- a/spec/unit/type/file/content_spec.rb +++ b/spec/unit/type/file/content_spec.rb @@ -362,7 +362,7 @@ describe content do end it "should send the correct indirection uri" do - @conn.expects(:request_get).with { |uri,headers| uri == "/production/file_content//path/to/source" }.yields(@response) + @conn.expects(:request_get).with { |uri,headers| uri == "/production/file_content/path/to/source" }.yields(@response) @content.write(@fh) end diff --git a/spec/unit/util/rdoc/parser_spec.rb b/spec/unit/util/rdoc/parser_spec.rb index a9c8190a6..79195e657 100755 --- a/spec/unit/util/rdoc/parser_spec.rb +++ b/spec/unit/util/rdoc/parser_spec.rb @@ -85,8 +85,8 @@ describe RDoc::Parser do @parser.scan_top_level(@topcontainer) end - it "should set the module as global if we parse the global manifests (ie <site> module)" do - @parser.stubs(:split_module).returns("<site>") + it "should set the module as global if we parse the global manifests (ie __site__ module)" do + @parser.stubs(:split_module).returns(RDoc::Parser::SITE) @parser.stubs(:parse_elements) @topcontainer.expects(:global=).with(true) @@ -131,7 +131,7 @@ describe RDoc::Parser do it "should return <site> for manifests not under module path" do File.stubs(:expand_path).returns("/path/to/manifests/init.pp") File.stubs(:identical?).returns(false) - @parser.split_module("/path/to/manifests/init.pp").should == "<site>" + @parser.split_module("/path/to/manifests/init.pp").should == RDoc::Parser::SITE end end diff --git a/spec/unit/util/rdoc_spec.rb b/spec/unit/util/rdoc_spec.rb index 65df26156..58c2034ee 100755 --- a/spec/unit/util/rdoc_spec.rb +++ b/spec/unit/util/rdoc_spec.rb @@ -75,6 +75,19 @@ describe Puppet::Util::RDoc do Puppet::Util::RDoc.manifestdoc([]) end + it "should use a parser with the correct environment" do + FileTest.stubs(:file?).returns(true) + Puppet::Util::RDoc.stubs(:output) + + parser = stub_everything + Puppet::Parser::Parser.stubs(:new).with{ |env| env.is_a?(Puppet::Node::Environment) }.returns(parser) + + parser.expects(:file=).with("file") + parser.expects(:parse) + + Puppet::Util::RDoc.manifestdoc(["file"]) + end + it "should puppet parse all given files" do FileTest.stubs(:file?).returns(true) Puppet::Util::RDoc.stubs(:output) |