From 0056d4174959a4f07d669eae7b6a768d18891594 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Thu, 26 Aug 2010 15:13:52 +1000 Subject: Fixed extlookup documentation and spacing --- lib/puppet/parser/functions/extlookup.rb | 275 +++++++++++++++---------------- lib/puppet/parser/functions/require.rb | 2 +- 2 files changed, 134 insertions(+), 143 deletions(-) (limited to 'lib/puppet/parser') diff --git a/lib/puppet/parser/functions/extlookup.rb b/lib/puppet/parser/functions/extlookup.rb index a4786f5da..63d49e563 100644 --- a/lib/puppet/parser/functions/extlookup.rb +++ b/lib/puppet/parser/functions/extlookup.rb @@ -1,166 +1,157 @@ -# Puppet External Data Sources -# -# This is a parser function to read data from external files, this version -# uses CSV files but the concept can easily be adjust for databases, yaml -# or any other queryable data source. -# -# The object of this is to make it obvious when it's being used, rather than -# magically loading data in when an module is loaded I prefer to look at the code -# and see statements like: -# -# $snmp_contact = extlookup("snmp_contact") -# -# The above snippet will load the snmp_contact value from CSV files, this in its -# own is useful but a common construct in puppet manifests is something like this: -# -# case $domain { -# "myclient.com": { $snmp_contact = "John Doe " } -# default: { $snmp_contact = "My Support " } -# } -# -# Over time there will be a lot of this kind of thing spread all over your manifests -# and adding an additional client involves grepping through manifests to find all the -# places where you have constructs like this. -# -# This is a data problem and shouldn't be handled in code, a using this function you -# can do just that. -# -# First you configure it in site.pp: -# $extlookup_datadir = "/etc/puppet/manifests/extdata" -# $extlookup_precedence = ["%{fqdn}", "domain_%{domain}", "common"] -# -# The array tells the code how to resolve values, first it will try to find it in -# web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv -# -# Now create the following data files in /etc/puppet/manifests/extdata -# -# domain_myclient.com.csv: -# snmp_contact,John Doe -# root_contact,support@%{domain} -# client_trusted_ips,192.168.1.130,192.168.10.0/24 -# -# common.csv: -# snmp_contact,My Support -# root_contact,support@my.com -# -# Now you can replace the case statement with the simple single line to achieve -# the exact same outcome: -# -# $snmp_contact = extlookup("snmp_contact") -# -# The obove code shows some other features, you can use any fact or variable that -# is in scope by simply using %{varname} in your data files, you can return arrays -# by just having multiple values in the csv after the initial variable name. -# -# In the event that a variable is nowhere to be found a critical error will be raised -# that will prevent your manifest from compiling, this is to avoid accidentally putting -# in empty values etc. You can however specify a default value: -# -# $ntp_servers = extlookup("ntp_servers", "1.${country}.pool.ntp.org") -# -# In this case it will default to "1.${country}.pool.ntp.org" if nothing is defined in -# any data file. -# -# You can also specify an additional data file to search first before any others at use -# time, for example: -# -# $version = extlookup("rsyslog_version", "present", "packages") -# -# package{"rsyslog": ensure => $version } -# -# This will look for a version configured in packages.csv and then in the rest as configured -# by $extlookup_precedence if it's not found anywhere it will default to "present", this kind -# of use case makes puppet a lot nicer for managing large amounts of packages since you do not -# need to edit a load of manifests to do simple things like adjust a desired version number. -# -# For more information on installing and writing your own custom functions see: -# http://docs.puppetlabs.com/guides/custom_functions.html -# -# For further help contact Volcane on #puppet require 'csv' module Puppet::Parser::Functions - newfunction(:extlookup, :type => :rvalue) do |args| - key = args[0] - - default = args[1] - datafile = args[2] - - raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3 - - extlookup_datadir = lookupvar('extlookup_datadir') - extlookup_precedence = Array.new - - # precedence values can have variables embedded in them - # in the form %{fqdn}, you could for example do - # - # $extlookup_precedence = ["hosts/%{fqdn}", "common"] - # - # this will result in /path/to/extdata/hosts/your.box.com.csv - # being searched. - # - # This is for back compatibility to interpolate variables with %. - # % interpolation is a workaround for a problem that has been fixed: Puppet variable - # interpolation at top scope used to only happen on each run - extlookup_precedence = lookupvar('extlookup_precedence').collect do |var| - var.gsub(/%\{(.+?)\}/) do |capture| - lookupvar($1) - end - end + newfunction(:extlookup, + :type => :rvalue, + :doc => "This is a parser function to read data from external files, this version +uses CSV files but the concept can easily be adjust for databases, yaml +or any other queryable data source. - datafiles = Array.new +The object of this is to make it obvious when it's being used, rather than +magically loading data in when an module is loaded I prefer to look at the code +and see statements like: - # if we got a custom data file, put it first in the array of search files - if datafile != "" - datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv") - end + $snmp_contact = extlookup(\"snmp_contact\") + +The above snippet will load the snmp_contact value from CSV files, this in its +own is useful but a common construct in puppet manifests is something like this: + + case $domain { + \"myclient.com\": { $snmp_contact = \"John Doe \" } + default: { $snmp_contact = \"My Support \" } + } + +Over time there will be a lot of this kind of thing spread all over your manifests +and adding an additional client involves grepping through manifests to find all the +places where you have constructs like this. + +This is a data problem and shouldn't be handled in code, a using this function you +can do just that. + +First you configure it in site.pp: + + $extlookup_datadir = \"/etc/puppet/manifests/extdata\" + $extlookup_precedence = [\"%{fqdn}\", \"domain_%{domain}\", \"common\"] + +The array tells the code how to resolve values, first it will try to find it in +web1.myclient.com.csv then in domain_myclient.com.csv and finally in common.csv + +Now create the following data files in /etc/puppet/manifests/extdata: + + domain_myclient.com.csv: + snmp_contact,John Doe + root_contact,support@%{domain} + client_trusted_ips,192.168.1.130,192.168.10.0/24 + + common.csv: + snmp_contact,My Support + root_contact,support@my.com + +Now you can replace the case statement with the simple single line to achieve +the exact same outcome: + + $snmp_contact = extlookup(\"snmp_contact\") + +The obove code shows some other features, you can use any fact or variable that +is in scope by simply using %{varname} in your data files, you can return arrays +by just having multiple values in the csv after the initial variable name. + +In the event that a variable is nowhere to be found a critical error will be raised +that will prevent your manifest from compiling, this is to avoid accidentally putting +in empty values etc. You can however specify a default value: + + $ntp_servers = extlookup(\"ntp_servers\", \"1.${country}.pool.ntp.org\") + +In this case it will default to \"1.${country}.pool.ntp.org\" if nothing is defined in +any data file. + +You can also specify an additional data file to search first before any others at use +time, for example: + + $version = extlookup(\"rsyslog_version\", \"present\", \"packages\") + package{\"rsyslog\": ensure => $version } - extlookup_precedence.each do |d| - datafiles << extlookup_datadir + "/#{d}.csv" +This will look for a version configured in packages.csv and then in the rest as configured +by $extlookup_precedence if it's not found anywhere it will default to `present`, this kind +of use case makes puppet a lot nicer for managing large amounts of packages since you do not +need to edit a load of manifests to do simple things like adjust a desired version number. + +Precedence values can have variables embedded in them in the form %{fqdn}, you could for example do: + + $extlookup_precedence = [\"hosts/%{fqdn}\", \"common\"] + +This will result in /path/to/extdata/hosts/your.box.com.csv being searched. + +This is for back compatibility to interpolate variables with %. % interpolation is a workaround for a problem that has been fixed: Puppet variable interpolation at top scope used to only happen on each run.") do |args| + + key = args[0] + + default = args[1] + datafile = args[2] + + raise Puppet::ParseError, ("extlookup(): wrong number of arguments (#{args.length}; must be <= 3)") if args.length > 3 + + extlookup_datadir = lookupvar('extlookup_datadir') + extlookup_precedence = Array.new + + extlookup_precedence = lookupvar('extlookup_precedence').collect do |var| + var.gsub(/%\{(.+?)\}/) do |capture| + lookupvar($1) end + end - desired = nil + datafiles = Array.new - datafiles.each do |file| - if desired.nil? - if File.exists?(file) - result = CSV.read(file).find_all do |r| - r[0] == key - end + # if we got a custom data file, put it first in the array of search files + if datafile != "" + datafiles << extlookup_datadir + "/#{datafile}.csv" if File.exists?(extlookup_datadir + "/#{datafile}.csv") + end + extlookup_precedence.each do |d| + datafiles << extlookup_datadir + "/#{d}.csv" + end - # return just the single result if theres just one, - # else take all the fields in the csv and build an array - if result.length > 0 - if result[0].length == 2 - val = result[0][1].to_s + desired = nil - # parse %{}'s in the CSV into local variables using lookupvar() - while val =~ /%\{(.+?)\}/ - val.gsub!(/%\{#{$1}\}/, lookupvar($1)) - end + datafiles.each do |file| + if desired.nil? + if File.exists?(file) + result = CSV.read(file).find_all do |r| + r[0] == key + end - desired = val - elsif result[0].length > 1 - length = result[0].length - cells = result[0][1,length] + # return just the single result if theres just one, + # else take all the fields in the csv and build an array + if result.length > 0 + if result[0].length == 2 + val = result[0][1].to_s - # Individual cells in a CSV result are a weird data type and throws - # puppets yaml parsing, so just map it all to plain old strings - desired = cells.map do |c| - # parse %{}'s in the CSV into local variables using lookupvar() - while c =~ /%\{(.+?)\}/ - c.gsub!(/%\{#{$1}\}/, lookupvar($1)) - end + # parse %{}'s in the CSV into local variables using lookupvar() + while val =~ /%\{(.+?)\}/ + val.gsub!(/%\{#{$1}\}/, lookupvar($1)) + end + + desired = val + elsif result[0].length > 1 + length = result[0].length + cells = result[0][1,length] - c.to_s + # Individual cells in a CSV result are a weird data type and throws + # puppets yaml parsing, so just map it all to plain old strings + desired = cells.map do |c| + # parse %{}'s in the CSV into local variables using lookupvar() + while c =~ /%\{(.+?)\}/ + c.gsub!(/%\{#{$1}\}/, lookupvar($1)) end + + c.to_s end end end end end + end - desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" + desired || default or raise Puppet::ParseError, "No match found for '#{key}' in any data file during extlookup()" end end diff --git a/lib/puppet/parser/functions/require.rb b/lib/puppet/parser/functions/require.rb index f15046b91..64285307e 100644 --- a/lib/puppet/parser/functions/require.rb +++ b/lib/puppet/parser/functions/require.rb @@ -12,7 +12,7 @@ relationships between classes. This function is a superset of the class depends on the required class. Warning: using require in place of include can lead to unwanted dependency cycles. - + For instance the following manifest, with 'require' instead of 'include' would produce a nasty dependence cycle, because notify imposes a before between File[/foo] and Service[foo]: class myservice { -- cgit From 4a9c85763b7bf2db6da52daa9e8221eb59ffa9d2 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Sun, 5 Sep 2010 11:26:58 -0700 Subject: Fix for #4693 -- implicit stages should never be serialized My fix for #4542 was overly enthusiastic about assuring that all resources had a stage, resulting in stages designations being serialized for resources in manifests which did not use resources (everything was in implicit main). This broke 0.25.x compatibility, as all catalogs now refered to stages. This patch scales back the change for #4542 slightly, supressing the setting of main on the puppetmaster and relying on the default behaviour on the client (for 2.6.x and later, treat it as main; for 0.25.x, do nothing). --- lib/puppet/parser/compiler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet/parser') diff --git a/lib/puppet/parser/compiler.rb b/lib/puppet/parser/compiler.rb index 7504b276b..e1227e753 100644 --- a/lib/puppet/parser/compiler.rb +++ b/lib/puppet/parser/compiler.rb @@ -71,7 +71,7 @@ class Puppet::Parser::Compiler raise ArgumentError, "Could not find stage #{resource[:stage] || :main} specified by #{resource}" end - resource[:stage] ||= stage.title + resource[:stage] ||= stage.title unless stage.title == :main @catalog.add_edge(stage, resource) end -- cgit