From 23830504dfeb48b2d162e44b84b6f9dfa97e4e7e Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Thu, 22 Jul 2010 20:36:24 +0200 Subject: Fix #4302 - Compilation speed regression compared to 2.6 Each time the compiler was accessing the loaded types, we were checking if the manifests had changed. This incurred a large performance cost compared to 0.25 and introduced race conditions if manifests changed while a thread was in the middle of a compilation. This tentative fix, based on Brice's, makes sure each thread will get access to the same loaded types collection for the durration of a compilation, even if the manifests change. We now only check for changed files at the start of a compilation or if the environment changes, and we maintain a per environment thread lock so that only one thread at a time can be reloading any particular environment (and the need-check is done inside the synchronize block so that only the first will actually load it). As long as the manifests don't change, the threads will share the same collection, so there is only duplication in memory for a brief window surrounding a change. Signed-off-by: Brice Figureau Second-author: Markus Roberts --- lib/puppet/node/environment.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 762599cff..3f67474f9 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -1,4 +1,5 @@ require 'puppet/util/cacher' +require 'monitor' # Just define it, so this class has fewer load dependencies. class Puppet::Node @@ -67,14 +68,23 @@ class Puppet::Node::Environment def initialize(name) @name = name + extend MonitorMixin end def known_resource_types - if @known_resource_types.nil? or @known_resource_types.stale? - @known_resource_types = Puppet::Resource::TypeCollection.new(self) - @known_resource_types.perform_initial_import - end - @known_resource_types + # This makes use of short circuit evaluation to get the right thread-safe + # per environment semantics with an efficient most common cases; we almost + # always just return our thread's known-resource types. Only at the start + # of a compilation (after our thread var has been set to nil) or when the + # environment has changed do we delve deeper. + Thread.current[:known_resource_types] = nil if (krt = Thread.current[:known_resource_types]) && krt.environment != self + Thread.current[:known_resource_types] ||= synchronize { + if @known_resource_types.nil? or @known_resource_types.stale? + @known_resource_types = Puppet::Resource::TypeCollection.new(self) + @known_resource_types.perform_initial_import + end + @known_resource_types + } end def module(name) -- cgit From caca187dffbd6e628d7eda599c7f2939dd05fafc Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 13 Aug 2010 15:25:17 -0700 Subject: Moved perform_initial_import from Puppet::Resource::TypeCollection to Puppet::Node::Environment. This change is part of an ongoing effort to remove functionality from TypeCollection that is not related to keeping track of a collection of types. This reduces TypeCollection's linkage to the environment, which is a step toward decoupling it from the type loading mechanism. Also, added a spec test to verify that TypeCollection.version is correctly recomputed when types are re-imported. --- lib/puppet/node/environment.rb | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 3f67474f9..ad11a0476 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -81,7 +81,7 @@ class Puppet::Node::Environment Thread.current[:known_resource_types] ||= synchronize { if @known_resource_types.nil? or @known_resource_types.stale? @known_resource_types = Puppet::Resource::TypeCollection.new(self) - @known_resource_types.perform_initial_import + perform_initial_import end @known_resource_types } @@ -143,5 +143,25 @@ class Puppet::Node::Environment end end + private + + def perform_initial_import + return if Puppet.settings[:ignoreimport] + parser = Puppet::Parser::Parser.new(self) + if code = Puppet.settings.uninterpolated_value(:code, name.to_s) and code != "" + parser.string = code + else + file = Puppet.settings.value(:manifest, name.to_s) + return unless File.exist?(file) + parser.file = file + end + parser.parse + rescue => detail + msg = "Could not parse for environment #{self}: #{detail}" + error = Puppet::Error.new(msg) + error.set_backtrace(detail.backtrace) + raise error + end + @root = new(:'*root*') end -- cgit From 4da88fb4cd57871f16649d50572240ac3f7420f0 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 13 Aug 2010 15:43:34 -0700 Subject: [#4496]+[#4521]+[#4522] Add structures to the AST to represent type definitions (classes, definitions, and nodes). Previously, type definitions were not represented directly in the AST. Instead, the parser would instantiate types and insert them into known_resource_types as soon as they were parsed. This made it difficult to distinguish which types had come from the file that was just parsed and which types had been loaded previously, which led to bug 4496. A side-effect of this change is that the user is no longer allowed to define types inside of conditional constructs (such as if/else). This was allowed before but had unexpected semantics (bugs 4521 and 4522). It is still possible, however, to place an "include" statement inside a conditional construct, and have that "include" statement trigger the autoloading of a file that instantiates types. --- lib/puppet/node/environment.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index ad11a0476..3d55ac1a0 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -81,7 +81,7 @@ class Puppet::Node::Environment Thread.current[:known_resource_types] ||= synchronize { if @known_resource_types.nil? or @known_resource_types.stale? @known_resource_types = Puppet::Resource::TypeCollection.new(self) - perform_initial_import + @known_resource_types.import_ast(perform_initial_import, '') end @known_resource_types } @@ -146,13 +146,13 @@ class Puppet::Node::Environment private def perform_initial_import - return if Puppet.settings[:ignoreimport] + return empty_parse_result if Puppet.settings[:ignoreimport] parser = Puppet::Parser::Parser.new(self) if code = Puppet.settings.uninterpolated_value(:code, name.to_s) and code != "" parser.string = code else file = Puppet.settings.value(:manifest, name.to_s) - return unless File.exist?(file) + return empty_parse_result unless File.exist?(file) parser.file = file end parser.parse @@ -163,5 +163,11 @@ class Puppet::Node::Environment raise error end + def empty_parse_result + # Return an empty toplevel hostclass to use as the result of + # perform_initial_import when no file was actually loaded. + return Puppet::Parser::AST::Hostclass.new('') + end + @root = new(:'*root*') end -- cgit From c2ea112de1f08707aa301060b4df24bd0bb6072a Mon Sep 17 00:00:00 2001 From: Nick Lewis Date: Thu, 28 Oct 2010 12:49:19 -0700 Subject: (#5148) Add support for PSON to facts Previously, facts could be fetched via the REST API in PSON, but came back as the to_s representation of a Ruby object, rather than as proper PSON data. This patch adds to_pson and from_pson to facts, so they can be properly used with PSON. --- lib/puppet/node/facts.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/facts.rb b/lib/puppet/node/facts.rb index b77ad22d5..ad4b91e98 100755 --- a/lib/puppet/node/facts.rb +++ b/lib/puppet/node/facts.rb @@ -1,12 +1,17 @@ +require 'time' + require 'puppet/node' require 'puppet/indirector' +require 'puppet/util/pson' + # Manage a given node's facts. This either accepts facts and stores them, or # returns facts for a given node. class Puppet::Node::Facts # Set up indirection, so that nodes can be looked for in # the node sources. extend Puppet::Indirector + extend Puppet::Util::Pson # We want to expire any cached nodes if the facts are saved. module NodeExpirer @@ -54,6 +59,22 @@ class Puppet::Node::Facts strip_internal == other.send(:strip_internal) end + def self.from_pson(data) + result = new(data['name'], data['values']) + result.values[:_timestamp] = Time.parse(data['timestamp']) + result.expiration = Time.parse(data['expiration']) + result + end + + def to_pson(*args) + { + 'expiration' => expiration, + 'name' => name, + 'timestamp' => values[:_timestamp], + 'values' => values.reject {|k,v| k == :_timestamp}, + }.to_pson(*args) + end + private # Add internal data to the facts for storage. -- cgit From 45a9a97285d99db524d5330c236352b29e5884ed Mon Sep 17 00:00:00 2001 From: Matt Robinson Date: Thu, 28 Oct 2010 14:49:37 -0700 Subject: (#5132) Provide a query REST interface for inventory This REST interface returns a list of nodes that match a fact query. Fact queries can use (in)equality testing as a string comparison, and >, <, >=, <= numerical comparisons. Multiple tests can be done as AND comparisons, not OR. The fact queries need to be prefixed by facts, and the comparisons other than equality are specified with a .comparison_type after the fact name. This will be better explained in the REST documentation on the website. Searches that don't match anything now return empty array instead of a 404 error. --- lib/puppet/node/inventory.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lib/puppet/node/inventory.rb (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/inventory.rb b/lib/puppet/node/inventory.rb new file mode 100644 index 000000000..fd99163b0 --- /dev/null +++ b/lib/puppet/node/inventory.rb @@ -0,0 +1,7 @@ +require 'puppet/node' +require 'puppet/indirector' + +class Puppet::Node::Inventory + extend Puppet::Indirector + indirects :inventory, :terminus_setting => :inventory_terminus +end -- cgit From fb5f859cf4a89042a1768b6cbc2dbfc43da49c99 Mon Sep 17 00:00:00 2001 From: Jesse Wolfe Date: Mon, 1 Nov 2010 11:45:27 -0700 Subject: Fix #5164 Change Facts timestamp when they are received by the master This patch causes the puppet master to re-timestamp facts when they are received by the catalog compiler terminus. This makes the timestamps more trustworthy, as it means that they are all based upon the same clock's time. Paired-With: Paul Berry --- lib/puppet/node/facts.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/facts.rb b/lib/puppet/node/facts.rb index ad4b91e98..d84d54113 100755 --- a/lib/puppet/node/facts.rb +++ b/lib/puppet/node/facts.rb @@ -35,7 +35,7 @@ class Puppet::Node::Facts @name = name @values = values - add_internal + add_timestamp end def downcase_if_necessary @@ -75,13 +75,21 @@ class Puppet::Node::Facts }.to_pson(*args) end - private - # Add internal data to the facts for storage. - def add_internal - self.values[:_timestamp] = Time.now + def add_timestamp + self.timestamp = Time.now + end + + def timestamp=(time) + self.values[:_timestamp] = time end + def timestamp + self.values[:_timestamp] + end + + private + # Strip out that internal data. def strip_internal newvals = values.dup -- cgit From fbb096a1eb6394da3199c03645986a1315132272 Mon Sep 17 00:00:00 2001 From: Markus Roberts Date: Wed, 20 Oct 2010 11:33:20 -0700 Subject: Fix for #5055 -- adding to_sym to Puppet::Node::Environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The currious part is that this wasn't noticed before since it appears to block server-first migration to 2.6.x and doesn’t appear to be the consequence of a recent (2.6.3) change (unless, as is quite possible, I’m missing something). --- lib/puppet/node/environment.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib/puppet/node') diff --git a/lib/puppet/node/environment.rb b/lib/puppet/node/environment.rb index 3f67474f9..b64fb8a1f 100644 --- a/lib/puppet/node/environment.rb +++ b/lib/puppet/node/environment.rb @@ -124,6 +124,10 @@ class Puppet::Node::Environment name.to_s end + def to_sym + to_s.to_sym + end + # The only thing we care about when serializing an environment is its # identity; everything else is ephemeral and should not be stored or # transmitted. -- cgit