From 6b1dd81799a44288287d9ab0cdf46afa3aaf090a Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Thu, 5 Aug 2010 10:34:35 -0700 Subject: [#4472]+[#4483] Moved type-name resolution out of Puppet::Parser::TypeLoader. Moved type-name resolution out of Puppet::Parser::TypeLoader, and into its primary client, Puppet::Resource::TypeCollection. TypeCollection now always passes fully qualified type names to TypeLoader. This avoids duplicate type-name resolution logic between TypeLoader and TypeCollection. That in turn fixes bug 4472, which resulted from flaws in the type-name resolution logic in TypeLoader. In addition, it fixes bug 4483, which resulted from improper interleaving between looking up names using the TypeCollection and the TypeLoader. --- lib/puppet/parser/parser_support.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index c0fd37178..97d985cfb 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -103,11 +103,11 @@ class Puppet::Parser::Parser end def find_hostclass(namespace, name) - known_resource_types.find_or_load(namespace, name, :hostclass) + known_resource_types.find_hostclass(namespace, name) end def find_definition(namespace, name) - known_resource_types.find_or_load(namespace, name, :definition) + known_resource_types.find_definition(namespace, name) end def import(file) -- cgit From 4da88fb4cd57871f16649d50572240ac3f7420f0 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 13 Aug 2010 15:43:34 -0700 Subject: [#4496]+[#4521]+[#4522] Add structures to the AST to represent type definitions (classes, definitions, and nodes). Previously, type definitions were not represented directly in the AST. Instead, the parser would instantiate types and insert them into known_resource_types as soon as they were parsed. This made it difficult to distinguish which types had come from the file that was just parsed and which types had been loaded previously, which led to bug 4496. A side-effect of this change is that the user is no longer allowed to define types inside of conditional constructs (such as if/else). This was allowed before but had unexpected semantics (bugs 4521 and 4522). It is still possible, however, to place an "include" statement inside a conditional construct, and have that "include" statement trigger the autoloading of a file that instantiates types. --- lib/puppet/parser/parser_support.rb | 91 +++++++++++++++---------------------- 1 file changed, 36 insertions(+), 55 deletions(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index 97d985cfb..859897a16 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -133,26 +133,6 @@ class Puppet::Parser::Parser return ns, n end - # Create a new class, or merge with an existing class. - def newclass(name, options = {}) - known_resource_types.add Puppet::Resource::Type.new(:hostclass, name, ast_context(true).merge(options)) - end - - # Create a new definition. - def newdefine(name, options = {}) - known_resource_types.add Puppet::Resource::Type.new(:definition, name, ast_context(true).merge(options)) - end - - # Create a new node. Nodes are special, because they're stored in a global - # table, not according to namespaces. - def newnode(names, options = {}) - names = [names] unless names.instance_of?(Array) - context = ast_context(true) - names.collect do |name| - known_resource_types.add(Puppet::Resource::Type.new(:node, name, context.merge(options))) - end - end - def on_error(token,value,stack) if token == 0 # denotes end of file value = 'end of file' @@ -174,42 +154,43 @@ class Puppet::Parser::Parser # how should I do error handling here? def parse(string = nil) - return parse_ruby_file if self.file =~ /\.rb$/ - self.string = string if string - begin - @yydebug = false - main = yyparse(@lexer,:scan) - rescue Racc::ParseError => except - error = Puppet::ParseError.new(except) - error.line = @lexer.line - error.file = @lexer.file - error.set_backtrace except.backtrace - raise error - rescue Puppet::ParseError => except - except.line ||= @lexer.line - except.file ||= @lexer.file - raise except - rescue Puppet::Error => except - # and this is a framework error - except.line ||= @lexer.line - except.file ||= @lexer.file - raise except - rescue Puppet::DevError => except - except.line ||= @lexer.line - except.file ||= @lexer.file - raise except - rescue => except - error = Puppet::DevError.new(except.message) - error.line = @lexer.line - error.file = @lexer.file - error.set_backtrace except.backtrace - raise error - end - if main - # Store the results as the top-level class. - newclass("", :code => main) + if self.file =~ /\.rb$/ + parse_ruby_file + main = nil + else + self.string = string if string + begin + @yydebug = false + main = yyparse(@lexer,:scan) + rescue Racc::ParseError => except + error = Puppet::ParseError.new(except) + error.line = @lexer.line + error.file = @lexer.file + error.set_backtrace except.backtrace + raise error + rescue Puppet::ParseError => except + except.line ||= @lexer.line + except.file ||= @lexer.file + raise except + rescue Puppet::Error => except + # and this is a framework error + except.line ||= @lexer.line + except.file ||= @lexer.file + raise except + rescue Puppet::DevError => except + except.line ||= @lexer.line + except.file ||= @lexer.file + raise except + rescue => except + error = Puppet::DevError.new(except.message) + error.line = @lexer.line + error.file = @lexer.file + error.set_backtrace except.backtrace + raise error + end end - return known_resource_types + # Store the results as the top-level class. + return Puppet::Parser::AST::Hostclass.new('', :code => main) ensure @lexer.clear end -- cgit From df088c9ba16dce50c17a79920c1ac186db67b9e9 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Wed, 25 Aug 2010 11:29:23 -0700 Subject: [4638] Cleanup of plurals and inheritance relationships in AST Changed the grammar so that the following "plural" constructs always parse as an ASTArray: - funcvalues - rvalues - resourceinstances - anyparams - params - caseopts - casevalues And the following "singluar" construct never parses as an ASTArray: - statement The previous behavior was for these constructs to parse as a scalar when they represented a single item and an ASTArray when they contained zero or multiple items. ("Statement" could sometimes represent a single item because a single resource declaration could represent multiple resources). This complicated other grammar rules and caused ambiguous handling of nested arrays. Also made these changes to the AST class hierarchy: - ResourceInstance no longer derives from ASTArray. This relationship was not meaningful because a ResourceInstance is a (title, parameters) pair, not an array, and it produced complications when we wanted to represent an array of ResourceInstance objects. - Resource no longer derives from ResourceReference. No significant functionality was being inherited and the relationship doesn't make sense in an AST context. - ResourceOverride no longer derives from Resource. No significant functionality was being inherited and the relationship doesn't make sense in an AST context. - Resource can now represent a compound resource instance such as "notify { foo: ; bar: }". This saves the parser from having to use represent a statement as an array of objects. - ASTArray's evaluate method never flattens out arrays of arrays. --- lib/puppet/parser/parser_support.rb | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index 859897a16..1c4947891 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -29,18 +29,9 @@ class Puppet::Parser::Parser message end - # Create an AST array out of all of the args - def aryfy(*args) - if args[0].instance_of?(AST::ASTArray) - result = args.shift - args.each { |arg| - result.push arg - } - else - result = ast AST::ASTArray, :children => args - end - - result + # Create an AST array containing a single element + def aryfy(arg) + ast AST::ASTArray, :children => [arg] end # Create an AST object, and automatically add the file and line information if -- cgit From 6b278503021c4404904f56ced6995d0fbfa5b8fe Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Thu, 2 Sep 2010 18:10:37 -0700 Subject: [#4657] Customer-supplied .rb files are not compatible with multiple environments or staleness check Changed the resource type API to create AST objects rather than directly instantiating resource types. This allows the same code paths to be used to handle the results of parsing both .pp and .rb files. This makes .rb files work properly in multiple environments, because the types are now instantiated by code that is aware of which environment the compilation is happening in. It also reduces the risk of future changes breaking .rb file support. Also, switched to using "instance_eval" rather than "require" to evaluate the contents of the .rb file. This ensures that if the file has to be recompiled (because it became stale), it will actually get re-evaluated. As a side benefit, ResourceTypeAPI is now a class rather than a mixin to Object, so its methods do not pollute the global namespace. To reduce the risk of customers coming to rely on implementation details of the resource type API, changed its methods to return nil, and removed methods from it that were misleadingly labeled as "private". --- lib/puppet/parser/parser_support.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index 859897a16..a9df33f8b 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -155,8 +155,7 @@ class Puppet::Parser::Parser # how should I do error handling here? def parse(string = nil) if self.file =~ /\.rb$/ - parse_ruby_file - main = nil + main = parse_ruby_file else self.string = string if string begin @@ -196,7 +195,13 @@ class Puppet::Parser::Parser end def parse_ruby_file - require self.file + # Execute the contents of the file inside its own "main" object so + # that it can call methods in the resource type API. + main_object = Puppet::DSL::ResourceTypeAPI.new + main_object.instance_eval(File.read(self.file)) + + # Then extract any types that were created. + Puppet::Parser::AST::ASTArray.new :children => main_object.instance_eval { @__created_ast_objects__ } end def string=(string) -- cgit From ce9bf1edcaac4901de6e0a7da413d1742d216eb0 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Tue, 7 Sep 2010 18:01:42 -0700 Subject: Modified the error message that is generated when a class, definition, or node occurs in a conditional construct so that it contains the proper line number. --- lib/puppet/parser/parser_support.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index 859897a16..2b2571d87 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -68,13 +68,13 @@ class Puppet::Parser::Parser end # Raise a Parse error. - def error(message) + def error(message, options = {}) if brace = @lexer.expected message += "; expected '%s'" end except = Puppet::ParseError.new(message) - except.line = @lexer.line - except.file = @lexer.file if @lexer.file + except.line = options[:line] || @lexer.line + except.file = options[:file] || @lexer.file raise except end -- cgit From 2b50f30c703aca5c4f3e89961d64a94d886296bd Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 17 Sep 2010 17:59:56 -0700 Subject: [#4771] Import of manifests with the same name only happens once The function import_if_possible, which was supposed to be responsible for making sure that no two threads tried to import the same file at the same time, was not making this decision based on the full pathname of the file, since it was being invoked before pathnames were resolved. As a result, if we attempted to import two distinct files with the same name at the same time (either in two threads or in a single thread due to recursion), one of the files would not always get imported. Fixed this problem by moving the thread-safety logic to happen after filenames are resolved to absolute paths. This made it possible to simplify the thread-safety logic significantly. --- lib/puppet/parser/parser_support.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index c0fd37178..4f3a4ddff 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -111,7 +111,7 @@ class Puppet::Parser::Parser end def import(file) - known_resource_types.loader.import_if_possible(file, @lexer.file) + known_resource_types.loader.import(file, @lexer.file) end def initialize(env) -- cgit From f95006148c3a0b4d7e8ee1812b1993b674f050e4 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Fri, 3 Sep 2010 11:17:35 -0700 Subject: [#4716] ResourceTypeAPI exposes implementation details that are likely to change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made the following modifications to ResourceTypeAPI: (1) returned nil from “define”, “hostclass”, and “node”. (2) renamed “mk_resource_type” and “munge_type_arguments” to “__mk_resource_type__” and “__munge_type_arguments__” to discourage customers from calling them. (3) Made ResourceTypeAPI a class rather than a module, and changed the parser to evaluate the contents of pure ruby manifests using a instances of this class. (4) Changed ResourceTypeAPI to insert newly instantiated types into Thread.current[:known_resource_types] rather than the default environment's known_resource_types. This effectively backports the fix for issue #4657 to 2.6.x. Also backported the new spec tests from #4657. --- lib/puppet/parser/parser_support.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index 4f3a4ddff..c90c1978f 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -215,7 +215,9 @@ class Puppet::Parser::Parser end def parse_ruby_file - require self.file + # Execute the contents of the file inside its own "main" object so + # that it can call methods in the resource type API. + Puppet::DSL::ResourceTypeAPI.new.instance_eval(File.read(self.file)) end def string=(string) -- cgit From 9604f1c4cd5a368da08c6f3219b44908a9b9921c Mon Sep 17 00:00:00 2001 From: Brice Figureau Date: Thu, 11 Nov 2010 15:51:21 +0100 Subject: Fix #5252 - line number mis-attribution during parsing It is a resurgence of #2366 that appeared because of the commit 8971d8. Before this commit, for associating documentation comments, we were preferring line numbers coming from the parser currently reducing rule, instead of the current lexer line number (which can be in advance of several tokens due to the nature of LALR parsers). We now merge the ast line number before fetching the comment from the lexer. Signed-off-by: Brice Figureau --- lib/puppet/parser/parser_support.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/puppet/parser/parser_support.rb') diff --git a/lib/puppet/parser/parser_support.rb b/lib/puppet/parser/parser_support.rb index c90c1978f..7bbebb124 100644 --- a/lib/puppet/parser/parser_support.rb +++ b/lib/puppet/parser/parser_support.rb @@ -46,12 +46,12 @@ class Puppet::Parser::Parser # Create an AST object, and automatically add the file and line information if # available. def ast(klass, hash = {}) - klass.new ast_context(klass.use_docs).merge(hash) + klass.new ast_context(klass.use_docs, hash[:line]).merge(hash) end - def ast_context(include_docs = false) + def ast_context(include_docs = false, ast_line = nil) result = { - :line => lexer.line, + :line => ast_line || lexer.line, :file => lexer.file } result[:doc] = lexer.getcomment(result[:line]) if include_docs -- cgit