diff options
Diffstat (limited to 'lib/puppet/parser/lexer.rb')
| -rw-r--r-- | lib/puppet/parser/lexer.rb | 1004 |
1 files changed, 502 insertions, 502 deletions
diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb index 0c95142f9..6a9f1cfc4 100644 --- a/lib/puppet/parser/lexer.rb +++ b/lib/puppet/parser/lexer.rb @@ -5,573 +5,573 @@ require 'puppet' module Puppet - class LexError < RuntimeError; end + class LexError < RuntimeError; end end module Puppet::Parser; end class Puppet::Parser::Lexer - attr_reader :last, :file, :lexing_context, :token_queue + attr_reader :last, :file, :lexing_context, :token_queue - attr_accessor :line, :indefine + attr_accessor :line, :indefine - def lex_error msg - raise Puppet::LexError.new(msg) - end - - class Token - attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text, :accumulate - - def initialize(regex, name) - if regex.is_a?(String) - @name, @string = name, regex - @regex = Regexp.new(Regexp.escape(@string)) - else - @name, @regex = name, regex - end - end - - # MQR: Why not just alias? - %w{skip accumulate}.each do |method| - define_method(method+"?") do - self.send(method) - end - end - - def to_s - if self.string - @string - else - @name.to_s - end - end - - def acceptable?(context={}) - # By default tokens are aceeptable in any context - true - end - end - - # Maintain a list of tokens. - class TokenList - attr_reader :regex_tokens, :string_tokens - - def [](name) - @tokens[name] - end - - # Create a new token. - def add_token(name, regex, options = {}, &block) - token = Token.new(regex, name) - raise(ArgumentError, "Token #{name} already exists") if @tokens.include?(name) - @tokens[token.name] = token - if token.string - @string_tokens << token - @tokens_by_string[token.string] = token - else - @regex_tokens << token - end - - options.each do |name, option| - token.send(name.to_s + "=", option) - end - - token.meta_def(:convert, &block) if block_given? - - token - end - - def initialize - @tokens = {} - @regex_tokens = [] - @string_tokens = [] - @tokens_by_string = {} - end - - # Look up a token by its value, rather than name. - def lookup(string) - @tokens_by_string[string] - end - - # Define more tokens. - def add_tokens(hash) - hash.each do |regex, name| - add_token(name, regex) - end - end - - # Sort our tokens by length, so we know once we match, we're done. - # This helps us avoid the O(n^2) nature of token matching. - def sort_tokens - @string_tokens.sort! { |a, b| b.string.length <=> a.string.length } - end - end + def lex_error msg + raise Puppet::LexError.new(msg) + end - TOKENS = TokenList.new - - TOKENS.add_tokens( - - '[' => :LBRACK, - ']' => :RBRACK, - '{' => :LBRACE, - '}' => :RBRACE, - '(' => :LPAREN, - - ')' => :RPAREN, - '=' => :EQUALS, - '+=' => :APPENDS, - '==' => :ISEQUAL, - '>=' => :GREATEREQUAL, - '>' => :GREATERTHAN, - '<' => :LESSTHAN, - '<=' => :LESSEQUAL, - '!=' => :NOTEQUAL, - '!' => :NOT, - ',' => :COMMA, - '.' => :DOT, - ':' => :COLON, - '@' => :AT, - '<<|' => :LLCOLLECT, - '->' => :IN_EDGE, - '<-' => :OUT_EDGE, - '~>' => :IN_EDGE_SUB, - '<~' => :OUT_EDGE_SUB, - '|>>' => :RRCOLLECT, - '<|' => :LCOLLECT, - '|>' => :RCOLLECT, - ';' => :SEMIC, - '?' => :QMARK, - '\\' => :BACKSLASH, - '=>' => :FARROW, - '+>' => :PARROW, - '+' => :PLUS, - '-' => :MINUS, - '/' => :DIV, - '*' => :TIMES, - '<<' => :LSHIFT, - '>>' => :RSHIFT, - '=~' => :MATCH, - '!~' => :NOMATCH, - %r{([a-z][-\w]*)?(::[a-z][-\w]*)+} => :CLASSNAME, # Require '::' in the class name, else we'd compete with NAME - %r{((::){0,1}[A-Z][-\w]*)+} => :CLASSREF, - "<string>" => :STRING, - "<dqstring up to first interpolation>" => :DQPRE, - "<dqstring between two interpolations>" => :DQMID, - "<dqstring after final interpolation>" => :DQPOST, - "<boolean>" => :BOOLEAN - ) - - TOKENS.add_token :NUMBER, %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b} do |lexer, value| - [TOKENS[:NAME], value] - end - #:stopdoc: # Issue #4161 - def (TOKENS[:NUMBER]).acceptable?(context={}) - ![:DQPRE,:DQMID].include? context[:after] - end - #:startdoc: + class Token + attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text, :accumulate - TOKENS.add_token :NAME, %r{[a-z0-9][-\w]*} do |lexer, value| - string_token = self - # we're looking for keywords here - if tmp = KEYWORDS.lookup(value) - string_token = tmp - if [:TRUE, :FALSE].include?(string_token.name) - value = eval(value) - string_token = TOKENS[:BOOLEAN] - end - end - [string_token, value] + def initialize(regex, name) + if regex.is_a?(String) + @name, @string = name, regex + @regex = Regexp.new(Regexp.escape(@string)) + else + @name, @regex = name, regex + end end - [:NAME,:CLASSNAME,:CLASSREF].each { |name_token| - #:stopdoc: # Issue #4161 - def (TOKENS[name_token]).acceptable?(context={}) - ![:DQPRE,:DQMID].include? context[:after] - end - #:startdoc: - } - TOKENS.add_token :COMMENT, %r{#.*}, :accumulate => true, :skip => true do |lexer,value| - value.sub!(/# ?/,'') - [self, value] + # MQR: Why not just alias? + %w{skip accumulate}.each do |method| + define_method(method+"?") do + self.send(method) + end end - TOKENS.add_token :MLCOMMENT, %r{/\*(.*?)\*/}m, :accumulate => true, :skip => true do |lexer, value| - lexer.line += value.count("\n") - value.sub!(/^\/\* ?/,'') - value.sub!(/ ?\*\/$/,'') - [self,value] + def to_s + if self.string + @string + else + @name.to_s + end end - TOKENS.add_token :REGEX, %r{/[^/\n]*/} do |lexer, value| - # Make sure we haven't matched an escaped / - while value[-2..-2] == '\\' - other = lexer.scan_until(%r{/}) - value += other - end - regex = value.sub(%r{\A/}, "").sub(%r{/\Z}, '').gsub("\\/", "/") - [self, Regexp.new(regex)] - end - - #:stopdoc: # Issue #4161 - def (TOKENS[:REGEX]).acceptable?(context={}) - [:NODE,:LBRACE,:RBRACE,:MATCH,:NOMATCH,:COMMA].include? context[:after] + def acceptable?(context={}) + # By default tokens are aceeptable in any context + true end - #:startdoc: + end - TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true + # Maintain a list of tokens. + class TokenList + attr_reader :regex_tokens, :string_tokens - TOKENS.add_token :SQUOTE, "'" do |lexer, value| - [TOKENS[:STRING], lexer.slurpstring(value).first ] + def [](name) + @tokens[name] end - DQ_initial_token_types = {'$' => :DQPRE,'"' => :STRING} - DQ_continuation_token_types = {'$' => :DQMID,'"' => :DQPOST} + # Create a new token. + def add_token(name, regex, options = {}, &block) + token = Token.new(regex, name) + raise(ArgumentError, "Token #{name} already exists") if @tokens.include?(name) + @tokens[token.name] = token + if token.string + @string_tokens << token + @tokens_by_string[token.string] = token + else + @regex_tokens << token + end - TOKENS.add_token :DQUOTE, /"/ do |lexer, value| - lexer.tokenize_interpolated_string(DQ_initial_token_types) - end + options.each do |name, option| + token.send(name.to_s + "=", option) + end - TOKENS.add_token :DQCONT, /\}/ do |lexer, value| - lexer.tokenize_interpolated_string(DQ_continuation_token_types) - end - #:stopdoc: # Issue #4161 - def (TOKENS[:DQCONT]).acceptable?(context={}) - context[:string_interpolation_depth] > 0 - end - #:startdoc: + token.meta_def(:convert, &block) if block_given? - TOKENS.add_token :DOLLAR_VAR, %r{\$(\w*::)*\w+} do |lexer, value| - [TOKENS[:VARIABLE],value[1..-1]] + token end - TOKENS.add_token :VARIABLE, %r{(\w*::)*\w+} + def initialize + @tokens = {} + @regex_tokens = [] + @string_tokens = [] + @tokens_by_string = {} + end + + # Look up a token by its value, rather than name. + def lookup(string) + @tokens_by_string[string] + end + + # Define more tokens. + def add_tokens(hash) + hash.each do |regex, name| + add_token(name, regex) + end + end + + # Sort our tokens by length, so we know once we match, we're done. + # This helps us avoid the O(n^2) nature of token matching. + def sort_tokens + @string_tokens.sort! { |a, b| b.string.length <=> a.string.length } + end + end + + TOKENS = TokenList.new + + TOKENS.add_tokens( + + '[' => :LBRACK, + ']' => :RBRACK, + '{' => :LBRACE, + '}' => :RBRACE, + '(' => :LPAREN, + + ')' => :RPAREN, + '=' => :EQUALS, + '+=' => :APPENDS, + '==' => :ISEQUAL, + '>=' => :GREATEREQUAL, + '>' => :GREATERTHAN, + '<' => :LESSTHAN, + '<=' => :LESSEQUAL, + '!=' => :NOTEQUAL, + '!' => :NOT, + ',' => :COMMA, + '.' => :DOT, + ':' => :COLON, + '@' => :AT, + '<<|' => :LLCOLLECT, + '->' => :IN_EDGE, + '<-' => :OUT_EDGE, + '~>' => :IN_EDGE_SUB, + '<~' => :OUT_EDGE_SUB, + '|>>' => :RRCOLLECT, + '<|' => :LCOLLECT, + '|>' => :RCOLLECT, + ';' => :SEMIC, + '?' => :QMARK, + '\\' => :BACKSLASH, + '=>' => :FARROW, + '+>' => :PARROW, + '+' => :PLUS, + '-' => :MINUS, + '/' => :DIV, + '*' => :TIMES, + '<<' => :LSHIFT, + '>>' => :RSHIFT, + '=~' => :MATCH, + '!~' => :NOMATCH, + %r{([a-z][-\w]*)?(::[a-z][-\w]*)+} => :CLASSNAME, # Require '::' in the class name, else we'd compete with NAME + %r{((::){0,1}[A-Z][-\w]*)+} => :CLASSREF, + "<string>" => :STRING, + "<dqstring up to first interpolation>" => :DQPRE, + "<dqstring between two interpolations>" => :DQMID, + "<dqstring after final interpolation>" => :DQPOST, + "<boolean>" => :BOOLEAN + ) + + TOKENS.add_token :NUMBER, %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b} do |lexer, value| + [TOKENS[:NAME], value] + end + #:stopdoc: # Issue #4161 + def (TOKENS[:NUMBER]).acceptable?(context={}) + ![:DQPRE,:DQMID].include? context[:after] + end + #:startdoc: + + TOKENS.add_token :NAME, %r{[a-z0-9][-\w]*} do |lexer, value| + string_token = self + # we're looking for keywords here + if tmp = KEYWORDS.lookup(value) + string_token = tmp + if [:TRUE, :FALSE].include?(string_token.name) + value = eval(value) + string_token = TOKENS[:BOOLEAN] + end + end + [string_token, value] + end + [:NAME,:CLASSNAME,:CLASSREF].each { |name_token| #:stopdoc: # Issue #4161 - def (TOKENS[:VARIABLE]).acceptable?(context={}) - [:DQPRE,:DQMID].include? context[:after] + def (TOKENS[name_token]).acceptable?(context={}) + ![:DQPRE,:DQMID].include? context[:after] end #:startdoc: - - - TOKENS.sort_tokens - - @@pairs = { - "{" => "}", - "(" => ")", - "[" => "]", - "<|" => "|>", - "<<|" => "|>>" + } + + TOKENS.add_token :COMMENT, %r{#.*}, :accumulate => true, :skip => true do |lexer,value| + value.sub!(/# ?/,'') + [self, value] + end + + TOKENS.add_token :MLCOMMENT, %r{/\*(.*?)\*/}m, :accumulate => true, :skip => true do |lexer, value| + lexer.line += value.count("\n") + value.sub!(/^\/\* ?/,'') + value.sub!(/ ?\*\/$/,'') + [self,value] + end + + TOKENS.add_token :REGEX, %r{/[^/\n]*/} do |lexer, value| + # Make sure we haven't matched an escaped / + while value[-2..-2] == '\\' + other = lexer.scan_until(%r{/}) + value += other + end + regex = value.sub(%r{\A/}, "").sub(%r{/\Z}, '').gsub("\\/", "/") + [self, Regexp.new(regex)] + end + + #:stopdoc: # Issue #4161 + def (TOKENS[:REGEX]).acceptable?(context={}) + [:NODE,:LBRACE,:RBRACE,:MATCH,:NOMATCH,:COMMA].include? context[:after] + end + #:startdoc: + + TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true + + TOKENS.add_token :SQUOTE, "'" do |lexer, value| + [TOKENS[:STRING], lexer.slurpstring(value).first ] + end + + DQ_initial_token_types = {'$' => :DQPRE,'"' => :STRING} + DQ_continuation_token_types = {'$' => :DQMID,'"' => :DQPOST} + + TOKENS.add_token :DQUOTE, /"/ do |lexer, value| + lexer.tokenize_interpolated_string(DQ_initial_token_types) + end + + TOKENS.add_token :DQCONT, /\}/ do |lexer, value| + lexer.tokenize_interpolated_string(DQ_continuation_token_types) + end + #:stopdoc: # Issue #4161 + def (TOKENS[:DQCONT]).acceptable?(context={}) + context[:string_interpolation_depth] > 0 + end + #:startdoc: + + TOKENS.add_token :DOLLAR_VAR, %r{\$(\w*::)*\w+} do |lexer, value| + [TOKENS[:VARIABLE],value[1..-1]] + end + + TOKENS.add_token :VARIABLE, %r{(\w*::)*\w+} + #:stopdoc: # Issue #4161 + def (TOKENS[:VARIABLE]).acceptable?(context={}) + [:DQPRE,:DQMID].include? context[:after] + end + #:startdoc: + + + TOKENS.sort_tokens + + @@pairs = { + "{" => "}", + "(" => ")", + "[" => "]", + "<|" => "|>", + "<<|" => "|>>" + } + + KEYWORDS = TokenList.new + + + KEYWORDS.add_tokens( + + "case" => :CASE, + "class" => :CLASS, + "default" => :DEFAULT, + "define" => :DEFINE, + "import" => :IMPORT, + "if" => :IF, + "elsif" => :ELSIF, + "else" => :ELSE, + "inherits" => :INHERITS, + "node" => :NODE, + "and" => :AND, + "or" => :OR, + "undef" => :UNDEF, + "false" => :FALSE, + "true" => :TRUE, + + "in" => :IN + ) + + def clear + initvars + end + + def expected + return nil if @expected.empty? + name = @expected[-1] + TOKENS.lookup(name) or lex_error "Could not find expected token #{name}" + end + + # scan the whole file + # basically just used for testing + def fullscan + array = [] + + self.scan { |token, str| + # Ignore any definition nesting problems + @indefine = false + array.push([token,str]) } - - KEYWORDS = TokenList.new - - - KEYWORDS.add_tokens( - - "case" => :CASE, - "class" => :CLASS, - "default" => :DEFAULT, - "define" => :DEFINE, - "import" => :IMPORT, - "if" => :IF, - "elsif" => :ELSIF, - "else" => :ELSE, - "inherits" => :INHERITS, - "node" => :NODE, - "and" => :AND, - "or" => :OR, - "undef" => :UNDEF, - "false" => :FALSE, - "true" => :TRUE, - - "in" => :IN - ) - - def clear - initvars - end - - def expected - return nil if @expected.empty? - name = @expected[-1] - TOKENS.lookup(name) or lex_error "Could not find expected token #{name}" - end - - # scan the whole file - # basically just used for testing - def fullscan - array = [] - - self.scan { |token, str| - # Ignore any definition nesting problems - @indefine = false - array.push([token,str]) - } - array - end - - def file=(file) - @file = file - @line = 1 - @scanner = StringScanner.new(File.read(file)) - end - - def shift_token - @token_queue.shift - end - - def find_string_token - # We know our longest string token is three chars, so try each size in turn - # until we either match or run out of chars. This way our worst-case is three - # tries, where it is otherwise the number of string token we have. Also, - # the lookups are optimized hash lookups, instead of regex scans. - # - s = @scanner.peek(3) - token = TOKENS.lookup(s[0,3]) || TOKENS.lookup(s[0,2]) || TOKENS.lookup(s[0,1]) - [ token, token && @scanner.scan(token.regex) ] - end - - # Find the next token that matches a regex. We look for these first. - def find_regex_token - @regex += 1 - best_token = nil - best_length = 0 - - # I tried optimizing based on the first char, but it had - # a slightly negative affect and was a good bit more complicated. - TOKENS.regex_tokens.each do |token| - if length = @scanner.match?(token.regex) and token.acceptable?(lexing_context) - # We've found a longer match - if length > best_length - best_length = length - best_token = token - end - end + array + end + + def file=(file) + @file = file + @line = 1 + @scanner = StringScanner.new(File.read(file)) + end + + def shift_token + @token_queue.shift + end + + def find_string_token + # We know our longest string token is three chars, so try each size in turn + # until we either match or run out of chars. This way our worst-case is three + # tries, where it is otherwise the number of string token we have. Also, + # the lookups are optimized hash lookups, instead of regex scans. + # + s = @scanner.peek(3) + token = TOKENS.lookup(s[0,3]) || TOKENS.lookup(s[0,2]) || TOKENS.lookup(s[0,1]) + [ token, token && @scanner.scan(token.regex) ] + end + + # Find the next token that matches a regex. We look for these first. + def find_regex_token + @regex += 1 + best_token = nil + best_length = 0 + + # I tried optimizing based on the first char, but it had + # a slightly negative affect and was a good bit more complicated. + TOKENS.regex_tokens.each do |token| + if length = @scanner.match?(token.regex) and token.acceptable?(lexing_context) + # We've found a longer match + if length > best_length + best_length = length + best_token = token end + end + end - return best_token, @scanner.scan(best_token.regex) if best_token - end - - # Find the next token, returning the string and the token. - def find_token - @find += 1 - shift_token || find_regex_token || find_string_token - end + return best_token, @scanner.scan(best_token.regex) if best_token + end - def indefine? - if defined?(@indefine) - @indefine - else - false - end - end + # Find the next token, returning the string and the token. + def find_token + @find += 1 + shift_token || find_regex_token || find_string_token + end - def initialize - @find = 0 - @regex = 0 - initvars + def indefine? + if defined?(@indefine) + @indefine + else + false end + end - def initvars - @line = 1 - @previous_token = nil - @scanner = nil - @file = nil - # AAARRGGGG! okay, regexes in ruby are bloody annoying - # no one else has "\n" =~ /\s/ - @skip = %r{[ \t\r]+} - - @namestack = [] - @token_queue = [] - @indefine = false - @expected = [] - @commentstack = [ ['', @line] ] - @lexing_context = { - :after => nil, - :start_of_line => true, - :string_interpolation_depth => 0 - } - end + def initialize + @find = 0 + @regex = 0 + initvars + end - # Make any necessary changes to the token and/or value. - def munge_token(token, value) - @line += 1 if token.incr_line + def initvars + @line = 1 + @previous_token = nil + @scanner = nil + @file = nil + # AAARRGGGG! okay, regexes in ruby are bloody annoying + # no one else has "\n" =~ /\s/ + @skip = %r{[ \t\r]+} - skip if token.skip_text + @namestack = [] + @token_queue = [] + @indefine = false + @expected = [] + @commentstack = [ ['', @line] ] + @lexing_context = { + :after => nil, + :start_of_line => true, + :string_interpolation_depth => 0 + } + end - return if token.skip and not token.accumulate? + # Make any necessary changes to the token and/or value. + def munge_token(token, value) + @line += 1 if token.incr_line - token, value = token.convert(self, value) if token.respond_to?(:convert) + skip if token.skip_text - return unless token + return if token.skip and not token.accumulate? - if token.accumulate? - comment = @commentstack.pop - comment[0] << value + "\n" - @commentstack.push(comment) - end + token, value = token.convert(self, value) if token.respond_to?(:convert) - return if token.skip + return unless token - return token, { :value => value, :line => @line } + if token.accumulate? + comment = @commentstack.pop + comment[0] << value + "\n" + @commentstack.push(comment) end - # Go up one in the namespace. - def namepop - @namestack.pop - end + return if token.skip - # Collect the current namespace. - def namespace - @namestack.join("::") - end + return token, { :value => value, :line => @line } + end - # This value might have :: in it, but we don't care -- it'll be - # handled normally when joining, and when popping we want to pop - # this full value, however long the namespace is. - def namestack(value) - @namestack << value - end - - def rest - @scanner.rest - end - - # this is the heart of the lexer - def scan - #Puppet.debug("entering scan") - lex_error "Invalid or empty string" unless @scanner - - # Skip any initial whitespace. - skip + # Go up one in the namespace. + def namepop + @namestack.pop + end - until token_queue.empty? and @scanner.eos? do - yielded = false - matched_token, value = find_token + # Collect the current namespace. + def namespace + @namestack.join("::") + end - # error out if we didn't match anything at all - lex_error "Could not match #{@scanner.rest[/^(\S+|\s+|.*)/]}" unless matched_token + # This value might have :: in it, but we don't care -- it'll be + # handled normally when joining, and when popping we want to pop + # this full value, however long the namespace is. + def namestack(value) + @namestack << value + end - newline = matched_token.name == :RETURN + def rest + @scanner.rest + end - # this matches a blank line; eat the previously accumulated comments - getcomment if lexing_context[:start_of_line] and newline - lexing_context[:start_of_line] = newline + # this is the heart of the lexer + def scan + #Puppet.debug("entering scan") + lex_error "Invalid or empty string" unless @scanner - final_token, token_value = munge_token(matched_token, value) + # Skip any initial whitespace. + skip - unless final_token - skip - next - end + until token_queue.empty? and @scanner.eos? do + yielded = false + matched_token, value = find_token - lexing_context[:after] = final_token.name unless newline - lexing_context[:string_interpolation_depth] += 1 if final_token.name == :DQPRE - lexing_context[:string_interpolation_depth] -= 1 if final_token.name == :DQPOST + # error out if we didn't match anything at all + lex_error "Could not match #{@scanner.rest[/^(\S+|\s+|.*)/]}" unless matched_token - value = token_value[:value] + newline = matched_token.name == :RETURN + + # this matches a blank line; eat the previously accumulated comments + getcomment if lexing_context[:start_of_line] and newline + lexing_context[:start_of_line] = newline - if match = @@pairs[value] and final_token.name != :DQUOTE and final_token.name != :SQUOTE - @expected << match - elsif exp = @expected[-1] and exp == value and final_token.name != :DQUOTE and final_token.name != :SQUOTE - @expected.pop - end + final_token, token_value = munge_token(matched_token, value) - if final_token.name == :LBRACE - commentpush - end - - yield [final_token.name, token_value] + unless final_token + skip + next + end - if @previous_token - namestack(value) if @previous_token.name == :CLASS + lexing_context[:after] = final_token.name unless newline + lexing_context[:string_interpolation_depth] += 1 if final_token.name == :DQPRE + lexing_context[:string_interpolation_depth] -= 1 if final_token.name == :DQPOST - if @previous_token.name == :DEFINE - if indefine? - msg = "Cannot nest definition #{value} inside #{@indefine}" - self.indefine = false - raise Puppet::ParseError, msg - end + value = token_value[:value] - @indefine = value - end - end - @previous_token = final_token - skip - end - @scanner = nil + if match = @@pairs[value] and final_token.name != :DQUOTE and final_token.name != :SQUOTE + @expected << match + elsif exp = @expected[-1] and exp == value and final_token.name != :DQUOTE and final_token.name != :SQUOTE + @expected.pop + end - # This indicates that we're done parsing. - yield [false,false] - end + if final_token.name == :LBRACE + commentpush + end - # Skip any skipchars in our remaining string. - def skip - @scanner.skip(@skip) - end + yield [final_token.name, token_value] - # Provide some limited access to the scanner, for those - # tokens that need it. - def scan_until(regex) - @scanner.scan_until(regex) - end + if @previous_token + namestack(value) if @previous_token.name == :CLASS - # we've encountered the start of a string... - # slurp in the rest of the string and return it - Valid_escapes_in_strings = %w{ \\ $ ' " n t s }+["\n"] - def slurpstring(terminators) - # we search for the next quote that isn't preceded by a - # backslash; the caret is there to match empty strings - str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'" - @line += str.count("\n") # literal carriage returns add to the line count. - str.gsub!(/\\(.)/) { - case ch=$1 - when 'n'; "\n" - when 't'; "\t" - when 's'; " " - else - if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'") - ch - else - Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" - "\\#{ch}" - end - end - } - [ str[0..-2],str[-1,1] ] - end + if @previous_token.name == :DEFINE + if indefine? + msg = "Cannot nest definition #{value} inside #{@indefine}" + self.indefine = false + raise Puppet::ParseError, msg + end - def tokenize_interpolated_string(token_type) - value,terminator = slurpstring('"$') - token_queue << [TOKENS[token_type[terminator]],value] - while terminator == '$' and not @scanner.scan(/\{/) - token_queue << [TOKENS[:VARIABLE],@scanner.scan(%r{(\w*::)*\w+|[0-9]})] - value,terminator = slurpstring('"$') - token_queue << [TOKENS[DQ_continuation_token_types[terminator]],value] + @indefine = value end - token_queue.shift - end - - # just parse a string, not a whole file - def string=(string) - @scanner = StringScanner.new(string) - end - - # returns the content of the currently accumulated content cache - def commentpop - @commentstack.pop[0] - end - - def getcomment(line = nil) - comment = @commentstack.last - if line.nil? or comment[1] <= line - @commentstack.pop - @commentstack.push(['', @line]) - return comment[0] + end + @previous_token = final_token + skip + end + @scanner = nil + + # This indicates that we're done parsing. + yield [false,false] + end + + # Skip any skipchars in our remaining string. + def skip + @scanner.skip(@skip) + end + + # Provide some limited access to the scanner, for those + # tokens that need it. + def scan_until(regex) + @scanner.scan_until(regex) + end + + # we've encountered the start of a string... + # slurp in the rest of the string and return it + Valid_escapes_in_strings = %w{ \\ $ ' " n t s }+["\n"] + def slurpstring(terminators) + # we search for the next quote that isn't preceded by a + # backslash; the caret is there to match empty strings + str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'" + @line += str.count("\n") # literal carriage returns add to the line count. + str.gsub!(/\\(.)/) { + case ch=$1 + when 'n'; "\n" + when 't'; "\t" + when 's'; " " + else + if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'") + ch + else + Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}" + "\\#{ch}" end - '' - end - - def commentpush - @commentstack.push(['', @line]) - end + end + } + [ str[0..-2],str[-1,1] ] + end + + def tokenize_interpolated_string(token_type) + value,terminator = slurpstring('"$') + token_queue << [TOKENS[token_type[terminator]],value] + while terminator == '$' and not @scanner.scan(/\{/) + token_queue << [TOKENS[:VARIABLE],@scanner.scan(%r{(\w*::)*\w+|[0-9]})] + value,terminator = slurpstring('"$') + token_queue << [TOKENS[DQ_continuation_token_types[terminator]],value] + end + token_queue.shift + end + + # just parse a string, not a whole file + def string=(string) + @scanner = StringScanner.new(string) + end + + # returns the content of the currently accumulated content cache + def commentpop + @commentstack.pop[0] + end + + def getcomment(line = nil) + comment = @commentstack.last + if line.nil? or comment[1] <= line + @commentstack.pop + @commentstack.push(['', @line]) + return comment[0] + end + '' + end + + def commentpush + @commentstack.push(['', @line]) + end end |