Add a doc attribute to AST nodes and fill it with the last seen comments

The lexer maintains a stack of last seen comments.
On blank lines the lexer flush the comments.
On each opening brace the lexer enters a new stack level.
On each block AST nodes, the stack is popped.

Each AST nodes has a doc property that is filled with the
last seen comments on node creation (in fact only on important node
creation representing statements).

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

1 files changed, 52 insertions, 8 deletions

diff --git a/lib/puppet/parser/lexer.rb b/lib/puppet/parser/lexer.rb
index dd6c29d9f..69a46d0c1 100644
--- a/lib/puppet/parser/lexer.rb
+++ b/lib/puppet/parser/lexer.rb
@@ -17,7 +17,7 @@ class Puppet::Parser::Lexer
 
     # Our base token class.
     class Token
-        attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text
+        attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text, :accumulate
 
         def initialize(regex, name)
             if regex.is_a?(String)
@@ -28,8 +28,10 @@ class Puppet::Parser::Lexer
             end
         end
 
-        def skip?
-            self.skip
+        %w{skip accumulate}.each do |method|
+            define_method(method+"?") do
+                self.send(method)
+            end
         end
 
         def to_s
@@ -155,11 +157,16 @@ class Puppet::Parser::Lexer
         [string_token, value]
     end
 
-    TOKENS.add_token :COMMENT, %r{#.*}, :skip => true
+    TOKENS.add_token :COMMENT, %r{#.*}, :accumulate => true, :skip => true do |lexer,value|
+        value.sub!(/# ?/,'')
+        [self, value]
+    end
 
-    TOKENS.add_token :MLCOMMENT, %r{/\*(.*?)\*/}m do |lexer, value|
+    TOKENS.add_token :MLCOMMENT, %r{/\*(.*?)\*/}m, :accumulate => true, :skip => true do |lexer, value|
         lexer.line += value.count("\n")
-        [nil,nil]
+        value.sub!(/^\/\* ?/,'')
+        value.sub!(/ ?\*\/$/,'')
+        [self,value]
     end
 
     TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true
@@ -325,6 +332,7 @@ class Puppet::Parser::Lexer
         @namestack = []
         @indefine = false
         @expected = []
+        @commentstack = ['']
     end
 
     # Make any necessary changes to the token and/or value.
@@ -333,12 +341,18 @@ class Puppet::Parser::Lexer
 
         skip() if token.skip_text
 
-        return if token.skip
+        return if token.skip and not token.accumulate?
 
         token, value = token.convert(self, value) if token.respond_to?(:convert)
 
         return unless token
 
+        if token.accumulate?
+            @commentstack.last << value + "\n"
+        end
+
+        return if token.skip
+
         return token, value
     end
 
@@ -389,6 +403,18 @@ class Puppet::Parser::Lexer
                 raise "Could not match '%s'" % nword
             end
 
+            if matched_token.name == :RETURN
+                # this matches a blank line
+                if @last_return
+                    # eat the previously accumulated comments
+                    getcomment
+                end
+                # since :RETURN skips, we won't survive to munge_token
+                @last_return = true
+            else
+                @last_return = false
+            end
+
             final_token, value = munge_token(matched_token, value)
 
             next unless final_token
@@ -399,6 +425,10 @@ class Puppet::Parser::Lexer
                 @expected.pop
             end
 
+            if final_token.name == :LBRACE
+                commentpush
+            end
+
             yield [final_token.name, value]
 
             if @previous_token
@@ -414,7 +444,6 @@ class Puppet::Parser::Lexer
                     @indefine = value
                 end
             end
-
             @previous_token = final_token
             skip()
         end
@@ -453,4 +482,19 @@ class Puppet::Parser::Lexer
     def string=(string)
         @scanner = StringScanner.new(string)
     end
+
+    # returns the content of the currently accumulated content cache
+    def commentpop
+        return @commentstack.pop
+    end
+
+    def getcomment
+        comment = @commentstack.pop
+        @commentstack.push('')
+        return comment
+    end
+
+    def commentpush
+        @commentstack.push('')
+    end
 end