Make sure the parser sees the correct line number

Careful inspection of the parser code show that when we
associate a source line number for an AST node, we use the
current line number of the currently lexed token.
In many case, this is correct, but there are some cases where
this is incorrect.
Unfortunately due to how LALR parser works the ast node creation
of a statement can appear _after_ we lexed another token after
the current statement:

1. $foo = 1
2.
3. class test

When the parser asks for the class token, it can reduce the
assignement statement into the AST VarDef node, because no other
grammar rule match. Unfortunately we already lexed the class token
so we affect to the VarDef node the line number 3 instead of 1.

This is not a real issue for error reporting, but becomes a real
concern when we associate documentation comments to AST node for
puppetdoc.

The solution is to enhance the tokens lexed and returned to the parser
to carry their declaration line number.
Thus a token value becomes a hash: { :value => tokenvalue, :line }

Next, each time we create an AST node, we use the line number of
the correct token (ie the foo line number in the previous example).

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

1 files changed, 24 insertions, 10 deletions

diff --git a/spec/unit/parser/lexer.rb b/spec/unit/parser/lexer.rb
index 2952b73c8..c6b6e82fa 100755
--- a/spec/unit/parser/lexer.rb
+++ b/spec/unit/parser/lexer.rb
@@ -4,6 +4,20 @@ require File.dirname(__FILE__) + '/../../spec_helper'
 
 require 'puppet/parser/lexer'
 
+# This is a special matcher to match easily lexer output
+Spec::Matchers.create :be_like do |ary|
+    match do |result|
+        r = true
+        result.zip(ary) do |a,b|
+            unless a[0] == b[0] and ((a[1].is_a?(Hash) and a[1][:value] == b[1]) or a[1] == b[1])
+                r = false
+                break
+            end
+        end
+        r
+    end
+end
+
 describe Puppet::Parser::Lexer do
     describe "when reading strings" do
         before { @lexer = Puppet::Parser::Lexer.new }
@@ -474,7 +488,7 @@ describe Puppet::Parser::Lexer, "when lexing comments" do
 
     it "should skip whitespace before lexing the next token after a non-token" do
         @lexer.string = "/* 1\n\n */ \ntest"
-        @lexer.fullscan.include?([:NAME, "test"]).should be_true
+        @lexer.fullscan.should be_like([[:NAME, "test"],[false,false]])
     end
 end
 
@@ -491,7 +505,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
         }
         strings.each { |str,ary|
             @lexer.string = str
-            @lexer.fullscan().should == ary
+            @lexer.fullscan().should be_like(ary)
         }
     end
 
@@ -515,7 +529,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
 }
         strings.each { |str,array|
             @lexer.string = str
-            @lexer.fullscan().should == array
+            @lexer.fullscan().should be_like(array)
         }
     end
 
@@ -535,7 +549,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
 
     it "should correctly identify keywords" do
         @lexer.string = "case"
-        @lexer.fullscan.should == [[:CASE, "case"], [false, false]]
+        @lexer.fullscan.should be_like([[:CASE, "case"], [false, false]])
     end
 
     it "should correctly match strings" do
@@ -545,11 +559,11 @@ describe "Puppet::Parser::Lexer in the old tests" do
 
         names.each { |t|
             @lexer.string = t
-            @lexer.fullscan.should == [[:NAME,t],[false,false]]
+            @lexer.fullscan.should be_like([[:NAME,t],[false,false]])
         }
         types.each { |t|
             @lexer.string = t
-            @lexer.fullscan.should == [[:CLASSREF,t],[false,false]]
+            @lexer.fullscan.should be_like([[:CLASSREF,t],[false,false]])
         }
     end
 
@@ -558,7 +572,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
 
        string.each { |t|
             @lexer.string = t
-            @lexer.fullscan.should == [[:NAME,t],[false,false]]
+            @lexer.fullscan.should be_like([[:NAME,t],[false,false]])
        }
     end
 
@@ -575,7 +589,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
 
         @lexer.string = string
 
-        @lexer.fullscan.should == [[:AT, "@"], [:NAME, "type"], [:LBRACE, "{"], [false,false]]
+        @lexer.fullscan.should be_like([[:AT, "@"], [:NAME, "type"], [:LBRACE, "{"], [false,false]])
     end
 
     it "should correctly deal with namespaces" do
@@ -618,7 +632,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
 
             @lexer.scan do |t, s|
                 t.should == :VARIABLE
-                string.sub(/^\$/, '').should == s
+                string.sub(/^\$/, '').should == s[:value]
                 break
             end
         end
@@ -630,7 +644,7 @@ describe "Puppet::Parser::Lexer in the old tests" do
 
         string.each do |foo|
             @lexer.string = foo
-            @lexer.fullscan[0].should == [:CLASSREF, foo]
+            @lexer.fullscan.should be_like([[:CLASSREF, foo],[false,false]])
         end
     end