merges r25402 from trunk into ruby_1_9_1. fixes the backport task #1921.

--
* parse.y (parser_here_document): dispatch delayed heredoc
  contents.  based on a patch from Andy Keep in [ruby-core:24855].

git-svn-id: http://svn.ruby-lang.org/repos/ruby/branches/ruby_1_9_1@26008 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

6 files changed, 75 insertions, 27 deletions

diff --git a/ChangeLog b/ChangeLog
index e9837eea6..bcf2417db 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+Tue Oct 20 14:50:51 2009  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* parse.y (parser_here_document): dispatch delayed heredoc
+	  contents.  based on a patch from Andy Keep in [ruby-core:24855].
+
 Fri Jan 30 18:04:23 2009  NAKAMURA Usaku  <usa@ruby-lang.org>
 
 	* win32/win32.c (rb_w32_write): limit write size to 32KB if the file
diff --git a/parse.y b/parse.y
index b27875a3d..2fbcf565d 100644
--- a/parse.y
+++ b/parse.y
@@ -4785,21 +4785,42 @@ ripper_yylval_id(ID x)
 # define yylval_id() yylval.id
 #endif
 
-#ifdef RIPPER
+#ifndef RIPPER
+#define ripper_flush(p) (void)(p)
+#else
 #define ripper_flush(p) (p->tokp = p->parser_lex_p)
 
 #define yylval_rval *(TYPE(yylval.val) == T_NODE ? &yylval.node->nd_rval : &yylval.val)
 
-static void
-ripper_dispatch_scan_event(struct parser_params *parser, int t)
+static int
+ripper_has_scan_event(struct parser_params *parser)
 {
-    VALUE str;
 
     if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp");
-    if (lex_p == parser->tokp) return;
-    str = STR_NEW(parser->tokp, lex_p - parser->tokp);
-    yylval_rval = ripper_dispatch1(parser, ripper_token2eventid(t), str);
+    return lex_p > parser->tokp;
+}
+
+static VALUE
+ripper_scan_event_val(struct parser_params *parser, int t)
+{
+    VALUE str = STR_NEW(parser->tokp, lex_p - parser->tokp);
+    VALUE rval = ripper_dispatch1(parser, ripper_token2eventid(t), str);
     ripper_flush(parser);
+    return rval;
+}
+
+static void
+ripper_dispatch_scan_event(struct parser_params *parser, int t)
+{
+    if (!ripper_has_scan_event(parser)) return;
+    yylval_rval = ripper_scan_event_val(parser, t);
+}
+
+static void
+ripper_dispatch_ignored_scan_event(struct parser_params *parser, int t)
+{
+    if (!ripper_has_scan_event(parser)) return;
+    (void)ripper_scan_event_val(parser, t);
 }
 
 static void
@@ -5289,9 +5310,7 @@ parser_nextc(struct parser_params *parser)
 	    parser->line_count++;
 	    lex_pbeg = lex_p = RSTRING_PTR(v);
 	    lex_pend = lex_p + RSTRING_LEN(v);
-#ifdef RIPPER
 	    ripper_flush(parser);
-#endif
 	    lex_lastline = v;
 	}
     }
@@ -5946,9 +5965,7 @@ parser_heredoc_identifier(struct parser_params *parser)
 				  len,				/* nd_nth */
 				  lex_lastline);		/* nd_orig */
     nd_set_line(lex_strterm, ruby_sourceline);
-#ifdef RIPPER
     ripper_flush(parser);
-#endif
     return term == '`' ? tXSTRING_BEG : tSTRING_BEG;
 }
 
@@ -5957,12 +5974,6 @@ parser_heredoc_restore(struct parser_params *parser, NODE *here)
 {
     VALUE line;
 
-#ifdef RIPPER
-    if (!NIL_P(parser->delayed))
-	ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
-    lex_goto_eol(parser);
-    ripper_dispatch_scan_event(parser, tHEREDOC_END);
-#endif
     line = here->nd_orig;
     lex_lastline = line;
     lex_pbeg = RSTRING_PTR(line);
@@ -5972,9 +5983,7 @@ parser_heredoc_restore(struct parser_params *parser, NODE *here)
     ruby_sourceline = nd_line(here);
     dispose_string(here->nd_lit);
     rb_gc_force_recycle((VALUE)here);
-#ifdef RIPPER
     ripper_flush(parser);
-#endif
 }
 
 static int
@@ -6000,6 +6009,7 @@ parser_here_document(struct parser_params *parser, NODE *here)
     const char *eos, *p, *pend;
     long len;
     VALUE str = 0;
+    rb_encoding *enc = parser->enc;
 
     eos = RSTRING_PTR(here->nd_lit);
     len = RSTRING_LEN(here->nd_lit) - 1;
@@ -6008,6 +6018,20 @@ parser_here_document(struct parser_params *parser, NODE *here)
     if ((c = nextc()) == -1) {
       error:
 	compile_error(PARSER_ARG "can't find string \"%s\" anywhere before EOF", eos);
+#ifdef RIPPER
+	if (NIL_P(parser->delayed)) {
+	    ripper_dispatch_scan_event(parser, tSTRING_CONTENT);
+	}
+	else {
+	    if (str ||
+		((len = lex_p - parser->tokp) > 0 &&
+		 (str = STR_NEW3(parser->tokp, len, enc, func), 1))) {
+		rb_str_append(parser->delayed, str);
+	    }
+	    ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
+	}
+	lex_goto_eol(parser);
+#endif
       restore:
 	heredoc_restore(lex_strterm);
 	lex_strterm = 0;
@@ -6047,7 +6071,6 @@ parser_here_document(struct parser_params *parser, NODE *here)
     }
     else {
 	/*	int mb = ENC_CODERANGE_7BIT, *mbp = &mb;*/
-	rb_encoding *enc = parser->enc;
 	newtok();
 	if (c == '#') {
 	    switch (c = nextc()) {
@@ -6076,6 +6099,12 @@ parser_here_document(struct parser_params *parser, NODE *here)
 	} while (!whole_match_p(eos, len, indent));
 	str = STR_NEW3(tok(), toklen(), enc, func);
     }
+#ifdef RIPPER
+    if (!NIL_P(parser->delayed))
+	ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
+    lex_goto_eol(parser);
+    ripper_dispatch_ignored_scan_event(parser, tHEREDOC_END);
+#endif
     heredoc_restore(lex_strterm);
     lex_strterm = NEW_STRTERM(-1, 0, 0);
     set_yylval_str(str);
diff --git a/test/ripper/dummyparser.rb b/test/ripper/dummyparser.rb
index 2c107b37c..b19a49281 100644
--- a/test/ripper/dummyparser.rb
+++ b/test/ripper/dummyparser.rb
@@ -58,7 +58,7 @@ class DummyParser < Ripper
     class << self; self; end.class_eval do
       define_method(name) do |*a, &b|
         result = super(*a, &b)
-        yield
+        yield(*a)
         result
       end
     end
diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb
index 80199ecce..2494434d8 100644
--- a/test/ripper/test_parser_events.rb
+++ b/test/ripper/test_parser_events.rb
@@ -202,6 +202,20 @@ class TestRipper_ParserEvents < Test::Unit::TestCase
     assert_equal true, thru_bodystmt
   end
 
+  def test_heredoc
+    bug1921 = '[ruby-core:24855]'
+    thru_heredoc_beg = false
+    tree = parse("<<EOS\nheredoc\nEOS\n", :on_heredoc_beg) {thru_heredoc_beg = true}
+    assert_equal true, thru_heredoc_beg
+    assert_match(/string_content\(\),heredoc\n/, tree, bug1921)
+    heredoc = nil
+    parse("<<EOS\nheredoc1\nheredoc2\nEOS\n", :on_string_add) {|n, s| heredoc = s}
+    assert_equal("heredoc1\nheredoc2\n", heredoc, bug1921)
+    heredoc = nil
+    parse("<<-EOS\nheredoc1\nheredoc2\n\tEOS\n", :on_string_add) {|n, s| heredoc = s}
+    assert_equal("heredoc1\nheredoc2\n", heredoc, bug1921)
+  end
+
 =begin
   def test_brace_block
     assert_equal true, $thru__brace_block
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
index 659916570..10f185e79 100644
--- a/test/ripper/test_scanner_events.rb
+++ b/test/ripper/test_scanner_events.rb
@@ -628,9 +628,9 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                  scan('tstring_content', "<<EOS\nheredoc\nEOS")
     assert_equal ["heredoc\n"],
                  scan('tstring_content', "<<EOS\nheredoc\nEOS\n")
-    assert_equal ["heredoc \n"],
-                 scan('tstring_content', "<<EOS\nheredoc \nEOS \n")
-    assert_equal ["heredoc\n"],
+    assert_equal ["here\ndoc \nEOS \n"],
+                 scan('tstring_content', "<<EOS\nhere\ndoc \nEOS \n")
+    assert_equal ["heredoc\n\tEOS \n"],
                  scan('tstring_content', "<<-EOS\nheredoc\n\tEOS \n")
   end
 
@@ -641,9 +641,9 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                  scan('heredoc_end', "<<EOS\nheredoc\nEOS")
     assert_equal ["EOS\n"],
                  scan('heredoc_end', "<<EOS\nheredoc\nEOS\n")
-    assert_equal ["EOS \n"],
+    assert_equal [],
                  scan('heredoc_end', "<<EOS\nheredoc\nEOS \n")
-    assert_equal ["\tEOS \n"],
+    assert_equal [],
                  scan('heredoc_end', "<<-EOS\nheredoc\n\tEOS \n")
   end
 
diff --git a/version.h b/version.h
index 733f5ef18..350992a0c 100644
--- a/version.h
+++ b/version.h
@@ -1,5 +1,5 @@
 #define RUBY_VERSION "1.9.1"
-#define RUBY_PATCHLEVEL 361
+#define RUBY_PATCHLEVEL 362
 #define RUBY_VERSION_MAJOR 1
 #define RUBY_VERSION_MINOR 9
 #define RUBY_VERSION_TEENY 1