From c734976bf2e3a54cb2d894f6c0765201fb04069c Mon Sep 17 00:00:00 2001
From: nobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Wed, 24 Oct 2007 08:09:40 +0000
Subject: * parse.y (parser_tokspace): make space in token buffer.

* parse.y (parser_yylex): fix encoding of single character literal.


git-svn-id: http://svn.ruby-lang.org/repos/ruby/trunk@13766 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 parse.y | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

(limited to 'parse.y')

diff --git a/parse.y b/parse.y
index 8a46f2d71..3182ceee0 100644
--- a/parse.y
+++ b/parse.y
@@ -4494,6 +4494,7 @@ static int parser_here_document(struct parser_params*,NODE*);
 # define nextc()                   parser_nextc(parser)
 # define pushback(c)               parser_pushback(parser, c)
 # define newtok()                  parser_newtok(parser)
+# define tokspace(n)               parser_tokspace(parser, n)
 # define tokadd(c)                 parser_tokadd(parser, c)
 # define read_escape(m)            parser_read_escape(parser, m)
 # define tokadd_escape(t,m)        parser_tokadd_escape(parser, t, m)
@@ -4913,6 +4914,18 @@ parser_newtok(struct parser_params *parser)
     return tokenbuf;
 }
 
+static char *
+parser_tokspace(struct parser_params *parser, int n)
+{
+    int idx = tokidx + n;
+
+    if (idx >= toksiz) {
+	do {toksiz *= 2;} while (toksiz < idx);
+	REALLOC_N(tokenbuf, char, toksiz);
+    }
+    return &tokenbuf[tokidx];
+}
+
 static void
 parser_tokadd(struct parser_params *parser, int c)
 {
@@ -6086,21 +6099,24 @@ parser_yylex(struct parser_params *parser)
 	}
 	newtok();
 	if (parser_ismbchar()) {
+	    mb = ENC_CODERANGE_MULTI;
 	    tokadd_mbchar(c);
 	}
 	else if ((rb_enc_isalnum(c, parser->enc) || c == '_') &&
 		 lex_p < lex_pend && is_identchar(lex_p, lex_pend, parser->enc)) {
 	    goto ternary;
 	}
-	else if (c == '\\') {
-	    c = read_escape(0);
-	    tokadd(c);
+	else if (c == '\\' && (c = read_escape(0)) >= 0x80) {
+	    rb_encoding *enc = parser->enc;
+	    mb = ENC_CODERANGE_UNKNOWN;
+	    rb_enc_mbcput(c, tokspace(rb_enc_codelen(c, enc)), enc);
 	}
 	else {
+	    mb = ENC_CODERANGE_SINGLE;
 	    tokadd(c);
 	}
 	tokfix();
-	set_yylval_str(STR_NEW(tok(), toklen()));
+	set_yylval_str(STR_NEW3(tok(), toklen(), mb));
 	lex_state = EXPR_ENDARG;
 	return tCHAR;
 
@@ -6865,9 +6881,9 @@ parser_yylex(struct parser_params *parser)
 	break;
     }
 
-    mb = 0;
+    mb = ENC_CODERANGE_SINGLE;
     do {
-	if (!ISASCII(c)) mb = 1;
+	if (!ISASCII(c)) mb = ENC_CODERANGE_UNKNOWN;
 	tokadd_mbchar(c);
 	c = nextc();
     } while (parser_is_identchar());
@@ -6920,7 +6936,7 @@ parser_yylex(struct parser_params *parser)
 		}
 	    }
 
-	    if (!mb && lex_state != EXPR_DOT) {
+	    if (mb == ENC_CODERANGE_SINGLE && lex_state != EXPR_DOT) {
 		const struct kwtable *kw;
 
 		/* See if it is a reserved word.  */
-- 
cgit