* include/ruby/intern.h (rb_uv_to_utf8): declared.

* re.c (rb_reg_preprocess): new function for dynamic regexp with
  \u{} such as Regexp.new("\\u{6666}").
  (rb_reg_prepare_re): preprocess regexp for recompiling.
  (read_escaped_byte): new function.
  (unescape_escaped_nonascii): new function.
  (append_utf8): new function.
  (unescape_unicode_list): new function.
  (unescape_unicode_bmp): new function.
  (unescape_nonascii): new function.
  (rb_reg_initialize): preprocess regexp.

* pack.c (rb_uv_to_utf8): renamed from uv_to_utf8.

* parse.y (STR_NEW3): take func instead of has8 and hasmb.
  (parser_str_new): use default coderange mechanism except for regexp.
  (parser_tokadd_utf8): copy regexp source as-is.
  (parser_read_escape): UTF-8 stuff removed.
  (parser_tokadd_escape): has8bit and hasmb removed.
  (parser_tokadd_string): fix 8-bit single byte character with \u.
  (parser_parse_string): has8bit and hasmb removed.
  (parser_here_document): has8bit and hasmb removed.
  (parser_yylex): call parser_tokadd_utf8 instead of read_escape for
  UTF-8 character.


git-svn-id: http://svn.ruby-lang.org/repos/ruby/trunk@14072 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 20 insertions, 7 deletions

diff --git a/test/ruby/test_m17n.rb b/test/ruby/test_m17n.rb
index fd183967a..bb0a8a501 100644
--- a/test/ruby/test_m17n.rb
+++ b/test/ruby/test_m17n.rb
@@ -25,6 +25,17 @@ class TestM17N < Test::Unit::TestCase
     assert_encoding("EUC-JP", eval(e(%{"\\x80"})).encoding)
   end
 
+  def test_string_mixed_unicode
+    assert_raise(SyntaxError) { eval(a(%{"\xc0\xa0\\u{6666}"})) }
+    assert_raise(SyntaxError) { eval(e(%{"\xc0\xa0\\u{6666}"})) }
+    assert_raise(SyntaxError) { eval(s(%{"\xc0\xa0\\u{6666}"})) }
+    assert_nothing_raised { eval(u(%{"\xc0\xa0\\u{6666}"})) }
+    assert_raise(SyntaxError) { eval(a(%{"\\u{6666}\xc0\xa0"})) }
+    assert_raise(SyntaxError) { eval(e(%{"\\u{6666}\xc0\xa0"})) }
+    assert_raise(SyntaxError) { eval(s(%{"\\u{6666}\xc0\xa0"})) }
+    assert_nothing_raised { eval(u(%{"\\u{6666}\xc0\xa0"})) }
+  end
+
   def test_regexp_too_short_multibyte_character
     assert_raise(SyntaxError) { eval('/\xfe/e') }
     assert_raise(SyntaxError) { eval('/\x8e/e') }
@@ -38,11 +49,12 @@ class TestM17N < Test::Unit::TestCase
     assert_raise(SyntaxError) { eval('/\xfc\x80\x80\x80\x80/u') }
 
     # raw 8bit
-    #assert_raise(SyntaxError) { eval("/\xfe/e") }
-    #assert_raise(SyntaxError) { eval("/\xc0/u") }
+    assert_raise(SyntaxError) { eval("/\xfe/e") }
+    assert_raise(SyntaxError) { eval("/\xc0/u") }
 
     # invalid suffix
-    #assert_raise(SyntaxError) { eval('/\xc0\xff/u') }
+    assert_raise(SyntaxError) { eval('/\xc0\xff/u') }
+    assert_raise(SyntaxError) { eval('/\xc0 /u') }
     #assert_raise(SyntaxError) { eval('/\xc0\x20/u') }
   end
 
@@ -94,6 +106,9 @@ class TestM17N < Test::Unit::TestCase
   def test_regexp_generic
     assert_regexp_generic_ascii(/a/)
     assert_regexp_generic_ascii(Regexp.new(a("a")))
+    assert_regexp_generic_ascii(Regexp.new(e("a")))
+    assert_regexp_generic_ascii(Regexp.new(s("a")))
+    assert_regexp_generic_ascii(Regexp.new(u("a")))
 
     [/a/, Regexp.new(a("a"))].each {|r|
       assert_equal(0, r =~ a("a"))
@@ -112,7 +127,7 @@ class TestM17N < Test::Unit::TestCase
     assert_regexp_fixed_ascii8bit(/\xc0\xa1/n)
     assert_regexp_fixed_ascii8bit(eval(a(%{/\xc0\xa1/})))
     assert_regexp_fixed_ascii8bit(eval(a(%{/\xc0\xa1/n})))
-    # assert_regexp_fixed_ascii8bit(eval(a(%q{/\xc0\xa1/})))
+    assert_regexp_fixed_ascii8bit(eval(a(%q{/\xc0\xa1/})))
 
     [/a/n].each {|r|
       assert_equal(0, r =~ a("a"))
@@ -139,12 +154,11 @@ class TestM17N < Test::Unit::TestCase
 
   def test_regexp_euc
     assert_regexp_fixed_eucjp(/a/e)
-    assert_regexp_fixed_eucjp(Regexp.new(e("a")))
     assert_regexp_fixed_eucjp(/\xc0\xa1/e)
     assert_regexp_fixed_eucjp(eval(e(%{/\xc0\xa1/})))
     assert_regexp_fixed_eucjp(eval(e(%q{/\xc0\xa1/})))
 
-    [/a/e, Regexp.new(e("a"))].each {|r|
+    [/a/e].each {|r|
       assert_equal(0, r =~ a("a"))
       assert_equal(0, r =~ e("a"))
       assert_equal(0, r =~ s("a"))
@@ -169,7 +183,6 @@ class TestM17N < Test::Unit::TestCase
 
   def test_regexp_sjis
     assert_regexp_fixed_sjis(/a/s)
-    assert_regexp_fixed_sjis(Regexp.new(s("a")))
     assert_regexp_fixed_sjis(/\xc0\xa1/s)
     assert_regexp_fixed_sjis(eval(s(%{/\xc0\xa1/})))
     assert_regexp_fixed_sjis(eval(s(%q{/\xc0\xa1/})))