From b3f0ef089710b27e2b22921f732190f0e05e11b2 Mon Sep 17 00:00:00 2001 From: yugui Date: Sat, 5 Dec 2009 09:40:25 +0000 Subject: merges r25883 from trunk into ruby_1_9_1. fixes a backpor task #2168. -- * enc/trans/newline.trans (fun_so_universal_newline): generate \n after \r\n detection instead of just after \r. [ruby-list:45988] [ruby-core:25881] [ruby-core:26788] git-svn-id: http://svn.ruby-lang.org/repos/ruby/branches/ruby_1_9_1@26015 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 6 ++++++ enc/trans/newline.trans | 26 +++++++++++++++----------- test/ruby/test_econv.rb | 18 ++++++++++++++++-- test/ruby/test_io_m17n.rb | 2 -- version.h | 2 +- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/ChangeLog b/ChangeLog index 10d14deb5..16c470434 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Mon Nov 23 04:12:00 2009 Tanaka Akira + + * enc/trans/newline.trans (fun_so_universal_newline): generate \n + after \r\n detection instead of just after \r. + [ruby-list:45988] [ruby-core:25881] [ruby-core:26788] + Tue Dec 1 12:01:13 2009 Nobuyoshi Nakada * parse.y (dyna_push_gen, dyna_pop_gen): no needs to save both of diff --git a/enc/trans/newline.trans b/enc/trans/newline.trans index d62034aa4..a200ec00a 100644 --- a/enc/trans/newline.trans +++ b/enc/trans/newline.trans @@ -48,26 +48,26 @@ fun_so_universal_newline(void *statep, const unsigned char *s, size_t l, unsigne if (s[0] == '\n') { if (STATE == NORMAL) { NEWLINES_MET |= MET_LF; - o[0] = '\n'; - len = 1; } else { /* JUST_AFTER_CR */ NEWLINES_MET |= MET_CRLF; - len = 0; } + o[0] = '\n'; + len = 1; STATE = NORMAL; } else { - if (STATE == JUST_AFTER_CR) - NEWLINES_MET |= MET_CR; - if (s[0] == '\r') { + len = 0; + if (STATE == JUST_AFTER_CR) { o[0] = '\n'; len = 1; + NEWLINES_MET |= MET_CR; + } + if (s[0] == '\r') { STATE = JUST_AFTER_CR; } else { - o[0] = s[0]; - len = 1; + o[len++] = s[0]; STATE = NORMAL; } } @@ -79,10 +79,14 @@ static ssize_t universal_newline_finish(void *statep, unsigned char *o, size_t osize) { unsigned char *sp = statep; - if (STATE == JUST_AFTER_CR) + int len = 0; + if (STATE == JUST_AFTER_CR) { + o[0] = '\n'; + len = 1; NEWLINES_MET |= MET_CR; + } STATE = NORMAL; - return 0; + return len; } static const rb_transcoder @@ -91,7 +95,7 @@ rb_universal_newline = { TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 1, /* max_input */ - 1, /* max_output */ + 2, /* max_output */ asciicompat_converter, /* asciicompat_type */ 2, universal_newline_init, universal_newline_init, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_universal_newline, diff --git a/test/ruby/test_econv.rb b/test/ruby/test_econv.rb index 0f1d18a8e..69453b9be 100644 --- a/test/ruby/test_econv.rb +++ b/test/ruby/test_econv.rb @@ -399,7 +399,7 @@ class TestEncodingConverter < Test::Unit::TestCase src << "abc\r\ndef"; check_ec("abc\ndef", "", :source_buffer_empty, *a) src << "ghi\njkl"; check_ec("abc\ndefghi\njkl", "", :source_buffer_empty, *a) src << "mno\rpqr"; check_ec("abc\ndefghi\njklmno\npqr", "", :source_buffer_empty, *a) - src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu\n", "", :source_buffer_empty, *a) + src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu", "", :source_buffer_empty, *a) src << "\nvwx"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx", "", :source_buffer_empty, *a) src << "\nyz"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) end @@ -410,11 +410,25 @@ class TestEncodingConverter < Test::Unit::TestCase src << "abc\r\ndef"; check_ec("abc\ndef", "", :source_buffer_empty, *a) src << "ghi\njkl"; check_ec("abc\ndefghi\njkl", "", :source_buffer_empty, *a) src << "mno\rpqr"; check_ec("abc\ndefghi\njklmno\npqr", "", :source_buffer_empty, *a) - src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu\n", "", :source_buffer_empty, *a) + src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu", "", :source_buffer_empty, *a) src << "\nvwx"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx", "", :source_buffer_empty, *a) src << "\nyz"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) end + def test_universal_newline3 + ec = Encoding::Converter.new("", "", universal_newline: true) + a = ["", src="", ec, nil, 50, :partial_input=>true] + src << "abc\r\ndef"; check_ec("abc\ndef", "", :source_buffer_empty, *a) + src << "ghi\njkl"; check_ec("abc\ndefghi\njkl", "", :source_buffer_empty, *a) + src << "mno\rpqr"; check_ec("abc\ndefghi\njklmno\npqr", "", :source_buffer_empty, *a) + src << "stu\r"; check_ec("abc\ndefghi\njklmno\npqrstu", "", :source_buffer_empty, *a) + src << "\nvwx"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx", "", :source_buffer_empty, *a) + src << "\nyz"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) + src << "\r"; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a) + a[-1] = nil + src << ""; check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz\n", "", :finished, *a) + end + def test_crlf_newline ec = Encoding::Converter.new("UTF-8", "EUC-JP", crlf_newline: true) assert_econv("abc\r\ndef", :finished, 50, ec, "abc\ndef", "") diff --git a/test/ruby/test_io_m17n.rb b/test/ruby/test_io_m17n.rb index 71299181e..ac0645344 100644 --- a/test/ruby/test_io_m17n.rb +++ b/test/ruby/test_io_m17n.rb @@ -1506,7 +1506,6 @@ EOT assert_equal("a", f.getc) assert_equal("\n", f.getc) f.binmode - assert_equal("\n", f.getc) assert_equal("b", f.getc) assert_equal("\r", f.getc) assert_equal("\n", f.getc) @@ -1526,7 +1525,6 @@ EOT assert_equal("a", f.getc) assert_equal("\n", f.getc) f.binmode - assert_equal("\n", f.getc) assert_equal("b", f.getc) assert_equal("\r", f.getc) assert_equal("\n", f.getc) diff --git a/version.h b/version.h index c98fb8387..163282d7b 100644 --- a/version.h +++ b/version.h @@ -1,5 +1,5 @@ #define RUBY_VERSION "1.9.1" -#define RUBY_PATCHLEVEL 368 +#define RUBY_PATCHLEVEL 369 #define RUBY_VERSION_MAJOR 1 #define RUBY_VERSION_MINOR 9 #define RUBY_VERSION_TEENY 1 -- cgit