merges r21509 and r21512 from trunk into ruby_1_9_1.

* enc/trans/gb18030.trans, gb18030-tbl.rb:
  new Chinese GB18030 transcoding (from Yoshihiro Kambayashi)

* test/ruby/test_transcode.rb: added tests for the above
  (from Yoshihiro Kambayashi)

* transcode_data.h, transcode.c, tool/transcode_tblgen.rb:
  added support for GB18030-specific 4-byte sequences
  (with Yoshihiro Kambayashi)

git-svn-id: http://svn.ruby-lang.org/repos/ruby/branches/ruby_1_9_1@21560 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 84 insertions, 0 deletions

diff --git a/enc/trans/gb18030.trans b/enc/trans/gb18030.trans
new file mode 100644
index 000000000..700522c9f
--- /dev/null
+++ b/enc/trans/gb18030.trans
@@ -0,0 +1,84 @@
+#include "transcode_data.h"
+
+<%
+  require "gb18030-tbl"
+
+  transcode_tbl_only "GB18030", "UTF-8", [["{00-7f}", :nomap]] +
+                                         GB18030_TO_UCS_TBL + [
+                                        ["{90-e2}{30-39}{81-fe}{30-39}", :func_so],
+                                        ["e3{30-31}{81-fe}{30-39}", :func_so],
+                                        ["e332{81-99}{30-39}", :func_so],
+                                        ["e3329a{30-35}", :func_so],   #  "E3329A35" is U+10FFFF
+                                       ]
+  transcode_tbl_only "UTF-8", "GB18030", [["{00-7f}", :nomap]] +
+                                         GB18030_TO_UCS_TBL.map {|a,b| [b,a] } + [
+                                        ["f0{90-bf}{80-bf}{80-bf}", :func_so],
+                                        ["{f1-f3}{80-bf}{80-bf}{80-bf}", :func_so],
+                                        ["f4{80-8f}{80-bf}{80-bf}", :func_so]
+                                       ]
+%>
+
+<%= transcode_generated_code %>
+
+static ssize_t
+fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize)
+{
+    /* outside BMP only */
+    /* u: Unicode Scalar Value */
+    unsigned int u = (s[0]-0x90)*10*126*10 + (s[1]-0x30)*126*10 + (s[2]-0x81)*10 + (s[3]-0x30) + 0x10000;
+    o[0] = 0xF0 | (u>>18);
+    o[1] = 0x80 | ((u>>12)&0x3F);
+    o[2] = 0x80 | ((u>>6)&0x3F);
+    o[3] = 0x80 | (u&0x3F);
+    return 4;
+}
+
+static ssize_t
+fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize)
+{
+    /* outside BMP only */
+    /* u: Unicode Scalar Value */
+    unsigned int u = ((s[0]&0x07)<<18) | ((s[1]&0x3F)<<12) | ((s[2]&0x3F)<<6) | (s[3]&0x3F);
+    u -= 0x10000;
+    o[3] = 0x30 + u%10;
+    u /= 10;
+    o[2] = 0x81 + u%126;
+    u /= 126;
+    o[1] = 0x30 + u%10;
+    o[0] = 0x90 + u/10;
+    return 4;
+}
+
+
+static const rb_transcoder
+rb_from_GB18030 = {
+    "GB18030", "UTF-8", from_GB18030,
+    TRANSCODE_TABLE_INFO,
+    1, /* input_unit_length */
+    4, /* max_input */
+    3, /* max_output */
+    asciicompat_converter, /* asciicompat_type */
+    0, NULL, NULL, /* state_size, state_init, state_fini */
+    NULL, NULL, NULL, fun_so_from_gb18030,
+    NULL, NULL, NULL
+};
+static const rb_transcoder
+rb_to_GB18030 = {
+    "UTF-8", "GB18030", to_GB18030,
+    TRANSCODE_TABLE_INFO,
+    1, /* input_unit_length */
+    4, /* max_input */
+    4, /* max_output */
+    asciicompat_converter, /* asciicompat_type */
+    0, NULL, NULL, /* state_size, state_init, state_fini */
+    NULL, NULL, NULL, fun_so_to_gb18030,
+    NULL, NULL, NULL
+};
+
+
+void
+Init_gb18030(void)
+{
+    rb_register_transcoder(&rb_from_GB18030);
+    rb_register_transcoder(&rb_to_GB18030);
+}