* ext/ripper/lib/ripper/tokenizer.rb: new file.

git-svn-id: http://svn.ruby-lang.org/repos/ruby/trunk@6934 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 39 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 086f48f09..b6ccfb21e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+Mon Sep 20 15:17:47 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* ext/ripper/lib/ripper/tokenizer.rb: new file.
+
 Mon Sep 20 15:13:52 2004  Minero Aoki  <aamine@loveruby.net>
 
 	* test/ripper/test_scanner_events.rb: test #lineno and #column.
diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
new file mode 100644
index 000000000..0bd966ea5
--- /dev/null
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -0,0 +1,35 @@
+#
+# ripper/tokenizer.rb
+#
+# Copyright (C) 2004 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper'
+
+class Ripper
+
+  def Ripper.tokenize(str)
+    Tokenizer.tokenize(str)
+  end
+
+  class Tokenizer < ::Ripper
+    def Tokenizer.tokenize(str)
+      new(str).tokenize
+    end
+
+    def tokenize
+      @tokens = []
+      parse
+      @tokens
+    end
+
+    def on__scan(type, tok)
+      @tokens.push tok
+    end
+  end
+
+end