From 1bd05a07d29dbe3f8b03ae73b879797aea8c653a Mon Sep 17 00:00:00 2001
From: Peng Wu <alexepico@gmail.com>
Date: Tue, 19 Apr 2011 10:12:12 +0800
Subject: fixes phrase large table

---
 utils/training/gen_ngram.cpp | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'utils')

diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp
index 3203cd1..918c9c8 100644
--- a/utils/training/gen_ngram.cpp
+++ b/utils/training/gen_ngram.cpp
@@ -100,6 +100,9 @@ int main(int argc, char * argv[]){
         glong phrase_len = 0;
         utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL);
 
+        if ( phrase_len == 0 )
+            continue;
+
 	phrase_token_t token = 0;
         int result = g_phrases->search( phrase_len, phrase, token);
 	if ( ! (result & SEARCH_OK) )
-- 
cgit