From c7aeb1c57cea4f458b3084a34397e44110297ac8 Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Thu, 9 Jun 2011 18:04:45 +0800 Subject: fixes read document --- utils/training/gen_deleted_ngram.cpp | 13 +++++++------ utils/training/gen_k_mixture_model.cpp | 16 +++++++--------- utils/training/gen_ngram.cpp | 13 +++++++------ 3 files changed, 21 insertions(+), 21 deletions(-) (limited to 'utils/training') diff --git a/utils/training/gen_deleted_ngram.cpp b/utils/training/gen_deleted_ngram.cpp index 93986d6..b39490f 100644 --- a/utils/training/gen_deleted_ngram.cpp +++ b/utils/training/gen_deleted_ngram.cpp @@ -78,13 +78,14 @@ int main(int argc, char * argv[]){ glong phrase_len = 0; utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); - if ( phrase_len == 0 ) - continue; - phrase_token_t token = 0; - int result = g_phrases->search( phrase_len, phrase, token); - if ( ! (result & SEARCH_OK) ) - token = 0; + if ( 0 != phrase_len ) { + int result = g_phrases->search( phrase_len, phrase, token); + if ( ! (result & SEARCH_OK) ) + token = 0; + g_free(phrase); + phrase = NULL; + } last_token = cur_token; cur_token = token; diff --git a/utils/training/gen_k_mixture_model.cpp b/utils/training/gen_k_mixture_model.cpp index 13ff04d..bd350b4 100644 --- a/utils/training/gen_k_mixture_model.cpp +++ b/utils/training/gen_k_mixture_model.cpp @@ -60,16 +60,14 @@ bool read_document(PhraseLargeTable * phrases, FILE * document, glong phrase_len = 0; utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); - if ( phrase_len == 0 ) - continue; - phrase_token_t token = 0; - int search_result = phrases->search( phrase_len, phrase, token ); - if ( ! (search_result & SEARCH_OK) ) - token = 0; - - g_free(phrase); - phrase = NULL; + if ( 0 != phrase_len ) { + int search_result = phrases->search( phrase_len, phrase, token ); + if ( ! (search_result & SEARCH_OK) ) + token = 0; + g_free(phrase); + phrase = NULL; + } last_token = cur_token; cur_token = token; diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp index 9e79f1d..2b0e1dd 100644 --- a/utils/training/gen_ngram.cpp +++ b/utils/training/gen_ngram.cpp @@ -91,13 +91,14 @@ int main(int argc, char * argv[]){ glong phrase_len = 0; utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); - if ( phrase_len == 0 ) - continue; - phrase_token_t token = 0; - int result = g_phrases->search( phrase_len, phrase, token); - if ( ! (result & SEARCH_OK) ) - token = 0; + if ( 0 != phrase_len ) { + int result = g_phrases->search( phrase_len, phrase, token); + if ( ! (result & SEARCH_OK) ) + token = 0; + g_free(phrase); + phrase = NULL; + } last_token = cur_token; cur_token = token; -- cgit