diff options
Diffstat (limited to 'utils/training/gen_ngram.cpp')
-rw-r--r-- | utils/training/gen_ngram.cpp | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp index 9e79f1d..2b0e1dd 100644 --- a/utils/training/gen_ngram.cpp +++ b/utils/training/gen_ngram.cpp @@ -91,13 +91,14 @@ int main(int argc, char * argv[]){ glong phrase_len = 0; utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); - if ( phrase_len == 0 ) - continue; - phrase_token_t token = 0; - int result = g_phrases->search( phrase_len, phrase, token); - if ( ! (result & SEARCH_OK) ) - token = 0; + if ( 0 != phrase_len ) { + int result = g_phrases->search( phrase_len, phrase, token); + if ( ! (result & SEARCH_OK) ) + token = 0; + g_free(phrase); + phrase = NULL; + } last_token = cur_token; cur_token = token; |