From f917d5add0b3d67b2577b094614ab563bad73679 Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Tue, 4 Sep 2012 12:55:13 +0800 Subject: add clear_tokens call --- utils/training/gen_deleted_ngram.cpp | 3 ++- utils/training/gen_k_mixture_model.cpp | 1 + utils/training/gen_ngram.cpp | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'utils/training') diff --git a/utils/training/gen_deleted_ngram.cpp b/utils/training/gen_deleted_ngram.cpp index abc3ee9..fe0a9d9 100644 --- a/utils/training/gen_deleted_ngram.cpp +++ b/utils/training/gen_deleted_ngram.cpp @@ -88,7 +88,8 @@ int main(int argc, char * argv[]){ phrase_token_t token = null_token; if ( 0 != phrase_len ) { - int result = phrase_table.search( phrase_len, phrase, tokens); + phrase_index.clear_tokens(tokens); + int result = phrase_table.search(phrase_len, phrase, tokens); int num = get_first_token(tokens, token); if ( !(result & SEARCH_OK) ) diff --git a/utils/training/gen_k_mixture_model.cpp b/utils/training/gen_k_mixture_model.cpp index c8bc5f2..c8a8b38 100644 --- a/utils/training/gen_k_mixture_model.cpp +++ b/utils/training/gen_k_mixture_model.cpp @@ -70,6 +70,7 @@ bool read_document(PhraseLargeTable2 * phrase_table, phrase_token_t token = null_token; if ( 0 != phrase_len ) { + phrase_index->clear_tokens(tokens); int search_result = phrase_table->search (phrase_len, phrase, tokens); int num = get_first_token(tokens, token); diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp index 20448b1..983f967 100644 --- a/utils/training/gen_ngram.cpp +++ b/utils/training/gen_ngram.cpp @@ -87,6 +87,7 @@ int main(int argc, char * argv[]){ phrase_token_t token = null_token; if ( 0 != phrase_len ) { + phrase_index.clear_tokens(tokens); int result = phrase_table.search(phrase_len, phrase, tokens); int num = get_first_token(tokens, token); if ( !(result & SEARCH_OK) ) -- cgit