From f917d5add0b3d67b2577b094614ab563bad73679 Mon Sep 17 00:00:00 2001
From: Peng Wu <alexepico@gmail.com>
Date: Tue, 4 Sep 2012 12:55:13 +0800
Subject: add clear_tokens call

---
 utils/training/gen_ngram.cpp | 1 +
 1 file changed, 1 insertion(+)

(limited to 'utils/training/gen_ngram.cpp')

diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp
index 20448b1..983f967 100644
--- a/utils/training/gen_ngram.cpp
+++ b/utils/training/gen_ngram.cpp
@@ -87,6 +87,7 @@ int main(int argc, char * argv[]){
 
 	phrase_token_t token = null_token;
         if ( 0 != phrase_len ) {
+            phrase_index.clear_tokens(tokens);
             int result = phrase_table.search(phrase_len, phrase, tokens);
             int num = get_first_token(tokens, token);
             if ( !(result & SEARCH_OK) )
-- 
cgit