1 files changed, 1 insertions, 8 deletions
diff --git a/utils/training/gen_deleted_ngram.cpp b/utils/training/gen_deleted_ngram.cpp
index 55679ed..157266a 100644
--- a/utils/training/gen_deleted_ngram.cpp
+++ b/utils/training/gen_deleted_ngram.cpp
@@ -70,12 +70,7 @@ int main(int argc, char * argv[]){
     Bigram bigram;
     bigram.attach(bigram_filename, ATTACH_CREATE|ATTACH_READWRITE);
 
-    PhraseTokens tokens;
-    memset(tokens, 0, sizeof(PhraseTokens));
-    phrase_index.prepare_tokens(tokens);
-
-    char* linebuf = NULL;
-    size_t size = 0;
+    char* linebuf = NULL; size_t size = 0;
     phrase_token_t last_token, cur_token = last_token = 0;
     while( getline(&linebuf, &size, stdin) ){
 	if ( feof(stdin) )
@@ -122,8 +117,6 @@ int main(int argc, char * argv[]){
         delete single_gram;
     }
 
-    phrase_index.destroy_tokens(tokens);
-
     free(linebuf);
     return 0;
 }