diff options
author | Peng Wu <alexepico@gmail.com> | 2011-05-10 13:50:55 +0800 |
---|---|---|
committer | Peng Wu <alexepico@gmail.com> | 2011-05-10 13:50:55 +0800 |
commit | f0f1771e3bbe95dffaa4d42596b7d4aace277d2c (patch) | |
tree | 4a34a0d3bd99a29f4617d935974a4055252b6ad3 /utils/training | |
parent | d4eb5445d98587c05be46747ab3da99d3df67a5a (diff) | |
download | libpinyin-f0f1771e3bbe95dffaa4d42596b7d4aace277d2c.tar.gz libpinyin-f0f1771e3bbe95dffaa4d42596b7d4aace277d2c.tar.xz libpinyin-f0f1771e3bbe95dffaa4d42596b7d4aace277d2c.zip |
use binary format of phrase large table
Diffstat (limited to 'utils/training')
-rw-r--r-- | utils/training/gen_deleted_ngram.cpp | 17 | ||||
-rw-r--r-- | utils/training/gen_ngram.cpp | 19 |
2 files changed, 7 insertions, 29 deletions
diff --git a/utils/training/gen_deleted_ngram.cpp b/utils/training/gen_deleted_ngram.cpp index 6db511d..cb1c4a0 100644 --- a/utils/training/gen_deleted_ngram.cpp +++ b/utils/training/gen_deleted_ngram.cpp @@ -60,20 +60,9 @@ int main(int argc, char * argv[]){ g_phrases = new PhraseLargeTable; //init phrase lookup - FILE * gb_file = fopen("../../data/gb_char.table", "r"); - if ( gb_file == NULL ){ - fprintf(stderr, "can't open gb_char.table!\n"); - exit(ENOENT); - } - g_phrases->load_text(gb_file); - fclose(gb_file); - FILE * gbk_file = fopen("../../data/gbk_char.table", "r"); - if ( gbk_file == NULL ){ - fprintf(stderr, "can't open gbk_char.table!\n"); - exit(ENOENT); - } - g_phrases->load_text(gbk_file); - fclose(gbk_file); + MemoryChunk * chunk = new MemoryChunk; + chunk->load("../../data/phrase_index.bin"); + g_phrases->load(chunk); Bigram bigram; bigram.attach(bigram_filename, ATTACH_CREATE|ATTACH_READWRITE); diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp index 71ec3f8..228abca 100644 --- a/utils/training/gen_ngram.cpp +++ b/utils/training/gen_ngram.cpp @@ -60,25 +60,14 @@ int main(int argc, char * argv[]){ g_phrases = new PhraseLargeTable; //init phrase lookup - FILE * gb_file = fopen("../../data/gb_char.table", "r"); - if ( gb_file == NULL ){ - fprintf(stderr, "can't open gb_char.table!\n"); - exit(ENOENT); - } - g_phrases->load_text(gb_file); - fclose(gb_file); - FILE * gbk_file = fopen("../../data/gbk_char.table", "r"); - if ( gbk_file == NULL ){ - fprintf(stderr, "can't open gbk_char.table!\n"); - exit(ENOENT); - } - g_phrases->load_text(gbk_file); - fclose(gbk_file); + MemoryChunk * chunk = new MemoryChunk; + chunk->load("../../data/phrase_index.bin"); + g_phrases->load(chunk); FacadePhraseIndex phrase_index; //gb_char binary file - MemoryChunk * chunk = new MemoryChunk; + chunk = new MemoryChunk; chunk->load("../../data/gb_char.bin"); phrase_index.load(1, chunk); |