move segment and training tools to utils

6 files changed, 648 insertions, 0 deletions

diff --git a/utils/segment/Makefile.am b/utils/segment/Makefile.am
new file mode 100644
index 0000000..a82627a
--- /dev/null
+++ b/utils/segment/Makefile.am
@@ -0,0 +1,28 @@
+## Makefile.am -- Process this file with automake to produce Makefile.in
+## Copyright (C) 2007 Peng Wu
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2, or (at your option)
+## any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+MAINTAINERCLEANFILES    = Makefile.in
+
+INCLUDES		= -I$(top_srcdir)/src/include \
+			  -I$(top_srcdir)/src/storage \
+			  @GLIB2_CPPFLAGS@
+
+noinst_PROGRAMS		= spseg
+
+spseg_SOURCES		= spseg.cpp
+
+spseg_LDADD		= @GLIB2_LDFLAGS@ ../storage/libstorage.la
diff --git a/utils/segment/spseg.cpp b/utils/segment/spseg.cpp
new file mode 100644
index 0000000..f4522f2
--- /dev/null
+++ b/utils/segment/spseg.cpp
@@ -0,0 +1,209 @@
+/* 
+ *  libpinyin
+ *  Library to deal with pinyin.
+ *  
+ *  Copyright (C) 2010 Peng Wu
+ *  
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <locale.h>
+#include <glib.h>
+#include "novel_types.h"
+#include "phrase_large_table.h"
+
+/* graph shortest path sentence segment. */
+
+/* Note:
+ * Currently libpinyin only supports ucs2 characters, as this is a
+ * pre-processor tool for raw corpus, it will skip all sentences
+ * which contains non-ucs2 characters.
+ */
+
+static PhraseLargeTable * g_phrases = NULL;
+
+struct SegmentStep{
+    phrase_token_t m_handle;
+    utf16_t * m_phrase;
+    size_t m_phrase_len;
+    //use formula W = number of words. Zero handle means one word.
+    guint m_nword;
+    //backtrace information, -1 one step backward.
+    gint m_backward_nstep;
+public:
+    SegmentStep(){
+        m_handle = 0;
+        m_phrase = NULL;
+        m_phrase_len = 0;
+        m_nword = UINT_MAX;
+        m_backward_nstep = -0;
+    }
+};
+
+bool backtrace(GArray * steps, glong phrase_len, GArray * strings);
+
+//Note: do not free phrase, as it is used by strings (array of segment).
+bool segment(PhraseLargeTable * phrases, //Lookup Phrase
+             utf16_t * phrase,
+             glong phrase_len,
+             GArray * strings /* Array of Segment *. */){
+    /* Prepare for shortest path segment dynamic programming. */
+    GArray * steps = g_array_new(TRUE, TRUE, sizeof(SegmentStep));
+    SegmentStep step;
+    for ( glong i = 0; i < phrase_len + 1; ++i ){
+        g_array_append_val(steps, step);
+    }
+
+    SegmentStep * first_step = &g_array_index(steps, SegmentStep, 0);
+    first_step->m_nword = 0;
+
+    for ( glong i = 0; i < phrase_len + 1; ++i ) {
+        SegmentStep * step_begin = &g_array_index(steps, SegmentStep, i);
+        size_t nword = step_begin->m_nword;
+        for ( glong k = i + 1; k < phrase_len + 1; ++k ) {
+            size_t len = k - i;
+            utf16_t * cur_phrase = phrase + i;
+
+            phrase_token_t token = 0;
+            int result = g_phrases->search(len, cur_phrase, token);
+            if ( !(result & SEARCH_OK) ){
+                token = 0;
+                if ( 1 != len )
+                    continue;
+            }
+            ++nword;
+            SegmentStep * step_end = &g_array_index(steps, SegmentStep, k);
+            if ( nword < step_end->m_nword ) {
+                step_end->m_handle = token;
+                step_end->m_phrase = cur_phrase;
+                step_end->m_phrase_len = len;
+                step_end->m_nword = nword;
+                step_end->m_backward_nstep = i - k;
+            }
+            if ( !(result & SEARCH_CONTINUED) )
+                break;
+        }
+    }
+    return backtrace(steps, phrase_len, strings);
+}
+
+bool backtrace(GArray * steps, glong phrase_len, GArray * strings){
+    //backtracing to get the result.
+    size_t cur_step = phrase_len;
+    g_array_set_size(strings, 0);
+    while ( cur_step ){
+        SegmentStep * step = &g_array_index(steps, SegmentStep, cur_step);
+        g_array_append_val(strings, *step);
+        cur_step = cur_step + step->m_backward_nstep;
+        //intended to avoid leaking internal informations
+        step->m_nword = 0; step->m_backward_nstep = 0;
+    }
+
+    //reverse the strings
+    for ( size_t i = 0; i < strings->len / 2; ++i ) {
+        SegmentStep * head, * tail;
+        head = &g_array_index(strings, SegmentStep, i);
+        tail = &g_array_index(strings, SegmentStep, strings->len - 1 - i );
+        SegmentStep tmp;
+        tmp = *head;
+        *head = *tail;
+        *tail = tmp;
+    }
+
+    g_array_free(steps, TRUE);
+    return true;
+}
+
+void print_help(){
+    printf("Usage: mmseg [--generate-extra-enter]\n");
+    exit(1);
+}
+
+int main(int argc, char * argv[]){
+    int i = 1;
+    bool gen_extra_enter = false;
+
+    setlocale(LC_ALL, "");
+    //deal with options.
+    while ( i < argc ){
+        if ( strcmp ("--help", argv[i]) == 0) {
+            print_help();
+        }else if (strcmp("--generate-extra-enter", argv[i]) == 0) {
+            gen_extra_enter = true;
+        }
+        ++i;
+    }
+
+    //init phrase table
+    g_phrases =  new PhraseLargeTable;
+    FILE * gb_file = fopen("../../data/gb_char.table", "r");
+    if ( gb_file == NULL ){
+	fprintf(stderr, "can't open gb_char.table!\n");
+	exit(1);
+    }
+    g_phrases->load_text(gb_file);
+    fclose(gb_file);
+
+    FILE * gbk_file = fopen("../../data/gbk_char.table", "r");
+    if ( gbk_file == NULL ){
+	fprintf(stderr, "can't open gbk_char.table!\n");
+	exit(1);
+    }
+    g_phrases->load_text(gbk_file);
+    fclose(gbk_file);
+
+    MemoryChunk * chunk = new MemoryChunk;
+    g_phrases->store(chunk);
+    g_phrases->load(chunk);
+
+    char * linebuf = NULL;
+    size_t size = 0;
+    while(getline(&linebuf, &size, stdin)) {
+        if ( feof(stdin) )
+            break;
+        linebuf[strlen(linebuf) - 1] = '\0';
+
+        //check non-ucs2 characters
+        const glong num_of_chars = g_utf8_strlen(linebuf, -1);
+        glong len = 0;
+        utf16_t * sentence = g_utf8_to_utf16(linebuf, -1, NULL, &len, NULL);
+        if ( len != num_of_chars ) {
+            fprintf(stderr, "non-ucs2 characters encountered:%s.\n", linebuf);
+            continue;
+        }
+
+        //do segment stuff
+        GArray * strings = g_array_new(TRUE, TRUE, sizeof(SegmentStep));
+        segment(g_phrases, sentence, len, strings);
+
+        //print out the split phrase
+        for ( glong i = 0; i < strings->len; ++i ) {
+            SegmentStep * step = &g_array_index(strings, SegmentStep, i);
+            char * string = g_utf16_to_utf8( step->m_phrase, step->m_phrase_len, NULL, NULL, NULL);
+            printf("%s\n", string);
+            g_free(string);
+        }
+
+        //print extra enter
+        if ( gen_extra_enter )
+            printf("\n");
+
+        g_array_free(strings, TRUE);
+        g_free(sentence);
+    }
+    return 0;
+}
diff --git a/utils/training/Makefile.am b/utils/training/Makefile.am
new file mode 100644
index 0000000..520e4e1
--- /dev/null
+++ b/utils/training/Makefile.am
@@ -0,0 +1,36 @@
+## Makefile.am -- Process this file with automake to produce Makefile.in
+## Copyright (C) 2007 Peng Wu
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2, or (at your option)
+## any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+MAINTAINERCLEANFILES    = Makefile.in
+
+INCLUDES		= -I$(top_srcdir)/src/include \
+			  -I$(top_srcdir)/src/storage \
+			  @GLIB2_CPPFLAGS@
+
+noinst_PROGRAMS		= gen_ngram gen_unigram estimate_interpolation
+
+gen_ngram_SOURCES	= gen_ngram.cpp
+
+gen_ngram_LDADD		= ../storage/libstorage.la @GLIB2_LDFLAGS@
+
+gen_unigram_SOURCES     = gen_unigram.cpp
+
+gen_unigram_LDADD       = ../storage/libstorage.la @GLIB2_LDFLAGS@
+
+estimate_interpolation_SOURCES = estimate_interpolation.cpp
+
+estimate_interpolation_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@
diff --git a/utils/training/estimate_interpolation.cpp b/utils/training/estimate_interpolation.cpp
new file mode 100644
index 0000000..aaae17b
--- /dev/null
+++ b/utils/training/estimate_interpolation.cpp
@@ -0,0 +1,146 @@
+/* 
+ *  libpinyin
+ *  Library to deal with pinyin.
+ *  
+ *  Copyright (C) 2006-2008 Peng Wu
+ *  
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+#include <math.h>
+#include <glib.h>
+#include "memory_chunk.h"
+#include "novel_types.h"
+#include "phrase_index.h"
+#include "ngram.h"
+
+parameter_t compute_interpolation(SingleGram * deleted_bigram,
+				  FacadePhraseIndex * unigram,
+				  SingleGram * bigram){
+    bool success;
+    parameter_t lambda = 0, next_lambda = 0.6;
+    parameter_t epsilon = 0.001;
+    
+    while ( fabs(lambda - next_lambda) > epsilon){
+	lambda = next_lambda;
+	next_lambda = 0;
+	guint32 table_num = 0;
+	parameter_t numerator = 0;
+	parameter_t part_of_denominator = 0;
+	
+	BigramPhraseWithCountArray array = g_array_new(FALSE, FALSE, sizeof(BigramPhraseItemWithCount));
+	deleted_bigram->retrieve_all(array);
+
+	for ( int i = 0; i < array->len; ++i){
+	    BigramPhraseItemWithCount* item = &g_array_index(array, BigramPhraseItemWithCount, i);
+	    //get the phrase token
+	    phrase_token_t token = item->m_token;
+	    guint32 deleted_freq = item->m_count;
+
+	    {
+		guint32 freq = 0;
+		parameter_t elem_poss = 0;
+		if ( bigram && bigram->get_freq(token, freq)){
+		    guint32 total_freq;
+		    assert(bigram->get_total_freq(total_freq));
+		    assert(0 != total_freq);
+		    elem_poss = freq / (parameter_t) total_freq;
+		}
+		numerator = lambda * elem_poss;
+	    }
+
+	    {
+		guint32 freq = 0;
+		parameter_t elem_poss = 0;
+		PhraseItem item;
+		if (!unigram->get_phrase_item(token, item)){
+		    guint32 freq = item.get_unigram_frequency();
+		    guint32 total_freq = unigram->get_phrase_index_total_freq();
+		    elem_poss = freq / (parameter_t)total_freq;
+		}
+		part_of_denominator = ( 1 - lambda) * elem_poss;
+	    }
+	    
+	    if ( 0 == (numerator + part_of_denominator))
+		continue;
+	    
+	    next_lambda += deleted_freq * (numerator / (numerator + part_of_denominator));
+	}
+	assert(deleted_bigram->get_total_freq(table_num));
+	next_lambda /= table_num;
+
+	g_array_free(array, TRUE);
+    }
+    lambda = next_lambda;
+    return lambda;
+}
+    
+int main(int argc, char * argv[]){
+    FacadePhraseIndex phrase_index;
+    
+    //gb_char binary file
+    MemoryChunk * chunk = new MemoryChunk;
+    chunk->load("../../data/gb_char.bin");
+    phrase_index.load(1, chunk);
+    
+    //gbk_char binary file
+    chunk = new MemoryChunk;
+    chunk->load("../../data/gbk_char.bin");
+    phrase_index.load(2, chunk);
+
+    Bigram bigram;
+    bigram.attach("../../data/bigram.db", NULL);
+
+    Bigram deleted_bigram;
+    deleted_bigram.attach("../../data/deleted_bigram.db", NULL);
+
+    GArray * system_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
+    GArray * user_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
+
+    deleted_bigram.get_all_items(system_items, user_items);
+    assert(0 == user_items->len);
+    g_array_free(user_items, TRUE);
+
+    parameter_t lambda_sum = 0;
+    int lambda_count = 0;
+
+    for ( int i = 0; i < system_items->len; ++i ){
+	phrase_token_t * token = &g_array_index(system_items, phrase_token_t, i);
+	SingleGram * system = NULL, * user = NULL;
+	bigram.load(*token, system, user);
+	assert(NULL == user);
+	SingleGram * deleted_system = NULL, * deleted_user = NULL;
+	deleted_bigram.load(*token, deleted_system, deleted_user);
+	assert(NULL == deleted_user);
+	
+	parameter_t lambda = compute_interpolation(deleted_system, &phrase_index, system);
+	
+	printf("lambda:%f\n", lambda);
+
+	lambda_sum += lambda;
+	lambda_count ++;
+
+	if (system) delete system;
+	delete deleted_system;
+    }
+
+    printf("average lambda:%f\n", (lambda_sum/lambda_count));
+    g_array_free(system_items, TRUE);
+}
+
diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp
new file mode 100644
index 0000000..a1f5b6d
--- /dev/null
+++ b/utils/training/gen_ngram.cpp
@@ -0,0 +1,158 @@
+/* 
+ *  libpinyin
+ *  Library to deal with pinyin.
+ *  
+ *  Copyright (C) 2006-2007 Peng Wu
+ *  
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+#include <glib.h>
+#include "memory_chunk.h"
+#include "novel_types.h"
+#include "phrase_index.h"
+#include "ngram.h"
+#include "phrase_large_table.h"
+
+
+static PhraseLargeTable * g_phrases = NULL;
+
+void print_help(){
+    printf("gen_ngram [--skip-pi-gram-training] [--skip-unigram-training]\n");
+    printf("          [--bigram-file <FILENAME>]\n");
+    exit(1);
+}
+
+int main(int argc, char * argv[]){
+	int i = 1;
+    bool train_pi_gram = true;
+    bool train_unigram = true;
+    const char * bigram_filename = "../../data/bigram.db";
+
+    setlocale(LC_ALL,"");
+    while ( i < argc ){
+	if ( strcmp("--help", argv[i] ) == 0){
+	    print_help();
+	}else if ( strcmp("--skip-pi-gram-training", argv[i] ) == 0) {
+	    train_pi_gram = false;
+	}else if ( strcmp("--skip-unigram-training", argv[i] ) == 0) {
+	    train_unigram = false;
+	}else if ( strcmp("--bigram-file", argv[i] ) == 0){
+            if ( ++i >= argc )
+                print_help();
+            bigram_filename = argv[i];
+	}
+	++i;
+    }
+    
+    g_phrases = new PhraseLargeTable;
+    //init phrase lookup
+    FILE * gb_file = fopen("../../data/gb_char.table", "r");
+    if ( gb_file == NULL ){
+	fprintf(stderr, "can't open gb_char.table!\n");
+	exit(1);
+    }
+    g_phrases->load_text(gb_file);
+    fclose(gb_file);
+    FILE * gbk_file = fopen("../../data/gbk_char.table", "r");
+    if ( gbk_file == NULL ){
+	fprintf(stderr, "can't open gbk_char.table!\n");
+	exit(1);
+    }
+    g_phrases->load_text(gbk_file);
+    fclose(gbk_file);
+
+    FacadePhraseIndex phrase_index;
+    
+    //gb_char binary file
+    MemoryChunk * chunk = new MemoryChunk;
+    chunk->load("../../data/gb_char.bin");
+    phrase_index.load(1, chunk);
+    
+    //gbk_char binary file
+    chunk = new MemoryChunk;
+    chunk->load("../../data/gbk_char.bin");
+    phrase_index.load(2, chunk);
+    
+    Bigram bigram;
+    bigram.attach(NULL, bigram_filename);
+    
+    
+    char* linebuf = (char *)malloc ( 1024 * sizeof (char) );
+    size_t size = 1024;
+    phrase_token_t last_token, cur_token = last_token = 0;
+    while( getline(&linebuf, &size, stdin) ){
+	if ( feof(stdin) )
+	    break;
+        linebuf[strlen(linebuf)-1] = '\0';
+
+        glong phrase_len = 0;
+        utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL);
+
+	phrase_token_t token = 0;
+        int result = g_phrases->search( phrase_len, phrase, token);
+	if ( ! (result & SEARCH_OK) )
+	    token = 0;
+
+	last_token = cur_token;
+	cur_token = token;
+	if ( cur_token ){
+	    //training uni-gram
+	    if ( train_unigram )
+		phrase_index.add_unigram_frequency(cur_token, 1);
+	}
+	if ( cur_token ){
+	    SingleGram * system = NULL, * user = NULL;
+	    if ( 0 == last_token ){
+		if (train_pi_gram)
+		    bigram.load(sentence_start, system, user);
+	    } else
+		bigram.load(last_token, system, user);
+	    assert(NULL == system);
+	    if ( NULL == user ){
+		user = new SingleGram;
+	    }
+	    guint32 freq, total_freq;
+	    //increase freq
+	    user->get_freq(cur_token, freq);
+	    user->set_freq(cur_token, freq + 1);
+	    //increase total freq
+	    user->get_total_freq(total_freq);
+	    user->set_total_freq(total_freq + 1);
+	    if ( 0 == last_token ){
+		if ( train_pi_gram )
+		    bigram.store(sentence_start, user);
+	    }else
+		bigram.store(last_token, user);
+	    delete user;
+	}
+    }
+    
+    MemoryChunk * new_chunk = new MemoryChunk;
+    phrase_index.store(1, new_chunk);
+    new_chunk->save("../../data/gb_char.bin");
+    phrase_index.load(1, new_chunk);
+
+    new_chunk = new MemoryChunk;
+    phrase_index.store(2, new_chunk);
+    new_chunk->save("../../data/gbk_char.bin");
+    phrase_index.load(2, new_chunk);
+
+    return 0;
+}
diff --git a/utils/training/gen_unigram.cpp b/utils/training/gen_unigram.cpp
new file mode 100644
index 0000000..1c70665
--- /dev/null
+++ b/utils/training/gen_unigram.cpp
@@ -0,0 +1,71 @@
+/* 
+ *  libpinyin
+ *  Library to deal with pinyin.
+ *  
+ *  Copyright (C) 2006-2007 Peng Wu
+ *  
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <stdio.h>
+#include "memory_chunk.h"
+#include "novel_types.h"
+#include "phrase_index.h"
+
+//increase all unigram frequency by one.
+
+int main(int argc, char * argv[]){
+
+    FacadePhraseIndex phrase_index;
+    
+    //gb_char binary file
+    MemoryChunk * chunk = new MemoryChunk;
+    chunk->load("../../data/gb_char.bin");
+    phrase_index.load(1, chunk);
+    
+    //gbk_char binary file
+    chunk = new MemoryChunk;
+    chunk->load("../../data/gbk_char.bin");
+    phrase_index.load(2, chunk);
+
+    PhraseIndexRange range;
+    int result = phrase_index.get_range(1, range);
+    if ( result == ERROR_OK ) {
+        for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){
+            phrase_index.add_unigram_frequency(i, 1);
+        }
+    }
+
+#if 0
+    int result = phrase_index.get_range(2, range);
+    if ( result == ERROR_OK ) {
+        for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){
+            phrase_index.add_unigram_frequency(i, 1);
+        }
+    }
+#endif
+
+    MemoryChunk * new_chunk = new MemoryChunk;
+    phrase_index.store(1, new_chunk);
+    new_chunk->save("../../data/gb_char.bin");
+    phrase_index.load(1, new_chunk);
+
+    new_chunk = new MemoryChunk;
+    phrase_index.store(2, new_chunk);
+    new_chunk->save("../../data/gbk_char.bin");
+    phrase_index.load(2, new_chunk);
+
+    return 0;
+}