begin to write import k mixture model

3 files changed, 229 insertions, 10 deletions

diff --git a/utils/storage/import_interpolation.cpp b/utils/storage/import_interpolation.cpp
index 6c97109..bc2da68 100644
--- a/utils/storage/import_interpolation.cpp
+++ b/utils/storage/import_interpolation.cpp
@@ -97,14 +97,14 @@ bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
 
     do {
         assert(taglib_read(linebuf, line_type, values, required));
-        switch(line_type) {
+        switch (line_type) {
         case GRAM_1_ITEM_LINE:{
             /* handle \item in \1-gram */
             const char * string = (const char *) g_ptr_array_index(values, 0);
             phrase_token_t token = taglib_string_to_token(phrases, string);
-            char * value = NULL;
-            assert(g_hash_table_lookup_extended(required, "count", NULL, (gpointer *)&value));
-            glong count = atol(value);
+            gpointer value = NULL;
+            assert(g_hash_table_lookup_extended(required, "count", NULL, &value));
+            glong count = atol((const char *)value);
             phrase_index->add_unigram_frequency(token, count);
             break;
         }
@@ -115,7 +115,7 @@ bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
         default:
             assert(false);
         }
-    } while (my_getline(input) != -1) ;
+    } while (my_getline(input) != -1);
 
  end:
     taglib_pop_state();
@@ -132,7 +132,7 @@ bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
     phrase_token_t last_token = 0; SingleGram * last_single_gram = NULL;
     do {
         assert(taglib_read(linebuf, line_type, values, required));
-        switch(line_type) {
+        switch (line_type) {
         case GRAM_2_ITEM_LINE:{
             /* handle \item in \2-gram */
             /* two tokens */
@@ -141,10 +141,10 @@ bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
             string = (const char *) g_ptr_array_index(values, 1);
             phrase_token_t token2 = taglib_string_to_token(phrases, string);
 
+            gpointer value = NULL;
             /* tag: count */
-            char * value = NULL;
-            assert(g_hash_table_lookup_extended(required, "count", NULL, (gpointer *)&value));
-            glong count = atol(value);
+            assert(g_hash_table_lookup_extended(required, "count", NULL, &value));
+            glong count = atol((char *)value);
 
             if ( last_token != token1 ) {
                 if ( last_token && last_single_gram ) {
@@ -166,7 +166,7 @@ bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
             //save the freq
             guint32 total_freq = 0;
             assert(last_single_gram->get_total_freq(total_freq));
-            last_single_gram->insert_freq(token2, count);
+            assert(last_single_gram->insert_freq(token2, count));
             total_freq += count;
             assert(last_single_gram->set_total_freq(total_freq));
             break;
diff --git a/utils/training/Makefile.am b/utils/training/Makefile.am
index adc0c30..3a3d6b9 100644
--- a/utils/training/Makefile.am
+++ b/utils/training/Makefile.am
@@ -34,6 +34,7 @@ noinst_PROGRAMS		= gen_ngram \
 			  estimate_k_mixture_model \
 			  merge_k_mixture_model \
 			  prune_k_mixture_model \
+			  import_k_mixture_model \
 			  export_k_mixture_model
 
 gen_ngram_SOURCES	= gen_ngram.cpp
@@ -68,6 +69,10 @@ prune_k_mixture_model_SOURCES = prune_k_mixture_model.cpp
 
 prune_k_mixture_model_LDADD = ../../src/libpinyin.la @GLIB2_LDFLAGS@
 
+import_k_mixture_model_SOURCES = import_k_mixture_model.cpp
+
+import_k_mixture_model_LDADD = ../storage/libtagutils.la ../../src/libpinyin.la @GLIB2_LDFLAGS@
+
 export_k_mixture_model_SOURCES = export_k_mixture_model.cpp
 
 export_k_mixture_model_LDADD = ../storage/libtagutils.la ../../src/libpinyin.la @GLIB2_LDFLAGS@
\ No newline at end of file
diff --git a/utils/training/import_k_mixture_model.cpp b/utils/training/import_k_mixture_model.cpp
new file mode 100644
index 0000000..f669170
--- /dev/null
+++ b/utils/training/import_k_mixture_model.cpp
@@ -0,0 +1,214 @@
+/* 
+ *  libpinyin
+ *  Library to deal with pinyin.
+ *  
+ *  Copyright (C) 2011 Peng Wu <alexepico@gmail.com>
+ *  
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <stdio.h>
+#include "pinyin.h"
+#include "tag_utility.h"
+#include "k_mixture_model.h"
+
+enum LINE_TYPE{
+    BEGIN_LINE = 1,
+    END_LINE,
+    GRAM_1_LINE,
+    GRAM_2_LINE,
+    GRAM_1_ITEM_LINE,
+    GRAM_2_ITEM_LINE
+};
+
+static int line_type = 0;
+static GPtrArray * values = NULL;
+static GHashTable * required = NULL;
+/* variables for line buffer. */
+static char * linebuf = NULL;
+static size_t len = 0;
+
+bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
+                   KMixtureModelBigram * bigram);
+
+bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
+                  KMixtureModelBigram * bigram);
+
+static ssize_t my_getline(FILE * input){
+    ssize_t result = getline(&linebuf, &len, input);
+    if ( result == -1 )
+        return result;
+
+    linebuf[strlen(linebuf) - 1] = '\0';
+    return result;
+}
+
+bool parse_body(FILE * input, PhraseLargeTable * phrases,
+                KMixtureModelBigram * bigram){
+    taglib_push_state();
+
+    assert(taglib_add_tag(END_LINE, "\\end", 0, "", ""));
+    assert(taglib_add_tag(GRAM_1_LINE, "\\1-gram", 0, "", ""));
+    assert(taglib_add_tag(GRAM_2_LINE, "\\2-gram", 0, "", ""));
+
+    do {
+    retry:
+        assert(taglib_read(linebuf, line_type, values, required));
+        switch(line_type) {
+        case END_LINE:
+            goto end;
+        case GRAM_1_LINE:
+            my_getline(input);
+            parse_unigram(input, phrases, bigram);
+            goto retry;
+        case GRAM_2_LINE:
+            my_getline(input);
+            parse_bigram(input, phrases, bigram);
+            goto retry;
+        default:
+            assert(false);
+        }
+    } while (my_getline(input) != -1) ;
+
+ end:
+    taglib_pop_state();
+    return true;
+}
+
+bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
+                   KMixtureModelBigram * bigram){
+    taglib_push_state();
+
+    assert(taglib_add_tag(GRAM_1_ITEM_LINE, "\\item", 1, "count", ""));
+
+    do {
+        assert(taglib_read(linebuf, line_type, values, required));
+        switch (line_type) {
+        case GRAM_1_ITEM_LINE:{
+            /* handle \item in \1-gram */
+            const char * string = (const char *) g_ptr_array_index(values, 0);
+            phrase_token_t token = taglib_string_to_token(phrases, string);
+            gpointer value = NULL;
+            assert(g_hash_table_lookup_extended(required, "count",
+                                                NULL, &value));
+            glong count = atol((const char *)value);
+            KMixtureModelArrayHeader array_header;
+            memset(&array_header, 0, sizeof(KMixtureModelArrayHeader));
+            array_header.m_WC = count;
+            bigram->set_array_header(token, array_header);
+            break;
+        }
+        case END_LINE:
+        case GRAM_1_LINE:
+        case GRAM_2_LINE:
+            goto end;
+        default:
+            assert(false);
+        }
+    } while (my_getline(input) != -1);
+
+ end:
+    taglib_pop_state();
+    return true;
+}
+
+bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
+                  KMixtureModelBigram * bigram){
+    taglib_push_state();
+
+    assert(taglib_add_tag(GRAM_2_ITEM_LINE, "\\item", 2,
+                          "count:T:N_n_0:n_1:Mr", ""));
+
+    phrase_token_t last_token = 0;
+    KMixtureModelSingleGram * last_single_gram = NULL;
+    do {
+        assert(taglib_read(linebuf, line_type, values, required));
+        switch (line_type) {
+        case GRAM_2_ITEM_LINE:{
+            /* handle \item in \2-gram */
+            /* two tokens */
+            const char * string = (const char *) g_ptr_array_index(values, 0);
+            phrase_token_t token1 = taglib_string_to_token(phrases, string);
+            string = (const char *) g_ptr_array_index(values, 1);
+            phrase_token_t token2 = taglib_string_to_token(phrases, string);
+
+            gpointer value = NULL;
+            /* tag: count */
+            assert(g_hash_table_lookup_extended(required, "count", NULL, &value));
+            glong count = atol((char *)value);
+            /* tag: T */
+            assert(g_hash_table_lookup_extended(required, "T", NULL, &value));
+            glong T = atol((char *)value);
+            assert(count == T);
+            /* tag: N_n_0 */
+            assert(g_hash_table_lookup_extended(required, "N_n_0", NULL, &value));
+            glong N_n_0 = atol((char *)value);
+            /* tag: n_1 */
+            assert(g_hash_table_lookup_extended(required, "n_1", NULL, &value));
+            glong n_1 = atol((char *)value);
+            /* tag: Mr */
+            assert(g_hash_table_lookup_extended(required, "Mr", NULL, &value));
+            glong Mr = atol((char *)value);
+
+            KMixtureModelArrayItem array_item;
+            memset(&array_item, 0, sizeof(KMixtureModelArrayItem));
+            array_item.m_WC = count; array_item.m_N_n_0 = N_n_0;
+            array_item.m_n_1 = n_1; array_item.m_Mr = Mr;
+
+            if ( last_token != token1 ) {
+                if ( last_token && last_single_gram ) {
+                    bigram->store(last_token, last_single_gram);
+                    delete last_single_gram;
+                    //safe guard
+                    last_token = 0;
+                    last_single_gram = NULL;
+                }
+                KMixtureModelSingleGram * single_gram = NULL;
+                bigram->load(token1, single_gram);
+
+                //create the new single gram
+                if ( single_gram == NULL )
+                    single_gram = new KMixtureModelSingleGram;
+                last_token = token1;
+                last_single_gram = single_gram;
+            }
+            assert(last_single_gram->insert_array_item(token2, array_item));
+            break;
+        }
+        case END_LINE:
+        case GRAM_1_LINE:
+        case GRAM_2_LINE:
+            goto end;
+        default:
+            assert(false);
+        }
+    } while (my_getline(input) != -1);
+
+ end:
+    if ( last_token && last_single_gram ) {
+        bigram->store(last_token, last_single_gram);
+        delete last_single_gram;
+        //safe guard
+        last_token = 0;
+        last_single_gram = NULL;
+    }
+
+    taglib_pop_state();
+    return true;
+}
+
+int main(int argc, char * argv[]){
+    return 0;
+}