write parse unigram for import tool

1 files changed, 104 insertions, 10 deletions

diff --git a/utils/storage/import_interpolation.cpp b/utils/storage/import_interpolation.cpp
index 6e88fdb..fb5a18a 100644
--- a/utils/storage/import_interpolation.cpp
+++ b/utils/storage/import_interpolation.cpp
@@ -44,6 +44,9 @@ static GHashTable * required = NULL;
 static char * linebuf = NULL;
 static size_t len = 0;
 
+phrase_token_t string_to_token(PhraseLargeTable * phrases,
+                               const char * string);
+
 bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
                    FacadePhraseIndex * phrase_index);
 
@@ -51,6 +54,15 @@ bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
                   FacadePhraseIndex * phrase_index,
                   Bigram * bigram);
 
+static ssize_t my_getline(FILE * input){
+    ssize_t result = getline(&linebuf, &len, input);
+    if ( result == -1 )
+        return result;
+
+    linebuf[strlen(linebuf) - 1] = '\0';
+    return result;
+}
+
 bool parse_body(FILE * input, PhraseLargeTable * phrases,
                 FacadePhraseIndex * phrase_index,
                 Bigram * bigram){
@@ -60,25 +72,24 @@ bool parse_body(FILE * input, PhraseLargeTable * phrases,
     assert(taglib_add_tag(GRAM_1_LINE, "\\1-gram", 0, "", ""));
     assert(taglib_add_tag(GRAM_2_LINE, "\\2-gram", 0, "", ""));
 
-    do{
+    do {
     retry:
         assert(taglib_read(linebuf, line_type, values, required));
         switch(line_type) {
         case END_LINE:
             goto end;
-            break;
         case GRAM_1_LINE:
+            my_getline(input);
             parse_unigram(input, phrases, phrase_index);
             goto retry;
-            break;
         case GRAM_2_LINE:
+            my_getline(input);
             parse_bigram(input, phrases, phrase_index, bigram);
             goto retry;
-            break;
         default:
             assert(false);
         }
-    } while (getline(&linebuf, &len, input) != -1) ;
+    } while (my_getline(input) != -1) ;
 
  end:
     taglib_pop_state();
@@ -87,14 +98,46 @@ bool parse_body(FILE * input, PhraseLargeTable * phrases,
 
 bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
                    FacadePhraseIndex * phrase_index){
+    taglib_push_state();
+
+    assert(taglib_add_tag(GRAM_1_ITEM_LINE, "\\item", 1, "count", ""));
+
+    do {
+        assert(taglib_read(linebuf, line_type, values, required));
+        switch(line_type) {
+        case GRAM_1_ITEM_LINE:{
+            /* handle \item in \1-gram */
+            assert(values->len == 1);
+            const char * string = (const char *)g_ptr_array_index(values, 0);
+            phrase_token_t token = string_to_token(phrases, string);
+            char * value = NULL;
+            assert(g_hash_table_lookup_extended(required, "count", NULL, (gpointer *)&value));
+            glong count = atol(value);
+            phrase_index->add_unigram_frequency(token, count);
+            break;
+        }
+        case END_LINE:
+        case GRAM_1_LINE:
+        case GRAM_2_LINE:
+            goto end;
+        default:
+            assert(false);
+        }
+    } while (my_getline(input) != -1) ;
 
+ end:
+    taglib_pop_state();
     return true;
 }
 
 bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
                   FacadePhraseIndex * phrase_index,
                   Bigram * bigram){
+    taglib_push_state();
 
+    assert(taglib_add_tag(GRAM_2_ITEM_LINE, "\\item", 2, "count", ""));
+
+    taglib_pop_state();
     return true;
 }
 
@@ -128,21 +171,72 @@ int main(int argc, char * argv[]){
     values = g_ptr_array_new();
     required = g_hash_table_new(g_str_hash, g_str_equal);
 
+    //enter "\data" line
     assert(taglib_add_tag(BEGIN_LINE, "\\data", 0, "model", ""));
-    getline(&linebuf, &len, input);
-    linebuf[strlen(linebuf) - 1] = '\0';
+    my_getline(input);
 
+    //read "\data" line
     assert(taglib_read(linebuf, line_type, values, required));
     assert(line_type == BEGIN_LINE);
     char * value = NULL;
     assert(g_hash_table_lookup_extended(required, "model", NULL, (gpointer *)&value));
-    assert(strcmp("interpolation", value) == 0);
+    if ( !( strcmp("interpolation", value) == 0 ) ) {
+        fprintf(stderr, "error: interpolation model expected.\n");
+        exit(1);
+    }
 
-    getline(&linebuf, &len, input);
-    linebuf[strlen(linebuf) - 1] = '\0';
+    my_getline(input);
     parse_body(input, &phrases, &phrase_index, &bigram);
 
     taglib_fini();
 
+    chunk = new MemoryChunk;
+    phrase_index.store(1, chunk);
+    chunk->save("../../data/gb_char.bin");
+    phrase_index.load(1, chunk);
+
+    chunk = new MemoryChunk;
+    phrase_index.store(2, chunk);
+    chunk->save("../../data/gbk_char.bin");
+    phrase_index.load(2, chunk);
+
     return 0;
 }
+
+static phrase_token_t special_string_to_token(const char * string){
+    struct token_pair{
+        phrase_token_t token;
+        const char * string;
+    };
+
+    static const token_pair tokens [] = {
+        {sentence_start, "<start>"},
+        {0, NULL}
+    };
+
+    const token_pair * pair = tokens;
+    while (pair->string) {
+        if ( strcmp(string, pair->string ) == 0 ){
+            return pair->token;
+        }
+    }
+
+    fprintf(stderr, "error: unknown token:%s.\n", string);
+    return 0;
+}
+
+phrase_token_t string_to_token(PhraseLargeTable * phrases, const char * string){
+    phrase_token_t token = 0;
+    if ( string[0] == '<' ) {
+        return special_string_to_token(string);
+    }
+
+    glong phrase_len = g_utf8_strlen(string, -1);
+    utf16_t * phrase = g_utf8_to_utf16(string, -1, NULL, NULL, NULL);
+    int result = phrases->search(phrase_len, phrase, token);
+    if ( !(result & SEARCH_OK) )
+        fprintf(stderr, "error: unknown token:%s.\n", string);
+
+    g_free(phrase);
+    return token;
+}