move token string conversion function to taglib

1 files changed, 3 insertions, 44 deletions

diff --git a/utils/storage/import_interpolation.cpp b/utils/storage/import_interpolation.cpp
index 57c87c1..6c97109 100644
--- a/utils/storage/import_interpolation.cpp
+++ b/utils/storage/import_interpolation.cpp
@@ -40,9 +40,6 @@ static GHashTable * required = NULL;
 static char * linebuf = NULL;
 static size_t len = 0;
 
-phrase_token_t string_to_token(PhraseLargeTable * phrases,
-                               const char * string);
-
 bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
                    FacadePhraseIndex * phrase_index);
 
@@ -104,7 +101,7 @@ bool parse_unigram(FILE * input, PhraseLargeTable * phrases,
         case GRAM_1_ITEM_LINE:{
             /* handle \item in \1-gram */
             const char * string = (const char *) g_ptr_array_index(values, 0);
-            phrase_token_t token = string_to_token(phrases, string);
+            phrase_token_t token = taglib_string_to_token(phrases, string);
             char * value = NULL;
             assert(g_hash_table_lookup_extended(required, "count", NULL, (gpointer *)&value));
             glong count = atol(value);
@@ -140,9 +137,9 @@ bool parse_bigram(FILE * input, PhraseLargeTable * phrases,
             /* handle \item in \2-gram */
             /* two tokens */
             const char * string = (const char *) g_ptr_array_index(values, 0);
-            phrase_token_t token1 = string_to_token(phrases, string);
+            phrase_token_t token1 = taglib_string_to_token(phrases, string);
             string = (const char *) g_ptr_array_index(values, 1);
-            phrase_token_t token2 = string_to_token(phrases, string);
+            phrase_token_t token2 = taglib_string_to_token(phrases, string);
 
             /* tag: count */
             char * value = NULL;
@@ -262,41 +259,3 @@ int main(int argc, char * argv[]){
 
     return 0;
 }
-
-static phrase_token_t special_string_to_token(const char * string){
-    struct token_pair{
-        phrase_token_t token;
-        const char * string;
-    };
-
-    static const token_pair tokens [] = {
-        {sentence_start, "<start>"},
-        {0, NULL}
-    };
-
-    const token_pair * pair = tokens;
-    while (pair->string) {
-        if ( strcmp(string, pair->string ) == 0 ){
-            return pair->token;
-        }
-    }
-
-    fprintf(stderr, "error: unknown token:%s.\n", string);
-    return 0;
-}
-
-phrase_token_t string_to_token(PhraseLargeTable * phrases, const char * string){
-    phrase_token_t token = 0;
-    if ( string[0] == '<' ) {
-        return special_string_to_token(string);
-    }
-
-    glong phrase_len = g_utf8_strlen(string, -1);
-    utf16_t * phrase = g_utf8_to_utf16(string, -1, NULL, NULL, NULL);
-    int result = phrases->search(phrase_len, phrase, token);
-    if ( !(result & SEARCH_OK) )
-        fprintf(stderr, "error: unknown token:%s.\n", string);
-
-    g_free(phrase);
-    return token;
-}