From 1289df5e2ebfd7a9b6a5aa0b4f1739c48527ce19 Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Mon, 3 Sep 2012 16:14:09 +0800 Subject: update tag utility --- src/storage/tag_utility.cpp | 17 +++++++++++++---- src/storage/tag_utility.h | 6 ++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/storage/tag_utility.cpp b/src/storage/tag_utility.cpp index 2a9d3cf..5115f7d 100644 --- a/src/storage/tag_utility.cpp +++ b/src/storage/tag_utility.cpp @@ -4,7 +4,7 @@ #include #include "novel_types.h" #include "phrase_index.h" -#include "phrase_large_table.h" +#include "phrase_large_table2.h" #include "tag_utility.h" namespace pinyin{ @@ -330,15 +330,24 @@ static phrase_token_t taglib_special_string_to_token(const char * string){ return 0; } -phrase_token_t taglib_string_to_token(PhraseLargeTable * phrases, const char * string){ - phrase_token_t token = 0; +phrase_token_t taglib_string_to_token(PhraseLargeTable2 * phrase_table, + FacadePhraseIndex * phrase_index, + const char * string){ + phrase_token_t token = null_token; if ( string[0] == '<' ) { return taglib_special_string_to_token(string); } glong phrase_len = g_utf8_strlen(string, -1); ucs4_t * phrase = g_utf8_to_ucs4(string, -1, NULL, NULL, NULL); - int result = phrases->search(phrase_len, phrase, token); + + PhraseTokens tokens; + memset(tokens, 0, sizeof(PhraseTokens)); + phrase_index->prepare_tokens(tokens); + int result = phrase_table->search(phrase_len, phrase, tokens); + int num = get_first_token(tokens, token); + phrase_index->destroy_tokens(tokens); + if ( !(result & SEARCH_OK) ) fprintf(stderr, "error: unknown token:%s.\n", string); diff --git a/src/storage/tag_utility.h b/src/storage/tag_utility.h index 38acb68..a68877d 100644 --- a/src/storage/tag_utility.h +++ b/src/storage/tag_utility.h @@ -117,14 +117,16 @@ class FacadePhraseIndex; /** * taglib_string_to_token: - * @phrases: the phrase table for token lookup. + * @phrase_table: the phrase table for token lookup. + * @phrase_index: the phrase index to prepare PhraseTokens. * @string: the string of the phrase. * @returns: the phrase token found in phrase table. * * Translate one phrase into the token. * */ -phrase_token_t taglib_string_to_token(PhraseLargeTable * phrases, +phrase_token_t taglib_string_to_token(PhraseLargeTable2 * phrase_table, + FacadePhraseIndex * phrase_index, const char * string); /** -- cgit