diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/storage/phrase_large_table2.h | 24 |
1 files changed, 24 insertions, 0 deletions
diff --git a/src/storage/phrase_large_table2.h b/src/storage/phrase_large_table2.h index 9f9f718..36ef53c 100644 --- a/src/storage/phrase_large_table2.h +++ b/src/storage/phrase_large_table2.h @@ -108,6 +108,30 @@ public: } }; + +static inline int reduce_tokens(PhraseTokens tokens, + GArray * tokenarray) { + int num = 0; + + for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) { + GArray * array = tokens[i]; + if (NULL == array) + continue; + + num += array->len; + + for (size_t j = 0; j < array->len; ++j) { + phrase_token_t token = g_array_index(array, phrase_token_t, j); + g_array_append_val(tokenarray, token); + } + } + + /* the following line will be removed in future after code are verified. */ + assert(0 == num || 1 == num); + + return num; +} + /* for compatibility. */ static inline int get_first_token(PhraseTokens tokens, /* out */ phrase_token_t & token){ |