diff options
author | Peng Wu <alexepico@gmail.com> | 2011-04-27 16:11:06 +0800 |
---|---|---|
committer | Peng Wu <alexepico@gmail.com> | 2011-04-27 16:11:06 +0800 |
commit | 844ff3da1745e9433f893390e85eb64c5c3fd5aa (patch) | |
tree | abf5ea2fc1ca65b17cd2a273dc24cb8611acd09f | |
parent | c44fa3478994e94d0fea8f2984c1d45515ec7024 (diff) | |
download | libpinyin-844ff3da1745e9433f893390e85eb64c5c3fd5aa.tar.gz libpinyin-844ff3da1745e9433f893390e85eb64c5c3fd5aa.tar.xz libpinyin-844ff3da1745e9433f893390e85eb64c5c3fd5aa.zip |
write compute interpolation for flexible bi-gram
-rw-r--r-- | utils/training/estimate_k_mixture_model.cpp | 61 |
1 files changed, 61 insertions, 0 deletions
diff --git a/utils/training/estimate_k_mixture_model.cpp b/utils/training/estimate_k_mixture_model.cpp index dc3d23d..6399281 100644 --- a/utils/training/estimate_k_mixture_model.cpp +++ b/utils/training/estimate_k_mixture_model.cpp @@ -1,6 +1,67 @@ #include "pinyin.h" #include "k_mixture_model.h" +parameter_t compute_interpolation(KMixtureModelSingleGram * deleted_bigram, + KMixtureModelBigram * unigram, + KMixtureModelSingleGram * bigram){ + bool success; + parameter_t lambda = 0, next_lambda = 0.6; + parameter_t epsilon = 0.001; + + while ( fabs(lambda - next_lambda) > epsilon){ + lambda = next_lambda; + next_lambda = 0; + parameter_t numerator = 0; + parameter_t part_of_denominator = 0; + + FlexibleBigramPhraseArray array = g_array_new(FALSE, FALSE, sizeof(KMixtureModelArrayItemWithToken)); + deleted_bigram->retrieve_all(array); + + for ( int i = 0; i < array->len; ++i){ + KMixtureModelArrayItemWithToken * item = &g_array_index(array, KMixtureModelArrayItemWithToken, i); + //get the phrase token + phrase_token_t token = item->m_token; + guint32 deleted_count = item->m_item.m_WC; + + { + parameter_t elem_poss = 0; + KMixtureModelArrayItem item; + if ( bigram && bigram->get_array_item(token, item) ){ + KMixtureModelArrayHeader header; + assert(bigram->get_array_header(header)); + assert(0 != header.m_WC); + elem_poss = item.m_WC / (parameter_t) header.m_WC; + } + numerator = lambda * elem_poss; + } + + { + parameter_t elem_poss = 0; + KMixtureModelMagicHeader magic_header; + KMixtureModelArrayHeader array_header; + if (unigram->get_array_header(token, array_header)){ + /* Note: optimize here? */ + assert(unigram->get_magic_header(magic_header)); + assert(0 != magic_header.m_WC); + elem_poss = array_header.m_WC / (parameter_t) magic_header.m_WC; + } + part_of_denominator = (1 - lambda) * elem_poss; + } + if (0 == (numerator + part_of_denominator)) + continue; + + next_lambda += deleted_count * (numerator / (numerator + part_of_denominator)); + } + KMixtureModelArrayHeader header; + assert(deleted_bigram->get_array_header(header)); + next_lambda /= header.m_WC; + + g_array_free(array, TRUE); + } + lambda = next_lambda; + return lambda; +} + int main(int argc, char * argv[]){ return 0; } |