1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
/*
* libpinyin
* Library to deal with pinyin.
*
* Copyright (C) 2011 Peng Wu <alexepico@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include "pinyin.h"
#include "k_mixture_model.h"
parameter_t compute_interpolation(KMixtureModelSingleGram * deleted_bigram,
KMixtureModelBigram * unigram,
KMixtureModelSingleGram * bigram){
bool success;
parameter_t lambda = 0, next_lambda = 0.6;
parameter_t epsilon = 0.001;
while (fabs(lambda - next_lambda) > epsilon){
lambda = next_lambda;
next_lambda = 0;
parameter_t numerator = 0;
parameter_t part_of_denominator = 0;
FlexibleBigramPhraseArray array = g_array_new(FALSE, FALSE, sizeof(KMixtureModelArrayItemWithToken));
deleted_bigram->retrieve_all(array);
for ( int i = 0; i < array->len; ++i){
KMixtureModelArrayItemWithToken * item = &g_array_index(array, KMixtureModelArrayItemWithToken, i);
//get the phrase token
phrase_token_t token = item->m_token;
guint32 deleted_count = item->m_item.m_WC;
{
parameter_t elem_poss = 0;
KMixtureModelArrayItem item;
if ( bigram && bigram->get_array_item(token, item) ){
KMixtureModelArrayHeader header;
assert(bigram->get_array_header(header));
assert(0 != header.m_WC);
elem_poss = item.m_WC / (parameter_t) header.m_WC;
}
numerator = lambda * elem_poss;
}
{
parameter_t elem_poss = 0;
KMixtureModelMagicHeader magic_header;
KMixtureModelArrayHeader array_header;
if (unigram->get_array_header(token, array_header)){
/* Note: optimize here? */
assert(unigram->get_magic_header(magic_header));
assert(0 != magic_header.m_WC);
elem_poss = array_header.m_WC / (parameter_t) magic_header.m_WC;
}
part_of_denominator = (1 - lambda) * elem_poss;
}
if (0 == (numerator + part_of_denominator))
continue;
next_lambda += deleted_count * (numerator / (numerator + part_of_denominator));
}
KMixtureModelArrayHeader header;
assert(deleted_bigram->get_array_header(header));
assert(0 != header.m_WC);
next_lambda /= header.m_WC;
g_array_free(array, TRUE);
}
lambda = next_lambda;
return lambda;
}
int main(int argc, char * argv[]){
/* TODO: magic header signature check here. */
KMixtureModelBigram bigram;
bigram.attach("../../data/k_mixture_model_ngram.db");
KMixtureModelBigram deleted_bigram;
deleted_bigram.attach("../../data/k_mixture_model_deleted_ngram.db");
GArray * deleted_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
deleted_bigram.get_all_items(deleted_items);
parameter_t lambda_sum = 0;
int lambda_count = 0;
for( int i = 0; i < deleted_items->len; ++i ){
phrase_token_t * token = &g_array_index(deleted_items, phrase_token_t, i);
KMixtureModelSingleGram * single_gram = NULL;
bigram.load(*token, single_gram);
KMixtureModelSingleGram * deleted_single_gram = NULL;
deleted_bigram.load(*token, deleted_single_gram);
parameter_t lambda = compute_interpolation(deleted_single_gram, &bigram, single_gram);
printf("lambda:%f\n", lambda);
lambda_sum += lambda;
lambda_count ++;
if (single_gram)
delete single_gram;
delete deleted_single_gram;
}
printf("average lambda:%f\n", (lambda_sum/lambda_count));
g_array_free(deleted_items, TRUE);
return 0;
}
|
