diff options
Diffstat (limited to 'utils/training')
-rw-r--r-- | utils/training/Makefile.am | 36 | ||||
-rw-r--r-- | utils/training/estimate_interpolation.cpp | 146 | ||||
-rw-r--r-- | utils/training/gen_ngram.cpp | 158 | ||||
-rw-r--r-- | utils/training/gen_unigram.cpp | 71 |
4 files changed, 411 insertions, 0 deletions
diff --git a/utils/training/Makefile.am b/utils/training/Makefile.am new file mode 100644 index 0000000..520e4e1 --- /dev/null +++ b/utils/training/Makefile.am @@ -0,0 +1,36 @@ +## Makefile.am -- Process this file with automake to produce Makefile.in +## Copyright (C) 2007 Peng Wu +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 2, or (at your option) +## any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +MAINTAINERCLEANFILES = Makefile.in + +INCLUDES = -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/storage \ + @GLIB2_CPPFLAGS@ + +noinst_PROGRAMS = gen_ngram gen_unigram estimate_interpolation + +gen_ngram_SOURCES = gen_ngram.cpp + +gen_ngram_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@ + +gen_unigram_SOURCES = gen_unigram.cpp + +gen_unigram_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@ + +estimate_interpolation_SOURCES = estimate_interpolation.cpp + +estimate_interpolation_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@ diff --git a/utils/training/estimate_interpolation.cpp b/utils/training/estimate_interpolation.cpp new file mode 100644 index 0000000..aaae17b --- /dev/null +++ b/utils/training/estimate_interpolation.cpp @@ -0,0 +1,146 @@ +/* + * libpinyin + * Library to deal with pinyin. + * + * Copyright (C) 2006-2008 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <locale.h> +#include <math.h> +#include <glib.h> +#include "memory_chunk.h" +#include "novel_types.h" +#include "phrase_index.h" +#include "ngram.h" + +parameter_t compute_interpolation(SingleGram * deleted_bigram, + FacadePhraseIndex * unigram, + SingleGram * bigram){ + bool success; + parameter_t lambda = 0, next_lambda = 0.6; + parameter_t epsilon = 0.001; + + while ( fabs(lambda - next_lambda) > epsilon){ + lambda = next_lambda; + next_lambda = 0; + guint32 table_num = 0; + parameter_t numerator = 0; + parameter_t part_of_denominator = 0; + + BigramPhraseWithCountArray array = g_array_new(FALSE, FALSE, sizeof(BigramPhraseItemWithCount)); + deleted_bigram->retrieve_all(array); + + for ( int i = 0; i < array->len; ++i){ + BigramPhraseItemWithCount* item = &g_array_index(array, BigramPhraseItemWithCount, i); + //get the phrase token + phrase_token_t token = item->m_token; + guint32 deleted_freq = item->m_count; + + { + guint32 freq = 0; + parameter_t elem_poss = 0; + if ( bigram && bigram->get_freq(token, freq)){ + guint32 total_freq; + assert(bigram->get_total_freq(total_freq)); + assert(0 != total_freq); + elem_poss = freq / (parameter_t) total_freq; + } + numerator = lambda * elem_poss; + } + + { + guint32 freq = 0; + parameter_t elem_poss = 0; + PhraseItem item; + if (!unigram->get_phrase_item(token, item)){ + guint32 freq = item.get_unigram_frequency(); + guint32 total_freq = unigram->get_phrase_index_total_freq(); + elem_poss = freq / (parameter_t)total_freq; + } + part_of_denominator = ( 1 - lambda) * elem_poss; + } + + if ( 0 == (numerator + part_of_denominator)) + continue; + + next_lambda += deleted_freq * (numerator / (numerator + part_of_denominator)); + } + assert(deleted_bigram->get_total_freq(table_num)); + next_lambda /= table_num; + + g_array_free(array, TRUE); + } + lambda = next_lambda; + return lambda; +} + +int main(int argc, char * argv[]){ + FacadePhraseIndex phrase_index; + + //gb_char binary file + MemoryChunk * chunk = new MemoryChunk; + chunk->load("../../data/gb_char.bin"); + phrase_index.load(1, chunk); + + //gbk_char binary file + chunk = new MemoryChunk; + chunk->load("../../data/gbk_char.bin"); + phrase_index.load(2, chunk); + + Bigram bigram; + bigram.attach("../../data/bigram.db", NULL); + + Bigram deleted_bigram; + deleted_bigram.attach("../../data/deleted_bigram.db", NULL); + + GArray * system_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); + GArray * user_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); + + deleted_bigram.get_all_items(system_items, user_items); + assert(0 == user_items->len); + g_array_free(user_items, TRUE); + + parameter_t lambda_sum = 0; + int lambda_count = 0; + + for ( int i = 0; i < system_items->len; ++i ){ + phrase_token_t * token = &g_array_index(system_items, phrase_token_t, i); + SingleGram * system = NULL, * user = NULL; + bigram.load(*token, system, user); + assert(NULL == user); + SingleGram * deleted_system = NULL, * deleted_user = NULL; + deleted_bigram.load(*token, deleted_system, deleted_user); + assert(NULL == deleted_user); + + parameter_t lambda = compute_interpolation(deleted_system, &phrase_index, system); + + printf("lambda:%f\n", lambda); + + lambda_sum += lambda; + lambda_count ++; + + if (system) delete system; + delete deleted_system; + } + + printf("average lambda:%f\n", (lambda_sum/lambda_count)); + g_array_free(system_items, TRUE); +} + diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp new file mode 100644 index 0000000..a1f5b6d --- /dev/null +++ b/utils/training/gen_ngram.cpp @@ -0,0 +1,158 @@ +/* + * libpinyin + * Library to deal with pinyin. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <locale.h> +#include <glib.h> +#include "memory_chunk.h" +#include "novel_types.h" +#include "phrase_index.h" +#include "ngram.h" +#include "phrase_large_table.h" + + +static PhraseLargeTable * g_phrases = NULL; + +void print_help(){ + printf("gen_ngram [--skip-pi-gram-training] [--skip-unigram-training]\n"); + printf(" [--bigram-file <FILENAME>]\n"); + exit(1); +} + +int main(int argc, char * argv[]){ + int i = 1; + bool train_pi_gram = true; + bool train_unigram = true; + const char * bigram_filename = "../../data/bigram.db"; + + setlocale(LC_ALL,""); + while ( i < argc ){ + if ( strcmp("--help", argv[i] ) == 0){ + print_help(); + }else if ( strcmp("--skip-pi-gram-training", argv[i] ) == 0) { + train_pi_gram = false; + }else if ( strcmp("--skip-unigram-training", argv[i] ) == 0) { + train_unigram = false; + }else if ( strcmp("--bigram-file", argv[i] ) == 0){ + if ( ++i >= argc ) + print_help(); + bigram_filename = argv[i]; + } + ++i; + } + + g_phrases = new PhraseLargeTable; + //init phrase lookup + FILE * gb_file = fopen("../../data/gb_char.table", "r"); + if ( gb_file == NULL ){ + fprintf(stderr, "can't open gb_char.table!\n"); + exit(1); + } + g_phrases->load_text(gb_file); + fclose(gb_file); + FILE * gbk_file = fopen("../../data/gbk_char.table", "r"); + if ( gbk_file == NULL ){ + fprintf(stderr, "can't open gbk_char.table!\n"); + exit(1); + } + g_phrases->load_text(gbk_file); + fclose(gbk_file); + + FacadePhraseIndex phrase_index; + + //gb_char binary file + MemoryChunk * chunk = new MemoryChunk; + chunk->load("../../data/gb_char.bin"); + phrase_index.load(1, chunk); + + //gbk_char binary file + chunk = new MemoryChunk; + chunk->load("../../data/gbk_char.bin"); + phrase_index.load(2, chunk); + + Bigram bigram; + bigram.attach(NULL, bigram_filename); + + + char* linebuf = (char *)malloc ( 1024 * sizeof (char) ); + size_t size = 1024; + phrase_token_t last_token, cur_token = last_token = 0; + while( getline(&linebuf, &size, stdin) ){ + if ( feof(stdin) ) + break; + linebuf[strlen(linebuf)-1] = '\0'; + + glong phrase_len = 0; + utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); + + phrase_token_t token = 0; + int result = g_phrases->search( phrase_len, phrase, token); + if ( ! (result & SEARCH_OK) ) + token = 0; + + last_token = cur_token; + cur_token = token; + if ( cur_token ){ + //training uni-gram + if ( train_unigram ) + phrase_index.add_unigram_frequency(cur_token, 1); + } + if ( cur_token ){ + SingleGram * system = NULL, * user = NULL; + if ( 0 == last_token ){ + if (train_pi_gram) + bigram.load(sentence_start, system, user); + } else + bigram.load(last_token, system, user); + assert(NULL == system); + if ( NULL == user ){ + user = new SingleGram; + } + guint32 freq, total_freq; + //increase freq + user->get_freq(cur_token, freq); + user->set_freq(cur_token, freq + 1); + //increase total freq + user->get_total_freq(total_freq); + user->set_total_freq(total_freq + 1); + if ( 0 == last_token ){ + if ( train_pi_gram ) + bigram.store(sentence_start, user); + }else + bigram.store(last_token, user); + delete user; + } + } + + MemoryChunk * new_chunk = new MemoryChunk; + phrase_index.store(1, new_chunk); + new_chunk->save("../../data/gb_char.bin"); + phrase_index.load(1, new_chunk); + + new_chunk = new MemoryChunk; + phrase_index.store(2, new_chunk); + new_chunk->save("../../data/gbk_char.bin"); + phrase_index.load(2, new_chunk); + + return 0; +} diff --git a/utils/training/gen_unigram.cpp b/utils/training/gen_unigram.cpp new file mode 100644 index 0000000..1c70665 --- /dev/null +++ b/utils/training/gen_unigram.cpp @@ -0,0 +1,71 @@ +/* + * libpinyin + * Library to deal with pinyin. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <stdio.h> +#include "memory_chunk.h" +#include "novel_types.h" +#include "phrase_index.h" + +//increase all unigram frequency by one. + +int main(int argc, char * argv[]){ + + FacadePhraseIndex phrase_index; + + //gb_char binary file + MemoryChunk * chunk = new MemoryChunk; + chunk->load("../../data/gb_char.bin"); + phrase_index.load(1, chunk); + + //gbk_char binary file + chunk = new MemoryChunk; + chunk->load("../../data/gbk_char.bin"); + phrase_index.load(2, chunk); + + PhraseIndexRange range; + int result = phrase_index.get_range(1, range); + if ( result == ERROR_OK ) { + for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){ + phrase_index.add_unigram_frequency(i, 1); + } + } + +#if 0 + int result = phrase_index.get_range(2, range); + if ( result == ERROR_OK ) { + for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){ + phrase_index.add_unigram_frequency(i, 1); + } + } +#endif + + MemoryChunk * new_chunk = new MemoryChunk; + phrase_index.store(1, new_chunk); + new_chunk->save("../../data/gb_char.bin"); + phrase_index.load(1, new_chunk); + + new_chunk = new MemoryChunk; + phrase_index.store(2, new_chunk); + new_chunk->save("../../data/gbk_char.bin"); + phrase_index.load(2, new_chunk); + + return 0; +} |