diff options
author | Peng Wu <alexepico@gmail.com> | 2011-05-05 13:11:50 +0800 |
---|---|---|
committer | Peng Wu <alexepico@gmail.com> | 2011-05-05 13:11:50 +0800 |
commit | 5b972580775a1eeb1683b73d5ee6a9126e8271a5 (patch) | |
tree | fe44a5cfce98e750590c65fc8996b3bd571dfefe | |
parent | 1326788b7e88051375c54b9eef5f3d19457a7db4 (diff) | |
download | libpinyin-5b972580775a1eeb1683b73d5ee6a9126e8271a5.tar.gz libpinyin-5b972580775a1eeb1683b73d5ee6a9126e8271a5.tar.xz libpinyin-5b972580775a1eeb1683b73d5ee6a9126e8271a5.zip |
refine gen ngram
-rw-r--r-- | utils/training/Makefile.am | 5 | ||||
-rw-r--r-- | utils/training/gen_deleted_ngram.cpp | 132 | ||||
-rw-r--r-- | utils/training/gen_ngram.cpp | 10 |
3 files changed, 140 insertions, 7 deletions
diff --git a/utils/training/Makefile.am b/utils/training/Makefile.am index 661a8f6..fd51670 100644 --- a/utils/training/Makefile.am +++ b/utils/training/Makefile.am @@ -26,6 +26,7 @@ INCLUDES = -I$(top_srcdir)/src \ noinst_HEADERS = k_mixture_model.h noinst_PROGRAMS = gen_ngram \ + gen_deleted_ngram \ gen_unigram \ estimate_interpolation \ estimate_k_mixture_model \ @@ -35,6 +36,10 @@ gen_ngram_SOURCES = gen_ngram.cpp gen_ngram_LDADD = ../../src/libpinyin.la @GLIB2_LDFLAGS@ +gen_deleted_ngram_SOURCES = gen_deleted_ngram.cpp + +gen_deleted_ngram_LDADD = ../../src/libpinyin.la @GLIB2_LDFLAGS@ + gen_unigram_SOURCES = gen_unigram.cpp gen_unigram_LDADD = ../../src/libpinyin.la @GLIB2_LDFLAGS@ diff --git a/utils/training/gen_deleted_ngram.cpp b/utils/training/gen_deleted_ngram.cpp new file mode 100644 index 0000000..229eb73 --- /dev/null +++ b/utils/training/gen_deleted_ngram.cpp @@ -0,0 +1,132 @@ +/* + * libpinyin + * Library to deal with pinyin. + * + * Copyright (C) 2006-2007, 2011 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <locale.h> +#include <glib.h> +#include "pinyin.h" + +static PhraseLargeTable * g_phrases = NULL; + +void print_help(){ + printf("gen_deleted_ngram [--skip-pi-gram-training]\n"); + printf(" [--bigram-file <FILENAME>]\n"); +} + +int main(int argc, char * argv[]){ + int i = 1; + bool train_pi_gram = true; + const char * bigram_filename = "../../data/bigram.db"; + + setlocale(LC_ALL, ""); + while ( i < argc ){ + if ( strcmp("--help", argv[i]) == 0){ + print_help(); + exit(0); + }else if ( strcmp("--skip-pi-gram-training", argv[i]) == 0 ){ + train_pi_gram = false; + }else if ( strcmp("--bigram-file", argv[i]) == 0){ + if ( ++i >= argc ) { + print_help(); + exit(EINVAL); + } + bigram_filename = argv[i]; + }else{ + print_help(); + exit(EINVAL); + } + ++i; + } + + g_phrases = new PhraseLargeTable; + //init phrase lookup + FILE * gb_file = fopen("../../data/gb_char.table", "r"); + if ( gb_file == NULL ){ + fprintf(stderr, "can't open gb_char.table!\n"); + exit(ENOENT); + } + g_phrases->load_text(gb_file); + fclose(gb_file); + FILE * gbk_file = fopen("../../data/gbk_char.table", "r"); + if ( gbk_file == NULL ){ + fprintf(stderr, "can't open gbk_char.table!\n"); + exit(ENOENT); + } + g_phrases->load_text(gbk_file); + fclose(gbk_file); + + Bigram bigram; + bigram.attach(bigram_filename, ATTACH_CREATE|ATTACH_READWRITE); + + char* linebuf = (char *)malloc ( 1024 * sizeof (char) ); + size_t size = 1024; + phrase_token_t last_token, cur_token = last_token = 0; + while( getline(&linebuf, &size, stdin) ){ + if ( feof(stdin) ) + break; + linebuf[strlen(linebuf)-1] = '\0'; + + glong phrase_len = 0; + utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); + + if ( phrase_len == 0 ) + continue; + + phrase_token_t token = 0; + int result = g_phrases->search( phrase_len, phrase, token); + if ( ! (result & SEARCH_OK) ) + token = 0; + + last_token = cur_token; + cur_token = token; + if ( cur_token ){ + SingleGram * single_gram = NULL; + if ( 0 == last_token ){ + if (train_pi_gram) + bigram.load(sentence_start, single_gram); + } else + bigram.load(last_token, single_gram); + + if ( NULL == single_gram ){ + single_gram = new SingleGram; + } + guint32 freq, total_freq; + //increase freq + if (single_gram->get_freq(cur_token, freq)) + assert(single_gram->set_freq(cur_token, freq + 1)); + else + assert(single_gram->insert_freq(cur_token, 1)); + //increase total freq + single_gram->get_total_freq(total_freq); + single_gram->set_total_freq(total_freq + 1); + if ( 0 == last_token ){ + if ( train_pi_gram ) + bigram.store(sentence_start, single_gram); + }else + bigram.store(last_token, single_gram); + delete single_gram; + } + } + + return 0; +} diff --git a/utils/training/gen_ngram.cpp b/utils/training/gen_ngram.cpp index 9db5d58..2ade331 100644 --- a/utils/training/gen_ngram.cpp +++ b/utils/training/gen_ngram.cpp @@ -2,7 +2,7 @@ * libpinyin * Library to deal with pinyin. * - * Copyright (C) 2006-2007 Peng Wu + * Copyright (C) 2006-2007, 2011 Peng Wu * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,14 +29,13 @@ static PhraseLargeTable * g_phrases = NULL; void print_help(){ - printf("gen_ngram [--skip-pi-gram-training] [--skip-unigram-training]\n"); + printf("gen_ngram [--skip-pi-gram-training]\n"); printf(" [--bigram-file <FILENAME>]\n"); } int main(int argc, char * argv[]){ int i = 1; bool train_pi_gram = true; - bool train_unigram = true; const char * bigram_filename = "../../data/bigram.db"; setlocale(LC_ALL, ""); @@ -46,8 +45,6 @@ int main(int argc, char * argv[]){ exit(0); }else if ( strcmp("--skip-pi-gram-training", argv[i]) == 0 ){ train_pi_gram = false; - }else if ( strcmp("--skip-unigram-training", argv[i]) == 0 ){ - train_unigram = false; }else if ( strcmp("--bigram-file", argv[i]) == 0){ if ( ++i >= argc ) { print_help(); @@ -117,8 +114,7 @@ int main(int argc, char * argv[]){ cur_token = token; if ( cur_token ){ //training uni-gram - if ( train_unigram ) - phrase_index.add_unigram_frequency(cur_token, 1); + phrase_index.add_unigram_frequency(cur_token, 1); } if ( cur_token ){ SingleGram * single_gram = NULL; |