diff options
author | Peng Wu <alexepico@gmail.com> | 2010-10-27 11:12:41 +0800 |
---|---|---|
committer | Peng Wu <alexepico@gmail.com> | 2010-10-27 11:12:41 +0800 |
commit | e9e32f965272d5465b81d1ea62df3a7e3f97f134 (patch) | |
tree | b10d03cc6999dbbe7873486a242cd2c68f7fd0b1 /src | |
parent | cb42269587055a80907efd863c367d659a022a06 (diff) | |
download | libpinyin-e9e32f965272d5465b81d1ea62df3a7e3f97f134.tar.gz libpinyin-e9e32f965272d5465b81d1ea62df3a7e3f97f134.tar.xz libpinyin-e9e32f965272d5465b81d1ea62df3a7e3f97f134.zip |
move segment and training tools to utils
Diffstat (limited to 'src')
-rw-r--r-- | src/segment/Makefile.am | 28 | ||||
-rw-r--r-- | src/segment/spseg.cpp | 209 | ||||
-rw-r--r-- | src/training/Makefile.am | 36 | ||||
-rw-r--r-- | src/training/estimate_interpolation.cpp | 146 | ||||
-rw-r--r-- | src/training/gen_ngram.cpp | 158 | ||||
-rw-r--r-- | src/training/gen_unigram.cpp | 71 |
6 files changed, 0 insertions, 648 deletions
diff --git a/src/segment/Makefile.am b/src/segment/Makefile.am deleted file mode 100644 index a82627a..0000000 --- a/src/segment/Makefile.am +++ /dev/null @@ -1,28 +0,0 @@ -## Makefile.am -- Process this file with automake to produce Makefile.in -## Copyright (C) 2007 Peng Wu -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2, or (at your option) -## any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -MAINTAINERCLEANFILES = Makefile.in - -INCLUDES = -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/src/storage \ - @GLIB2_CPPFLAGS@ - -noinst_PROGRAMS = spseg - -spseg_SOURCES = spseg.cpp - -spseg_LDADD = @GLIB2_LDFLAGS@ ../storage/libstorage.la diff --git a/src/segment/spseg.cpp b/src/segment/spseg.cpp deleted file mode 100644 index f4522f2..0000000 --- a/src/segment/spseg.cpp +++ /dev/null @@ -1,209 +0,0 @@ -/* - * libpinyin - * Library to deal with pinyin. - * - * Copyright (C) 2010 Peng Wu - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include <stdio.h> -#include <string.h> -#include <locale.h> -#include <glib.h> -#include "novel_types.h" -#include "phrase_large_table.h" - -/* graph shortest path sentence segment. */ - -/* Note: - * Currently libpinyin only supports ucs2 characters, as this is a - * pre-processor tool for raw corpus, it will skip all sentences - * which contains non-ucs2 characters. - */ - -static PhraseLargeTable * g_phrases = NULL; - -struct SegmentStep{ - phrase_token_t m_handle; - utf16_t * m_phrase; - size_t m_phrase_len; - //use formula W = number of words. Zero handle means one word. - guint m_nword; - //backtrace information, -1 one step backward. - gint m_backward_nstep; -public: - SegmentStep(){ - m_handle = 0; - m_phrase = NULL; - m_phrase_len = 0; - m_nword = UINT_MAX; - m_backward_nstep = -0; - } -}; - -bool backtrace(GArray * steps, glong phrase_len, GArray * strings); - -//Note: do not free phrase, as it is used by strings (array of segment). -bool segment(PhraseLargeTable * phrases, //Lookup Phrase - utf16_t * phrase, - glong phrase_len, - GArray * strings /* Array of Segment *. */){ - /* Prepare for shortest path segment dynamic programming. */ - GArray * steps = g_array_new(TRUE, TRUE, sizeof(SegmentStep)); - SegmentStep step; - for ( glong i = 0; i < phrase_len + 1; ++i ){ - g_array_append_val(steps, step); - } - - SegmentStep * first_step = &g_array_index(steps, SegmentStep, 0); - first_step->m_nword = 0; - - for ( glong i = 0; i < phrase_len + 1; ++i ) { - SegmentStep * step_begin = &g_array_index(steps, SegmentStep, i); - size_t nword = step_begin->m_nword; - for ( glong k = i + 1; k < phrase_len + 1; ++k ) { - size_t len = k - i; - utf16_t * cur_phrase = phrase + i; - - phrase_token_t token = 0; - int result = g_phrases->search(len, cur_phrase, token); - if ( !(result & SEARCH_OK) ){ - token = 0; - if ( 1 != len ) - continue; - } - ++nword; - SegmentStep * step_end = &g_array_index(steps, SegmentStep, k); - if ( nword < step_end->m_nword ) { - step_end->m_handle = token; - step_end->m_phrase = cur_phrase; - step_end->m_phrase_len = len; - step_end->m_nword = nword; - step_end->m_backward_nstep = i - k; - } - if ( !(result & SEARCH_CONTINUED) ) - break; - } - } - return backtrace(steps, phrase_len, strings); -} - -bool backtrace(GArray * steps, glong phrase_len, GArray * strings){ - //backtracing to get the result. - size_t cur_step = phrase_len; - g_array_set_size(strings, 0); - while ( cur_step ){ - SegmentStep * step = &g_array_index(steps, SegmentStep, cur_step); - g_array_append_val(strings, *step); - cur_step = cur_step + step->m_backward_nstep; - //intended to avoid leaking internal informations - step->m_nword = 0; step->m_backward_nstep = 0; - } - - //reverse the strings - for ( size_t i = 0; i < strings->len / 2; ++i ) { - SegmentStep * head, * tail; - head = &g_array_index(strings, SegmentStep, i); - tail = &g_array_index(strings, SegmentStep, strings->len - 1 - i ); - SegmentStep tmp; - tmp = *head; - *head = *tail; - *tail = tmp; - } - - g_array_free(steps, TRUE); - return true; -} - -void print_help(){ - printf("Usage: mmseg [--generate-extra-enter]\n"); - exit(1); -} - -int main(int argc, char * argv[]){ - int i = 1; - bool gen_extra_enter = false; - - setlocale(LC_ALL, ""); - //deal with options. - while ( i < argc ){ - if ( strcmp ("--help", argv[i]) == 0) { - print_help(); - }else if (strcmp("--generate-extra-enter", argv[i]) == 0) { - gen_extra_enter = true; - } - ++i; - } - - //init phrase table - g_phrases = new PhraseLargeTable; - FILE * gb_file = fopen("../../data/gb_char.table", "r"); - if ( gb_file == NULL ){ - fprintf(stderr, "can't open gb_char.table!\n"); - exit(1); - } - g_phrases->load_text(gb_file); - fclose(gb_file); - - FILE * gbk_file = fopen("../../data/gbk_char.table", "r"); - if ( gbk_file == NULL ){ - fprintf(stderr, "can't open gbk_char.table!\n"); - exit(1); - } - g_phrases->load_text(gbk_file); - fclose(gbk_file); - - MemoryChunk * chunk = new MemoryChunk; - g_phrases->store(chunk); - g_phrases->load(chunk); - - char * linebuf = NULL; - size_t size = 0; - while(getline(&linebuf, &size, stdin)) { - if ( feof(stdin) ) - break; - linebuf[strlen(linebuf) - 1] = '\0'; - - //check non-ucs2 characters - const glong num_of_chars = g_utf8_strlen(linebuf, -1); - glong len = 0; - utf16_t * sentence = g_utf8_to_utf16(linebuf, -1, NULL, &len, NULL); - if ( len != num_of_chars ) { - fprintf(stderr, "non-ucs2 characters encountered:%s.\n", linebuf); - continue; - } - - //do segment stuff - GArray * strings = g_array_new(TRUE, TRUE, sizeof(SegmentStep)); - segment(g_phrases, sentence, len, strings); - - //print out the split phrase - for ( glong i = 0; i < strings->len; ++i ) { - SegmentStep * step = &g_array_index(strings, SegmentStep, i); - char * string = g_utf16_to_utf8( step->m_phrase, step->m_phrase_len, NULL, NULL, NULL); - printf("%s\n", string); - g_free(string); - } - - //print extra enter - if ( gen_extra_enter ) - printf("\n"); - - g_array_free(strings, TRUE); - g_free(sentence); - } - return 0; -} diff --git a/src/training/Makefile.am b/src/training/Makefile.am deleted file mode 100644 index 520e4e1..0000000 --- a/src/training/Makefile.am +++ /dev/null @@ -1,36 +0,0 @@ -## Makefile.am -- Process this file with automake to produce Makefile.in -## Copyright (C) 2007 Peng Wu -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2, or (at your option) -## any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -MAINTAINERCLEANFILES = Makefile.in - -INCLUDES = -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/src/storage \ - @GLIB2_CPPFLAGS@ - -noinst_PROGRAMS = gen_ngram gen_unigram estimate_interpolation - -gen_ngram_SOURCES = gen_ngram.cpp - -gen_ngram_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@ - -gen_unigram_SOURCES = gen_unigram.cpp - -gen_unigram_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@ - -estimate_interpolation_SOURCES = estimate_interpolation.cpp - -estimate_interpolation_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@ diff --git a/src/training/estimate_interpolation.cpp b/src/training/estimate_interpolation.cpp deleted file mode 100644 index aaae17b..0000000 --- a/src/training/estimate_interpolation.cpp +++ /dev/null @@ -1,146 +0,0 @@ -/* - * libpinyin - * Library to deal with pinyin. - * - * Copyright (C) 2006-2008 Peng Wu - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <locale.h> -#include <math.h> -#include <glib.h> -#include "memory_chunk.h" -#include "novel_types.h" -#include "phrase_index.h" -#include "ngram.h" - -parameter_t compute_interpolation(SingleGram * deleted_bigram, - FacadePhraseIndex * unigram, - SingleGram * bigram){ - bool success; - parameter_t lambda = 0, next_lambda = 0.6; - parameter_t epsilon = 0.001; - - while ( fabs(lambda - next_lambda) > epsilon){ - lambda = next_lambda; - next_lambda = 0; - guint32 table_num = 0; - parameter_t numerator = 0; - parameter_t part_of_denominator = 0; - - BigramPhraseWithCountArray array = g_array_new(FALSE, FALSE, sizeof(BigramPhraseItemWithCount)); - deleted_bigram->retrieve_all(array); - - for ( int i = 0; i < array->len; ++i){ - BigramPhraseItemWithCount* item = &g_array_index(array, BigramPhraseItemWithCount, i); - //get the phrase token - phrase_token_t token = item->m_token; - guint32 deleted_freq = item->m_count; - - { - guint32 freq = 0; - parameter_t elem_poss = 0; - if ( bigram && bigram->get_freq(token, freq)){ - guint32 total_freq; - assert(bigram->get_total_freq(total_freq)); - assert(0 != total_freq); - elem_poss = freq / (parameter_t) total_freq; - } - numerator = lambda * elem_poss; - } - - { - guint32 freq = 0; - parameter_t elem_poss = 0; - PhraseItem item; - if (!unigram->get_phrase_item(token, item)){ - guint32 freq = item.get_unigram_frequency(); - guint32 total_freq = unigram->get_phrase_index_total_freq(); - elem_poss = freq / (parameter_t)total_freq; - } - part_of_denominator = ( 1 - lambda) * elem_poss; - } - - if ( 0 == (numerator + part_of_denominator)) - continue; - - next_lambda += deleted_freq * (numerator / (numerator + part_of_denominator)); - } - assert(deleted_bigram->get_total_freq(table_num)); - next_lambda /= table_num; - - g_array_free(array, TRUE); - } - lambda = next_lambda; - return lambda; -} - -int main(int argc, char * argv[]){ - FacadePhraseIndex phrase_index; - - //gb_char binary file - MemoryChunk * chunk = new MemoryChunk; - chunk->load("../../data/gb_char.bin"); - phrase_index.load(1, chunk); - - //gbk_char binary file - chunk = new MemoryChunk; - chunk->load("../../data/gbk_char.bin"); - phrase_index.load(2, chunk); - - Bigram bigram; - bigram.attach("../../data/bigram.db", NULL); - - Bigram deleted_bigram; - deleted_bigram.attach("../../data/deleted_bigram.db", NULL); - - GArray * system_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); - GArray * user_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); - - deleted_bigram.get_all_items(system_items, user_items); - assert(0 == user_items->len); - g_array_free(user_items, TRUE); - - parameter_t lambda_sum = 0; - int lambda_count = 0; - - for ( int i = 0; i < system_items->len; ++i ){ - phrase_token_t * token = &g_array_index(system_items, phrase_token_t, i); - SingleGram * system = NULL, * user = NULL; - bigram.load(*token, system, user); - assert(NULL == user); - SingleGram * deleted_system = NULL, * deleted_user = NULL; - deleted_bigram.load(*token, deleted_system, deleted_user); - assert(NULL == deleted_user); - - parameter_t lambda = compute_interpolation(deleted_system, &phrase_index, system); - - printf("lambda:%f\n", lambda); - - lambda_sum += lambda; - lambda_count ++; - - if (system) delete system; - delete deleted_system; - } - - printf("average lambda:%f\n", (lambda_sum/lambda_count)); - g_array_free(system_items, TRUE); -} - diff --git a/src/training/gen_ngram.cpp b/src/training/gen_ngram.cpp deleted file mode 100644 index a1f5b6d..0000000 --- a/src/training/gen_ngram.cpp +++ /dev/null @@ -1,158 +0,0 @@ -/* - * libpinyin - * Library to deal with pinyin. - * - * Copyright (C) 2006-2007 Peng Wu - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <locale.h> -#include <glib.h> -#include "memory_chunk.h" -#include "novel_types.h" -#include "phrase_index.h" -#include "ngram.h" -#include "phrase_large_table.h" - - -static PhraseLargeTable * g_phrases = NULL; - -void print_help(){ - printf("gen_ngram [--skip-pi-gram-training] [--skip-unigram-training]\n"); - printf(" [--bigram-file <FILENAME>]\n"); - exit(1); -} - -int main(int argc, char * argv[]){ - int i = 1; - bool train_pi_gram = true; - bool train_unigram = true; - const char * bigram_filename = "../../data/bigram.db"; - - setlocale(LC_ALL,""); - while ( i < argc ){ - if ( strcmp("--help", argv[i] ) == 0){ - print_help(); - }else if ( strcmp("--skip-pi-gram-training", argv[i] ) == 0) { - train_pi_gram = false; - }else if ( strcmp("--skip-unigram-training", argv[i] ) == 0) { - train_unigram = false; - }else if ( strcmp("--bigram-file", argv[i] ) == 0){ - if ( ++i >= argc ) - print_help(); - bigram_filename = argv[i]; - } - ++i; - } - - g_phrases = new PhraseLargeTable; - //init phrase lookup - FILE * gb_file = fopen("../../data/gb_char.table", "r"); - if ( gb_file == NULL ){ - fprintf(stderr, "can't open gb_char.table!\n"); - exit(1); - } - g_phrases->load_text(gb_file); - fclose(gb_file); - FILE * gbk_file = fopen("../../data/gbk_char.table", "r"); - if ( gbk_file == NULL ){ - fprintf(stderr, "can't open gbk_char.table!\n"); - exit(1); - } - g_phrases->load_text(gbk_file); - fclose(gbk_file); - - FacadePhraseIndex phrase_index; - - //gb_char binary file - MemoryChunk * chunk = new MemoryChunk; - chunk->load("../../data/gb_char.bin"); - phrase_index.load(1, chunk); - - //gbk_char binary file - chunk = new MemoryChunk; - chunk->load("../../data/gbk_char.bin"); - phrase_index.load(2, chunk); - - Bigram bigram; - bigram.attach(NULL, bigram_filename); - - - char* linebuf = (char *)malloc ( 1024 * sizeof (char) ); - size_t size = 1024; - phrase_token_t last_token, cur_token = last_token = 0; - while( getline(&linebuf, &size, stdin) ){ - if ( feof(stdin) ) - break; - linebuf[strlen(linebuf)-1] = '\0'; - - glong phrase_len = 0; - utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL); - - phrase_token_t token = 0; - int result = g_phrases->search( phrase_len, phrase, token); - if ( ! (result & SEARCH_OK) ) - token = 0; - - last_token = cur_token; - cur_token = token; - if ( cur_token ){ - //training uni-gram - if ( train_unigram ) - phrase_index.add_unigram_frequency(cur_token, 1); - } - if ( cur_token ){ - SingleGram * system = NULL, * user = NULL; - if ( 0 == last_token ){ - if (train_pi_gram) - bigram.load(sentence_start, system, user); - } else - bigram.load(last_token, system, user); - assert(NULL == system); - if ( NULL == user ){ - user = new SingleGram; - } - guint32 freq, total_freq; - //increase freq - user->get_freq(cur_token, freq); - user->set_freq(cur_token, freq + 1); - //increase total freq - user->get_total_freq(total_freq); - user->set_total_freq(total_freq + 1); - if ( 0 == last_token ){ - if ( train_pi_gram ) - bigram.store(sentence_start, user); - }else - bigram.store(last_token, user); - delete user; - } - } - - MemoryChunk * new_chunk = new MemoryChunk; - phrase_index.store(1, new_chunk); - new_chunk->save("../../data/gb_char.bin"); - phrase_index.load(1, new_chunk); - - new_chunk = new MemoryChunk; - phrase_index.store(2, new_chunk); - new_chunk->save("../../data/gbk_char.bin"); - phrase_index.load(2, new_chunk); - - return 0; -} diff --git a/src/training/gen_unigram.cpp b/src/training/gen_unigram.cpp deleted file mode 100644 index 1c70665..0000000 --- a/src/training/gen_unigram.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * libpinyin - * Library to deal with pinyin. - * - * Copyright (C) 2006-2007 Peng Wu - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include <stdio.h> -#include "memory_chunk.h" -#include "novel_types.h" -#include "phrase_index.h" - -//increase all unigram frequency by one. - -int main(int argc, char * argv[]){ - - FacadePhraseIndex phrase_index; - - //gb_char binary file - MemoryChunk * chunk = new MemoryChunk; - chunk->load("../../data/gb_char.bin"); - phrase_index.load(1, chunk); - - //gbk_char binary file - chunk = new MemoryChunk; - chunk->load("../../data/gbk_char.bin"); - phrase_index.load(2, chunk); - - PhraseIndexRange range; - int result = phrase_index.get_range(1, range); - if ( result == ERROR_OK ) { - for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){ - phrase_index.add_unigram_frequency(i, 1); - } - } - -#if 0 - int result = phrase_index.get_range(2, range); - if ( result == ERROR_OK ) { - for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){ - phrase_index.add_unigram_frequency(i, 1); - } - } -#endif - - MemoryChunk * new_chunk = new MemoryChunk; - phrase_index.store(1, new_chunk); - new_chunk->save("../../data/gb_char.bin"); - phrase_index.load(1, new_chunk); - - new_chunk = new MemoryChunk; - phrase_index.store(2, new_chunk); - new_chunk->save("../../data/gbk_char.bin"); - phrase_index.load(2, new_chunk); - - return 0; -} |