summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorPeng Wu <alexepico@gmail.com>2010-10-27 11:12:41 +0800
committerPeng Wu <alexepico@gmail.com>2010-10-27 11:12:41 +0800
commite9e32f965272d5465b81d1ea62df3a7e3f97f134 (patch)
treeb10d03cc6999dbbe7873486a242cd2c68f7fd0b1 /src
parentcb42269587055a80907efd863c367d659a022a06 (diff)
downloadlibpinyin-e9e32f965272d5465b81d1ea62df3a7e3f97f134.tar.gz
libpinyin-e9e32f965272d5465b81d1ea62df3a7e3f97f134.tar.xz
libpinyin-e9e32f965272d5465b81d1ea62df3a7e3f97f134.zip
move segment and training tools to utils
Diffstat (limited to 'src')
-rw-r--r--src/segment/Makefile.am28
-rw-r--r--src/segment/spseg.cpp209
-rw-r--r--src/training/Makefile.am36
-rw-r--r--src/training/estimate_interpolation.cpp146
-rw-r--r--src/training/gen_ngram.cpp158
-rw-r--r--src/training/gen_unigram.cpp71
6 files changed, 0 insertions, 648 deletions
diff --git a/src/segment/Makefile.am b/src/segment/Makefile.am
deleted file mode 100644
index a82627a..0000000
--- a/src/segment/Makefile.am
+++ /dev/null
@@ -1,28 +0,0 @@
-## Makefile.am -- Process this file with automake to produce Makefile.in
-## Copyright (C) 2007 Peng Wu
-##
-## This program is free software; you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 2, or (at your option)
-## any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-## GNU General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with this program; if not, write to the Free Software
-## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-MAINTAINERCLEANFILES = Makefile.in
-
-INCLUDES = -I$(top_srcdir)/src/include \
- -I$(top_srcdir)/src/storage \
- @GLIB2_CPPFLAGS@
-
-noinst_PROGRAMS = spseg
-
-spseg_SOURCES = spseg.cpp
-
-spseg_LDADD = @GLIB2_LDFLAGS@ ../storage/libstorage.la
diff --git a/src/segment/spseg.cpp b/src/segment/spseg.cpp
deleted file mode 100644
index f4522f2..0000000
--- a/src/segment/spseg.cpp
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * libpinyin
- * Library to deal with pinyin.
- *
- * Copyright (C) 2010 Peng Wu
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-
-#include <stdio.h>
-#include <string.h>
-#include <locale.h>
-#include <glib.h>
-#include "novel_types.h"
-#include "phrase_large_table.h"
-
-/* graph shortest path sentence segment. */
-
-/* Note:
- * Currently libpinyin only supports ucs2 characters, as this is a
- * pre-processor tool for raw corpus, it will skip all sentences
- * which contains non-ucs2 characters.
- */
-
-static PhraseLargeTable * g_phrases = NULL;
-
-struct SegmentStep{
- phrase_token_t m_handle;
- utf16_t * m_phrase;
- size_t m_phrase_len;
- //use formula W = number of words. Zero handle means one word.
- guint m_nword;
- //backtrace information, -1 one step backward.
- gint m_backward_nstep;
-public:
- SegmentStep(){
- m_handle = 0;
- m_phrase = NULL;
- m_phrase_len = 0;
- m_nword = UINT_MAX;
- m_backward_nstep = -0;
- }
-};
-
-bool backtrace(GArray * steps, glong phrase_len, GArray * strings);
-
-//Note: do not free phrase, as it is used by strings (array of segment).
-bool segment(PhraseLargeTable * phrases, //Lookup Phrase
- utf16_t * phrase,
- glong phrase_len,
- GArray * strings /* Array of Segment *. */){
- /* Prepare for shortest path segment dynamic programming. */
- GArray * steps = g_array_new(TRUE, TRUE, sizeof(SegmentStep));
- SegmentStep step;
- for ( glong i = 0; i < phrase_len + 1; ++i ){
- g_array_append_val(steps, step);
- }
-
- SegmentStep * first_step = &g_array_index(steps, SegmentStep, 0);
- first_step->m_nword = 0;
-
- for ( glong i = 0; i < phrase_len + 1; ++i ) {
- SegmentStep * step_begin = &g_array_index(steps, SegmentStep, i);
- size_t nword = step_begin->m_nword;
- for ( glong k = i + 1; k < phrase_len + 1; ++k ) {
- size_t len = k - i;
- utf16_t * cur_phrase = phrase + i;
-
- phrase_token_t token = 0;
- int result = g_phrases->search(len, cur_phrase, token);
- if ( !(result & SEARCH_OK) ){
- token = 0;
- if ( 1 != len )
- continue;
- }
- ++nword;
- SegmentStep * step_end = &g_array_index(steps, SegmentStep, k);
- if ( nword < step_end->m_nword ) {
- step_end->m_handle = token;
- step_end->m_phrase = cur_phrase;
- step_end->m_phrase_len = len;
- step_end->m_nword = nword;
- step_end->m_backward_nstep = i - k;
- }
- if ( !(result & SEARCH_CONTINUED) )
- break;
- }
- }
- return backtrace(steps, phrase_len, strings);
-}
-
-bool backtrace(GArray * steps, glong phrase_len, GArray * strings){
- //backtracing to get the result.
- size_t cur_step = phrase_len;
- g_array_set_size(strings, 0);
- while ( cur_step ){
- SegmentStep * step = &g_array_index(steps, SegmentStep, cur_step);
- g_array_append_val(strings, *step);
- cur_step = cur_step + step->m_backward_nstep;
- //intended to avoid leaking internal informations
- step->m_nword = 0; step->m_backward_nstep = 0;
- }
-
- //reverse the strings
- for ( size_t i = 0; i < strings->len / 2; ++i ) {
- SegmentStep * head, * tail;
- head = &g_array_index(strings, SegmentStep, i);
- tail = &g_array_index(strings, SegmentStep, strings->len - 1 - i );
- SegmentStep tmp;
- tmp = *head;
- *head = *tail;
- *tail = tmp;
- }
-
- g_array_free(steps, TRUE);
- return true;
-}
-
-void print_help(){
- printf("Usage: mmseg [--generate-extra-enter]\n");
- exit(1);
-}
-
-int main(int argc, char * argv[]){
- int i = 1;
- bool gen_extra_enter = false;
-
- setlocale(LC_ALL, "");
- //deal with options.
- while ( i < argc ){
- if ( strcmp ("--help", argv[i]) == 0) {
- print_help();
- }else if (strcmp("--generate-extra-enter", argv[i]) == 0) {
- gen_extra_enter = true;
- }
- ++i;
- }
-
- //init phrase table
- g_phrases = new PhraseLargeTable;
- FILE * gb_file = fopen("../../data/gb_char.table", "r");
- if ( gb_file == NULL ){
- fprintf(stderr, "can't open gb_char.table!\n");
- exit(1);
- }
- g_phrases->load_text(gb_file);
- fclose(gb_file);
-
- FILE * gbk_file = fopen("../../data/gbk_char.table", "r");
- if ( gbk_file == NULL ){
- fprintf(stderr, "can't open gbk_char.table!\n");
- exit(1);
- }
- g_phrases->load_text(gbk_file);
- fclose(gbk_file);
-
- MemoryChunk * chunk = new MemoryChunk;
- g_phrases->store(chunk);
- g_phrases->load(chunk);
-
- char * linebuf = NULL;
- size_t size = 0;
- while(getline(&linebuf, &size, stdin)) {
- if ( feof(stdin) )
- break;
- linebuf[strlen(linebuf) - 1] = '\0';
-
- //check non-ucs2 characters
- const glong num_of_chars = g_utf8_strlen(linebuf, -1);
- glong len = 0;
- utf16_t * sentence = g_utf8_to_utf16(linebuf, -1, NULL, &len, NULL);
- if ( len != num_of_chars ) {
- fprintf(stderr, "non-ucs2 characters encountered:%s.\n", linebuf);
- continue;
- }
-
- //do segment stuff
- GArray * strings = g_array_new(TRUE, TRUE, sizeof(SegmentStep));
- segment(g_phrases, sentence, len, strings);
-
- //print out the split phrase
- for ( glong i = 0; i < strings->len; ++i ) {
- SegmentStep * step = &g_array_index(strings, SegmentStep, i);
- char * string = g_utf16_to_utf8( step->m_phrase, step->m_phrase_len, NULL, NULL, NULL);
- printf("%s\n", string);
- g_free(string);
- }
-
- //print extra enter
- if ( gen_extra_enter )
- printf("\n");
-
- g_array_free(strings, TRUE);
- g_free(sentence);
- }
- return 0;
-}
diff --git a/src/training/Makefile.am b/src/training/Makefile.am
deleted file mode 100644
index 520e4e1..0000000
--- a/src/training/Makefile.am
+++ /dev/null
@@ -1,36 +0,0 @@
-## Makefile.am -- Process this file with automake to produce Makefile.in
-## Copyright (C) 2007 Peng Wu
-##
-## This program is free software; you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 2, or (at your option)
-## any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-## GNU General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with this program; if not, write to the Free Software
-## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-MAINTAINERCLEANFILES = Makefile.in
-
-INCLUDES = -I$(top_srcdir)/src/include \
- -I$(top_srcdir)/src/storage \
- @GLIB2_CPPFLAGS@
-
-noinst_PROGRAMS = gen_ngram gen_unigram estimate_interpolation
-
-gen_ngram_SOURCES = gen_ngram.cpp
-
-gen_ngram_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@
-
-gen_unigram_SOURCES = gen_unigram.cpp
-
-gen_unigram_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@
-
-estimate_interpolation_SOURCES = estimate_interpolation.cpp
-
-estimate_interpolation_LDADD = ../storage/libstorage.la @GLIB2_LDFLAGS@
diff --git a/src/training/estimate_interpolation.cpp b/src/training/estimate_interpolation.cpp
deleted file mode 100644
index aaae17b..0000000
--- a/src/training/estimate_interpolation.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * libpinyin
- * Library to deal with pinyin.
- *
- * Copyright (C) 2006-2008 Peng Wu
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <locale.h>
-#include <math.h>
-#include <glib.h>
-#include "memory_chunk.h"
-#include "novel_types.h"
-#include "phrase_index.h"
-#include "ngram.h"
-
-parameter_t compute_interpolation(SingleGram * deleted_bigram,
- FacadePhraseIndex * unigram,
- SingleGram * bigram){
- bool success;
- parameter_t lambda = 0, next_lambda = 0.6;
- parameter_t epsilon = 0.001;
-
- while ( fabs(lambda - next_lambda) > epsilon){
- lambda = next_lambda;
- next_lambda = 0;
- guint32 table_num = 0;
- parameter_t numerator = 0;
- parameter_t part_of_denominator = 0;
-
- BigramPhraseWithCountArray array = g_array_new(FALSE, FALSE, sizeof(BigramPhraseItemWithCount));
- deleted_bigram->retrieve_all(array);
-
- for ( int i = 0; i < array->len; ++i){
- BigramPhraseItemWithCount* item = &g_array_index(array, BigramPhraseItemWithCount, i);
- //get the phrase token
- phrase_token_t token = item->m_token;
- guint32 deleted_freq = item->m_count;
-
- {
- guint32 freq = 0;
- parameter_t elem_poss = 0;
- if ( bigram && bigram->get_freq(token, freq)){
- guint32 total_freq;
- assert(bigram->get_total_freq(total_freq));
- assert(0 != total_freq);
- elem_poss = freq / (parameter_t) total_freq;
- }
- numerator = lambda * elem_poss;
- }
-
- {
- guint32 freq = 0;
- parameter_t elem_poss = 0;
- PhraseItem item;
- if (!unigram->get_phrase_item(token, item)){
- guint32 freq = item.get_unigram_frequency();
- guint32 total_freq = unigram->get_phrase_index_total_freq();
- elem_poss = freq / (parameter_t)total_freq;
- }
- part_of_denominator = ( 1 - lambda) * elem_poss;
- }
-
- if ( 0 == (numerator + part_of_denominator))
- continue;
-
- next_lambda += deleted_freq * (numerator / (numerator + part_of_denominator));
- }
- assert(deleted_bigram->get_total_freq(table_num));
- next_lambda /= table_num;
-
- g_array_free(array, TRUE);
- }
- lambda = next_lambda;
- return lambda;
-}
-
-int main(int argc, char * argv[]){
- FacadePhraseIndex phrase_index;
-
- //gb_char binary file
- MemoryChunk * chunk = new MemoryChunk;
- chunk->load("../../data/gb_char.bin");
- phrase_index.load(1, chunk);
-
- //gbk_char binary file
- chunk = new MemoryChunk;
- chunk->load("../../data/gbk_char.bin");
- phrase_index.load(2, chunk);
-
- Bigram bigram;
- bigram.attach("../../data/bigram.db", NULL);
-
- Bigram deleted_bigram;
- deleted_bigram.attach("../../data/deleted_bigram.db", NULL);
-
- GArray * system_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
- GArray * user_items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
-
- deleted_bigram.get_all_items(system_items, user_items);
- assert(0 == user_items->len);
- g_array_free(user_items, TRUE);
-
- parameter_t lambda_sum = 0;
- int lambda_count = 0;
-
- for ( int i = 0; i < system_items->len; ++i ){
- phrase_token_t * token = &g_array_index(system_items, phrase_token_t, i);
- SingleGram * system = NULL, * user = NULL;
- bigram.load(*token, system, user);
- assert(NULL == user);
- SingleGram * deleted_system = NULL, * deleted_user = NULL;
- deleted_bigram.load(*token, deleted_system, deleted_user);
- assert(NULL == deleted_user);
-
- parameter_t lambda = compute_interpolation(deleted_system, &phrase_index, system);
-
- printf("lambda:%f\n", lambda);
-
- lambda_sum += lambda;
- lambda_count ++;
-
- if (system) delete system;
- delete deleted_system;
- }
-
- printf("average lambda:%f\n", (lambda_sum/lambda_count));
- g_array_free(system_items, TRUE);
-}
-
diff --git a/src/training/gen_ngram.cpp b/src/training/gen_ngram.cpp
deleted file mode 100644
index a1f5b6d..0000000
--- a/src/training/gen_ngram.cpp
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * libpinyin
- * Library to deal with pinyin.
- *
- * Copyright (C) 2006-2007 Peng Wu
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <locale.h>
-#include <glib.h>
-#include "memory_chunk.h"
-#include "novel_types.h"
-#include "phrase_index.h"
-#include "ngram.h"
-#include "phrase_large_table.h"
-
-
-static PhraseLargeTable * g_phrases = NULL;
-
-void print_help(){
- printf("gen_ngram [--skip-pi-gram-training] [--skip-unigram-training]\n");
- printf(" [--bigram-file <FILENAME>]\n");
- exit(1);
-}
-
-int main(int argc, char * argv[]){
- int i = 1;
- bool train_pi_gram = true;
- bool train_unigram = true;
- const char * bigram_filename = "../../data/bigram.db";
-
- setlocale(LC_ALL,"");
- while ( i < argc ){
- if ( strcmp("--help", argv[i] ) == 0){
- print_help();
- }else if ( strcmp("--skip-pi-gram-training", argv[i] ) == 0) {
- train_pi_gram = false;
- }else if ( strcmp("--skip-unigram-training", argv[i] ) == 0) {
- train_unigram = false;
- }else if ( strcmp("--bigram-file", argv[i] ) == 0){
- if ( ++i >= argc )
- print_help();
- bigram_filename = argv[i];
- }
- ++i;
- }
-
- g_phrases = new PhraseLargeTable;
- //init phrase lookup
- FILE * gb_file = fopen("../../data/gb_char.table", "r");
- if ( gb_file == NULL ){
- fprintf(stderr, "can't open gb_char.table!\n");
- exit(1);
- }
- g_phrases->load_text(gb_file);
- fclose(gb_file);
- FILE * gbk_file = fopen("../../data/gbk_char.table", "r");
- if ( gbk_file == NULL ){
- fprintf(stderr, "can't open gbk_char.table!\n");
- exit(1);
- }
- g_phrases->load_text(gbk_file);
- fclose(gbk_file);
-
- FacadePhraseIndex phrase_index;
-
- //gb_char binary file
- MemoryChunk * chunk = new MemoryChunk;
- chunk->load("../../data/gb_char.bin");
- phrase_index.load(1, chunk);
-
- //gbk_char binary file
- chunk = new MemoryChunk;
- chunk->load("../../data/gbk_char.bin");
- phrase_index.load(2, chunk);
-
- Bigram bigram;
- bigram.attach(NULL, bigram_filename);
-
-
- char* linebuf = (char *)malloc ( 1024 * sizeof (char) );
- size_t size = 1024;
- phrase_token_t last_token, cur_token = last_token = 0;
- while( getline(&linebuf, &size, stdin) ){
- if ( feof(stdin) )
- break;
- linebuf[strlen(linebuf)-1] = '\0';
-
- glong phrase_len = 0;
- utf16_t * phrase = g_utf8_to_utf16(linebuf, -1, NULL, &phrase_len, NULL);
-
- phrase_token_t token = 0;
- int result = g_phrases->search( phrase_len, phrase, token);
- if ( ! (result & SEARCH_OK) )
- token = 0;
-
- last_token = cur_token;
- cur_token = token;
- if ( cur_token ){
- //training uni-gram
- if ( train_unigram )
- phrase_index.add_unigram_frequency(cur_token, 1);
- }
- if ( cur_token ){
- SingleGram * system = NULL, * user = NULL;
- if ( 0 == last_token ){
- if (train_pi_gram)
- bigram.load(sentence_start, system, user);
- } else
- bigram.load(last_token, system, user);
- assert(NULL == system);
- if ( NULL == user ){
- user = new SingleGram;
- }
- guint32 freq, total_freq;
- //increase freq
- user->get_freq(cur_token, freq);
- user->set_freq(cur_token, freq + 1);
- //increase total freq
- user->get_total_freq(total_freq);
- user->set_total_freq(total_freq + 1);
- if ( 0 == last_token ){
- if ( train_pi_gram )
- bigram.store(sentence_start, user);
- }else
- bigram.store(last_token, user);
- delete user;
- }
- }
-
- MemoryChunk * new_chunk = new MemoryChunk;
- phrase_index.store(1, new_chunk);
- new_chunk->save("../../data/gb_char.bin");
- phrase_index.load(1, new_chunk);
-
- new_chunk = new MemoryChunk;
- phrase_index.store(2, new_chunk);
- new_chunk->save("../../data/gbk_char.bin");
- phrase_index.load(2, new_chunk);
-
- return 0;
-}
diff --git a/src/training/gen_unigram.cpp b/src/training/gen_unigram.cpp
deleted file mode 100644
index 1c70665..0000000
--- a/src/training/gen_unigram.cpp
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * libpinyin
- * Library to deal with pinyin.
- *
- * Copyright (C) 2006-2007 Peng Wu
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-
-#include <stdio.h>
-#include "memory_chunk.h"
-#include "novel_types.h"
-#include "phrase_index.h"
-
-//increase all unigram frequency by one.
-
-int main(int argc, char * argv[]){
-
- FacadePhraseIndex phrase_index;
-
- //gb_char binary file
- MemoryChunk * chunk = new MemoryChunk;
- chunk->load("../../data/gb_char.bin");
- phrase_index.load(1, chunk);
-
- //gbk_char binary file
- chunk = new MemoryChunk;
- chunk->load("../../data/gbk_char.bin");
- phrase_index.load(2, chunk);
-
- PhraseIndexRange range;
- int result = phrase_index.get_range(1, range);
- if ( result == ERROR_OK ) {
- for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){
- phrase_index.add_unigram_frequency(i, 1);
- }
- }
-
-#if 0
- int result = phrase_index.get_range(2, range);
- if ( result == ERROR_OK ) {
- for ( size_t i = range.m_range_begin; i <= range.m_range_end; ++i){
- phrase_index.add_unigram_frequency(i, 1);
- }
- }
-#endif
-
- MemoryChunk * new_chunk = new MemoryChunk;
- phrase_index.store(1, new_chunk);
- new_chunk->save("../../data/gb_char.bin");
- phrase_index.load(1, new_chunk);
-
- new_chunk = new MemoryChunk;
- phrase_index.store(2, new_chunk);
- new_chunk->save("../../data/gbk_char.bin");
- phrase_index.load(2, new_chunk);
-
- return 0;
-}