diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 33 | ||||
| -rw-r--r-- | tests/Makefile.am | 46 | ||||
| -rw-r--r-- | tests/include/CMakeLists.txt | 9 | ||||
| -rw-r--r-- | tests/include/Makefile.am | 30 | ||||
| -rw-r--r-- | tests/include/test_memory_chunk.cpp | 64 | ||||
| -rw-r--r-- | tests/lookup/CMakeLists.txt | 21 | ||||
| -rw-r--r-- | tests/lookup/Makefile.am | 32 | ||||
| -rw-r--r-- | tests/lookup/test_phrase_lookup.cpp | 118 | ||||
| -rw-r--r-- | tests/lookup/test_pinyin_lookup.cpp | 125 | ||||
| -rw-r--r-- | tests/storage/CMakeLists.txt | 71 | ||||
| -rw-r--r-- | tests/storage/Makefile.am | 55 | ||||
| -rw-r--r-- | tests/storage/test_chewing_table.cpp | 148 | ||||
| -rw-r--r-- | tests/storage/test_flexible_ngram.cpp | 138 | ||||
| -rw-r--r-- | tests/storage/test_ngram.cpp | 87 | ||||
| -rw-r--r-- | tests/storage/test_parser2.cpp | 154 | ||||
| -rw-r--r-- | tests/storage/test_phrase_index.cpp | 122 | ||||
| -rw-r--r-- | tests/storage/test_phrase_index_logger.cpp | 67 | ||||
| -rw-r--r-- | tests/storage/test_phrase_table.cpp | 86 | ||||
| -rw-r--r-- | tests/storage/test_table_info.cpp | 87 | ||||
| -rw-r--r-- | tests/test_chewing.cpp | 68 | ||||
| -rw-r--r-- | tests/test_phrase.cpp | 74 | ||||
| -rw-r--r-- | tests/test_pinyin.cpp | 95 | ||||
| -rw-r--r-- | tests/tests_helper.h | 86 | ||||
| -rw-r--r-- | tests/timer.h | 48 |
24 files changed, 0 insertions, 1864 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 3338796..0000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -add_subdirectory(include) -add_subdirectory(storage) -add_subdirectory(lookup) - -add_executable( - test_pinyin - test_pinyin.cpp -) - -target_link_libraries( - test_pinyin - libzhuyin -) - -add_executable( - test_phrase - test_phrase.cpp -) - -target_link_libraries( - test_phrase - libzhuyin -) - -add_executable( - test_chewing - test_chewing.cpp -) - -target_link_libraries( - test_chewing - libzhuyin -) diff --git a/tests/Makefile.am b/tests/Makefile.am deleted file mode 100644 index 1594163..0000000 --- a/tests/Makefile.am +++ /dev/null @@ -1,46 +0,0 @@ -## Makefile.am -- Process this file with automake to produce Makefile.in -## Copyright (C) 2007 Peng Wu -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2, or (at your option) -## any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -AUTOMAKE_OPTIONS = gnu -SUBDIRS = include storage lookup - -MAINTAINERCLEANFILES = Makefile.in - -CLEANFILES = *.bak - -ACLOCAL = aclocal -I $(ac_aux_dir) - -INCLUDES = -I$(top_srcdir)/src \ - -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/src/storage \ - -I$(top_srcdir)/src/lookup \ - @GLIB2_CFLAGS@ - -LDADD = ../src/libzhuyin.la @GLIB2_LIBS@ - -noinst_HEADERS = timer.h \ - tests_helper.h - -noinst_PROGRAMS = test_pinyin \ - test_phrase \ - test_chewing - -test_pinyin_SOURCES = test_pinyin.cpp - -test_phrase_SOURCES = test_phrase.cpp - -test_chewing_SOURCES = test_chewing.cpp diff --git a/tests/include/CMakeLists.txt b/tests/include/CMakeLists.txt deleted file mode 100644 index dd82f90..0000000 --- a/tests/include/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -add_executable( - test_memory_chunk - test_memory_chunk.cpp -) - -target_link_libraries( - test_memory_chunk - libzhuyin -) diff --git a/tests/include/Makefile.am b/tests/include/Makefile.am deleted file mode 100644 index f52c5ac..0000000 --- a/tests/include/Makefile.am +++ /dev/null @@ -1,30 +0,0 @@ -## Makefile.am -- Process this file with automake to produce Makefile.in -## Copyright (C) 2007 Peng Wu -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2, or (at your option) -## any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -INCLUDES = -I$(top_srcdir)/src \ - -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/src/storage \ - -I$(top_srcdir)/src/lookup \ - @GLIB2_CFLAGS@ - -LDADD = @GLIB2_LIBS@ - -TESTS = test_memory_chunk - -noinst_PROGRAMS = test_memory_chunk - -test_memory_chunk_SOURCES = test_memory_chunk.cpp diff --git a/tests/include/test_memory_chunk.cpp b/tests/include/test_memory_chunk.cpp deleted file mode 100644 index acd1690..0000000 --- a/tests/include/test_memory_chunk.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include <stdio.h> -#include "zhuyin_internal.h" - -//Test Memory Chunk Functionality -int main(int argc, char * argv[]){ - MemoryChunk* chunk; - chunk = new MemoryChunk(); - int i = 12; - chunk->set_content(0, &i, sizeof(int)); - - int * p = (int *)chunk->begin(); - assert(chunk->size() == sizeof(int)); - printf("%d\n", *p); - printf("%ld\n", chunk->capacity()); - - p = & i; - chunk->set_chunk(p, sizeof(int), NULL); - short t = 5; - chunk->set_content(sizeof(int), &t, sizeof(short)); - assert( sizeof(int) + sizeof(short) == chunk->size()); - printf("%ld\n", chunk->capacity()); - - p = (int *)chunk->begin(); - short * p2 =(short *)(((char *) (chunk->begin())) + sizeof(int)); - printf("%d\t%d\n", *p, *p2); - - chunk->set_content(sizeof(int) + sizeof(short), &t, sizeof(short)); - - assert( sizeof(int) + (sizeof(short) << 1) == chunk->size()); - printf("%ld\n", chunk->capacity()); - p = (int *)chunk->begin(); - p2 =(short *)(((char *) (chunk->begin())) + sizeof(int)); - printf("%d\t%d\t%d\n", *p, *p2, *(p2 + 1)); - - chunk->set_size(sizeof(int) + sizeof(short) *3); - p = (int *)chunk->begin(); - p2 =(short *)(((char *) (chunk->begin())) + sizeof(int)); - - chunk->set_content(0, &i, sizeof(int)); - - *(p2+2) = 3; - printf("%d\t%d\t%d\t%d\n", *p, *p2, *(p2 + 1), *(p2+2)); - - int m = 10; - chunk->set_chunk(&m, sizeof(int), NULL); - int n = 12; - chunk->insert_content(sizeof(int), &n, sizeof(int)); - n = 11; - chunk->insert_content(sizeof(int), &n, sizeof(int)); - - int * p3 = (int *)chunk->begin(); - printf("%d\t%d\t%d\n", *p3, *(p3+1), *(p3+2)); - - chunk->remove_content(sizeof(int), sizeof(int)); - printf("%d\t%d\n", *p3, *(p3+1)); - - int tmp; - assert(chunk->get_content(sizeof(int), &tmp, sizeof(int))); - printf("%d\n", tmp); - - delete chunk; - - return 0; -} diff --git a/tests/lookup/CMakeLists.txt b/tests/lookup/CMakeLists.txt deleted file mode 100644 index 79dc1ba..0000000 --- a/tests/lookup/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -include_directories(..) - -add_executable( - test_pinyin_lookup - test_pinyin_lookup.cpp -) - -target_link_libraries( - test_pinyin_lookup - libzhuyin -) - -add_executable( - test_phrase_lookup - test_phrase_lookup.cpp -) - -target_link_libraries( - test_phrase_lookup - libzhuyin -) diff --git a/tests/lookup/Makefile.am b/tests/lookup/Makefile.am deleted file mode 100644 index 93c8f08..0000000 --- a/tests/lookup/Makefile.am +++ /dev/null @@ -1,32 +0,0 @@ -## Makefile.am -- Process this file with automake to produce Makefile.in -## Copyright (C) 2007 Peng Wu -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2, or (at your option) -## any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -INCLUDES = -I$(top_srcdir)/src \ - -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/src/storage \ - -I$(top_srcdir)/src/lookup \ - -I$(top_srcdir)/tests \ - @GLIB2_CFLAGS@ - -LDADD = ../../src/libzhuyin_internal.la @GLIB2_LIBS@ - -noinst_PROGRAMS = test_pinyin_lookup \ - test_phrase_lookup - -test_pinyin_lookup_SOURCES = test_pinyin_lookup.cpp - -test_phrase_lookup_SOURCES = test_phrase_lookup.cpp diff --git a/tests/lookup/test_phrase_lookup.cpp b/tests/lookup/test_phrase_lookup.cpp deleted file mode 100644 index 9e9e6d3..0000000 --- a/tests/lookup/test_phrase_lookup.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include <stdio.h> -#include <locale.h> -#include "zhuyin_internal.h" -#include "tests_helper.h" - - -bool try_phrase_lookup(PhraseLookup * phrase_lookup, - ucs4_t * ucs4_str, glong ucs4_len){ - char * result_string = NULL; - MatchResults results = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); - phrase_lookup->get_best_match(ucs4_len, ucs4_str, results); -#if 0 - for ( size_t i = 0; i < results->len; ++i) { - phrase_token_t * token = &g_array_index(results, phrase_token_t, i); - if ( *token == null_token ) - continue; - printf("%d:%d\t", i, *token); - } - printf("\n"); -#endif - phrase_lookup->convert_to_utf8(results, result_string); - if (result_string) - printf("%s\n", result_string); - else - fprintf(stderr, "Error: Un-segmentable sentence encountered!\n"); - g_array_free(results, TRUE); - g_free(result_string); - return true; -} - -int main(int argc, char * argv[]){ - setlocale(LC_ALL, ""); - - SystemTableInfo system_table_info; - - bool retval = system_table_info.load("../../data/table.conf"); - if (!retval) { - fprintf(stderr, "load table.conf failed.\n"); - exit(ENOENT); - } - - /* init phrase table */ - FacadePhraseTable2 phrase_table; - MemoryChunk * chunk = new MemoryChunk; - chunk->load("../../data/phrase_index.bin"); - phrase_table.load(chunk, NULL); - - const pinyin_table_info_t * phrase_files = - system_table_info.get_table_info(); - - /* init phrase index */ - FacadePhraseIndex phrase_index; - if (!load_phrase_index(phrase_files, &phrase_index)) - exit(ENOENT); - - /* init bi-gram */ - Bigram system_bigram; - system_bigram.attach("../../data/bigram.db", ATTACH_READONLY); - Bigram user_bigram; - - gfloat lambda = system_table_info.get_lambda(); - - /* init phrase lookup */ - PhraseLookup phrase_lookup(lambda, - &phrase_table, &phrase_index, - &system_bigram, &user_bigram); - - /* try one sentence */ - char * linebuf = NULL; - size_t size = 0; - ssize_t read; - while( (read = getline(&linebuf, &size, stdin)) != -1 ){ - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - /* check non-ucs4 characters */ - const glong num_of_chars = g_utf8_strlen(linebuf, -1); - glong len = 0; - ucs4_t * sentence = g_utf8_to_ucs4(linebuf, -1, NULL, &len, NULL); - if ( len != num_of_chars ) { - fprintf(stderr, "non-ucs4 characters are not accepted.\n"); - g_free(sentence); - continue; - } - - try_phrase_lookup(&phrase_lookup, sentence, len); - g_free(sentence); - } - - free(linebuf); - return 0; -} diff --git a/tests/lookup/test_pinyin_lookup.cpp b/tests/lookup/test_pinyin_lookup.cpp deleted file mode 100644 index a37c8d4..0000000 --- a/tests/lookup/test_pinyin_lookup.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2012 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include "timer.h" -#include <string.h> -#include "zhuyin_internal.h" -#include "tests_helper.h" - -size_t bench_times = 100; - -int main( int argc, char * argv[]){ - SystemTableInfo system_table_info; - - bool retval = system_table_info.load("../../data/table.conf"); - if (!retval) { - fprintf(stderr, "load table.conf failed.\n"); - exit(ENOENT); - } - - pinyin_option_t options = USE_TONE; - FacadeChewingTable largetable; - - MemoryChunk * chunk = new MemoryChunk; - chunk->load("../../data/pinyin_index.bin"); - largetable.load(options, chunk, NULL); - - const pinyin_table_info_t * phrase_files = - system_table_info.get_table_info(); - - FacadePhraseIndex phrase_index; - if (!load_phrase_index(phrase_files, &phrase_index)) - exit(ENOENT); - - Bigram system_bigram; - system_bigram.attach("../../data/bigram.db", ATTACH_READONLY); - Bigram user_bigram; - user_bigram.attach(NULL, ATTACH_CREATE|ATTACH_READWRITE); - - gfloat lambda = system_table_info.get_lambda(); - - PinyinLookup2 pinyin_lookup(lambda, options, - &largetable, &phrase_index, - &system_bigram, &user_bigram); - - /* prepare the prefixes for get_best_match. */ - TokenVector prefixes = g_array_new - (FALSE, FALSE, sizeof(phrase_token_t)); - g_array_append_val(prefixes, sentence_start); - - CandidateConstraints constraints = g_array_new - (TRUE, FALSE, sizeof(lookup_constraint_t)); - - MatchResults results = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); - - char* linebuf = NULL; size_t size = 0; ssize_t read; - while( (read = getline(&linebuf, &size, stdin)) != -1 ){ - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - FullPinyinParser2 parser; - ChewingKeyVector keys = g_array_new(FALSE, FALSE, sizeof(ChewingKey)); - ChewingKeyRestVector key_rests = - g_array_new(FALSE, FALSE, sizeof(ChewingKeyRest)); - parser.parse(options, keys, key_rests, linebuf, strlen(linebuf)); - - if ( 0 == keys->len ) /* invalid pinyin */ - continue; - - /* initialize constraints. */ - g_array_set_size(constraints, keys->len); - for ( size_t i = 0; i < constraints->len; ++i){ - lookup_constraint_t * constraint = &g_array_index(constraints, lookup_constraint_t, i); - constraint->m_type = NO_CONSTRAINT; - } - - guint32 start_time = record_time(); - for ( size_t i = 0; i < bench_times; ++i) - pinyin_lookup.get_best_match(prefixes, keys, constraints, results); - print_time(start_time, bench_times); - for ( size_t i = 0; i < results->len; ++i){ - phrase_token_t * token = &g_array_index(results, phrase_token_t, i); - if ( null_token == *token) - continue; - printf("pos:%ld,token:%d\t", i, *token); - } - printf("\n"); - char * sentence = NULL; - pinyin_lookup.convert_to_utf8(results, sentence); - printf("%s\n", sentence); - - g_array_free(keys, TRUE); - g_array_free(key_rests, TRUE); - g_free(sentence); - } - - g_array_free(prefixes, TRUE); - g_array_free(constraints, TRUE); - g_array_free(results, TRUE); - - free(linebuf); - return 0; -} diff --git a/tests/storage/CMakeLists.txt b/tests/storage/CMakeLists.txt deleted file mode 100644 index 3512370..0000000 --- a/tests/storage/CMakeLists.txt +++ /dev/null @@ -1,71 +0,0 @@ -include_directories(..) - -add_executable( - test_parser2 - test_parser2.cpp -) - -target_link_libraries( - test_parser2 - libzhuyin -) - -add_executable( - test_chewing_table - test_chewing_table.cpp -) - -target_link_libraries( - test_chewing_table - libzhuyin -) - -add_executable( - test_phrase_index - test_phrase_index.cpp -) - -target_link_libraries( - test_phrase_index - libzhuyin -) - -add_executable( - test_phrase_index_logger - test_phrase_index_logger.cpp -) - -target_link_libraries( - test_phrase_index_logger - libzhuyin -) - -add_executable( - test_phrase_table - test_phrase_table.cpp -) - -target_link_libraries( - test_phrase_table - libzhuyin -) - -add_executable( - test_ngram - test_ngram.cpp -) - -target_link_libraries( - test_ngram - libzhuyin -) - -add_executable( - test_flexible_ngram - test_flexible_ngram.cpp -) - -target_link_libraries( - test_flexible_ngram - libzhuyin -) diff --git a/tests/storage/Makefile.am b/tests/storage/Makefile.am deleted file mode 100644 index 10483e4..0000000 --- a/tests/storage/Makefile.am +++ /dev/null @@ -1,55 +0,0 @@ -## Makefile.am -- Process this file with automake to produce Makefile.in -## Copyright (C) 2007 Peng Wu -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2, or (at your option) -## any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -INCLUDES = -I$(top_srcdir)/src \ - -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/src/storage \ - -I$(top_srcdir)/src/lookup \ - -I$(top_srcdir)/tests \ - @GLIB2_CFLAGS@ - -LDADD = ../../src/libzhuyin_internal.la @GLIB2_LIBS@ - -TESTS = test_phrase_index_logger \ - test_ngram \ - test_flexible_ngram - -noinst_PROGRAMS = test_phrase_index \ - test_phrase_index_logger \ - test_phrase_table \ - test_ngram \ - test_flexible_ngram \ - test_parser2 \ - test_chewing_table \ - test_table_info - - -test_phrase_index_SOURCES = test_phrase_index.cpp - -test_phrase_index_logger_SOURCES = test_phrase_index_logger.cpp - -test_phrase_table_SOURCES = test_phrase_table.cpp - -test_ngram_SOURCES = test_ngram.cpp - -test_flexible_ngram_SOURCES = test_flexible_ngram.cpp - -test_parser2_SOURCES = test_parser2.cpp - -test_chewing_table_SOURCES = test_chewing_table.cpp - -test_table_info_SOURCES = test_table_info.cpp diff --git a/tests/storage/test_chewing_table.cpp b/tests/storage/test_chewing_table.cpp deleted file mode 100644 index e3354a1..0000000 --- a/tests/storage/test_chewing_table.cpp +++ /dev/null @@ -1,148 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#include "timer.h" -#include <string.h> -#include "zhuyin_internal.h" -#include "tests_helper.h" - -size_t bench_times = 1000; - -int main(int argc, char * argv[]) { - SystemTableInfo system_table_info; - - bool retval = system_table_info.load("../../data/table.conf"); - if (!retval) { - fprintf(stderr, "load table.conf failed.\n"); - exit(ENOENT); - } - - pinyin_option_t options = USE_TONE | PINYIN_INCOMPLETE; - ChewingLargeTable largetable(options); - FacadePhraseIndex phrase_index; - - const pinyin_table_info_t * phrase_files = - system_table_info.get_table_info(); - - if (!load_phrase_table(phrase_files, &largetable, NULL, &phrase_index)) - exit(ENOENT); - - MemoryChunk * new_chunk = new MemoryChunk; - largetable.store(new_chunk); - largetable.load(new_chunk); - - char* linebuf = NULL; size_t size = 0; ssize_t read; - while ((read = getline(&linebuf, &size, stdin)) != -1) { - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - FullPinyinParser2 parser; - ChewingKeyVector keys = g_array_new(FALSE, FALSE, sizeof(ChewingKey)); - ChewingKeyRestVector key_rests = - g_array_new(FALSE, FALSE, sizeof(ChewingKeyRest)); - - parser.parse(options, keys, key_rests, linebuf, strlen(linebuf)); - if (0 == keys->len) { - fprintf(stderr, "Invalid input.\n"); - continue; - } - - guint32 start = record_time(); - PhraseIndexRanges ranges; - memset(ranges, 0, sizeof(PhraseIndexRanges)); - - phrase_index.prepare_ranges(ranges); - - for (size_t i = 0; i < bench_times; ++i) { - phrase_index.clear_ranges(ranges); - largetable.search(keys->len, (ChewingKey *)keys->data, ranges); - } - print_time(start, bench_times); - - phrase_index.clear_ranges(ranges); - largetable.search(keys->len, (ChewingKey *)keys->data, ranges); - - for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) { - GArray * & range = ranges[i]; - if (!range) - continue; - - if (range->len) - printf("range items number:%d\n", range->len); - - for (size_t k = 0; k < range->len; ++k) { - PhraseIndexRange * onerange = - &g_array_index(range, PhraseIndexRange, k); - printf("start:%d\tend:%d\n", onerange->m_range_begin, - onerange->m_range_end); - - PhraseItem item; - for ( phrase_token_t token = onerange->m_range_begin; - token != onerange->m_range_end; ++token){ - - phrase_index.get_phrase_item( token, item); - - /* get phrase string */ - ucs4_t buffer[MAX_PHRASE_LENGTH + 1]; - item.get_phrase_string(buffer); - char * string = g_ucs4_to_utf8 - ( buffer, item.get_phrase_length(), - NULL, NULL, NULL); - printf("%s\t", string); - g_free(string); - - ChewingKey chewing_buffer[MAX_PHRASE_LENGTH]; - size_t npron = item.get_n_pronunciation(); - guint32 freq; - for (size_t m = 0; m < npron; ++m){ - item.get_nth_pronunciation(m, chewing_buffer, freq); - for (size_t n = 0; n < item.get_phrase_length(); - ++n){ - gchar * pinyins = - chewing_buffer[n].get_pinyin_string(); - printf("%s'", pinyins); - g_free(pinyins); - } - printf("\b\t%d\t", freq); - } - } - printf("\n"); - } - g_array_set_size(range, 0); - } - - phrase_index.destroy_ranges(ranges); - g_array_free(keys, TRUE); - g_array_free(key_rests, TRUE); - } - - if (linebuf) - free(linebuf); - - /* mask out all index items. */ - largetable.mask_out(0x0, 0x0); - - return 0; -} diff --git a/tests/storage/test_flexible_ngram.cpp b/tests/storage/test_flexible_ngram.cpp deleted file mode 100644 index 886d8e2..0000000 --- a/tests/storage/test_flexible_ngram.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2012 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include "zhuyin_internal.h" - -int main(int argc, char * argv[]) { - FlexibleSingleGram<guint32, guint32> single_gram; - typedef FlexibleSingleGram<guint32, guint32>::ArrayItemWithToken array_item_t; - - const guint32 total_freq = 16; - assert(single_gram.set_array_header(total_freq)); - - phrase_token_t tokens[6] = { 2, 6, 4, 3, 1, 3 }; - guint32 freqs[6] = { 1, 2, 4, 8, 16, 32}; - - guint32 freq; - - for ( size_t i = 0; i < G_N_ELEMENTS(tokens); ++i ){ - if ( single_gram.get_array_item(tokens[i], freq) ) - assert(single_gram.set_array_item(tokens[i], freqs[i])); - else - assert(single_gram.insert_array_item(tokens[i], freqs[i])); - } - - single_gram.get_array_item(3, freq); - assert(freq == 32); - - printf("--------------------------------------------------------\n"); - PhraseIndexRange range; - FlexibleBigramPhraseArray array = g_array_new(FALSE, FALSE, sizeof(array_item_t)); - range.m_range_begin = 0; range.m_range_end = 8; - single_gram.search(&range, array); - for ( size_t i = 0; i < array->len; ++i ){ - array_item_t * item = &g_array_index(array, array_item_t, i); - printf("item:%d:%d\n", item->m_token, item->m_item); - } - - assert(single_gram.get_array_header(freq)); - assert(freq == total_freq); - - FlexibleBigram<guint32, guint32, guint32> bigram("TEST"); - assert(bigram.attach("/tmp/training.db", ATTACH_READWRITE|ATTACH_CREATE)); - bigram.store(1, &single_gram); - assert(single_gram.insert_array_item(5, 8)); - assert(single_gram.remove_array_item(1, freq)); - assert(single_gram.set_array_header(32)); - assert(single_gram.get_array_header(freq)); - printf("new array header:%d\n", freq); - bigram.store(2, &single_gram); - - for (int m = 1; m <= 2; ++m ){ - printf("--------------------------------------------------------\n"); - FlexibleSingleGram<guint32, guint32> * train_gram; - bigram.load(m, train_gram); - g_array_set_size(array, 0); - range.m_range_begin = 0; range.m_range_end = 8; - train_gram->search(&range, array); - for ( size_t i = 0; i < array->len; ++i ){ - array_item_t * item = &g_array_index(array, array_item_t, i); - printf("item:%d:%d\n", item->m_token, item->m_item); - } - delete train_gram; - } - - GArray * items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); - bigram.get_all_items(items); - printf("-----------------------items----------------------------\n"); - for ( size_t i = 0; i < items->len; ++i ){ - phrase_token_t * token = &g_array_index(items, phrase_token_t, i); - printf("item:%d\n", *token); - } - - printf("-----------------------magic header---------------------\n"); - bigram.set_magic_header(total_freq); - bigram.get_magic_header(freq); - assert(total_freq == freq); - printf("magic header:%d\n", freq); - - printf("-----------------------array header---------------------\n"); - for ( int i = 1; i <= 2; ++i){ - bigram.get_array_header(i, freq); - printf("single gram: %d, freq:%d\n", i, freq); - } - - bigram.set_array_header(1, 1); - - printf("-----------------------array header---------------------\n"); - for ( int i = 1; i <= 2; ++i){ - bigram.get_array_header(i, freq); - printf("single gram: %d, freq:%d\n", i, freq); - } - - for (int m = 1; m <= 2; ++m ){ - printf("--------------------------------------------------------\n"); - FlexibleSingleGram<guint32, guint32> * train_gram; - bigram.load(m, train_gram); - g_array_set_size(array, 0); - range.m_range_begin = 0; range.m_range_end = 8; - train_gram->search(&range, array); - for ( size_t i = 0; i < array->len; ++i ){ - array_item_t * item = &g_array_index(array, array_item_t, i); - printf("item:%d:%d\n", item->m_token, item->m_item); - } - delete train_gram; - } - - assert(bigram.remove(1)); - - bigram.get_all_items(items); - printf("-----------------------items----------------------------\n"); - for ( size_t i = 0; i < items->len; ++i ){ - phrase_token_t * token = &g_array_index(items, phrase_token_t, i); - printf("item:%d\n", *token); - } - - g_array_free(items, TRUE); - g_array_free(array, TRUE); - return 0; -} diff --git a/tests/storage/test_ngram.cpp b/tests/storage/test_ngram.cpp deleted file mode 100644 index 7816acc..0000000 --- a/tests/storage/test_ngram.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include <stdio.h> -#include "zhuyin_internal.h" - - -int main(int argc, char * argv[]){ - SingleGram single_gram; - - const guint32 total_freq = 16; - assert(single_gram.set_total_freq(total_freq)); - - phrase_token_t tokens[6] = { 2, 6, 4, 3, 1, 3}; - guint32 freqs[6] = { 1, 2, 4, 8, 16, 32}; - - guint32 freq; - - for(size_t i = 0; i < 6 ;++i){ - if ( single_gram.get_freq(tokens[i], freq)) - assert(single_gram.set_freq(tokens[i], freqs[i])); - else - assert(single_gram.insert_freq(tokens[i], freqs[i])); - } - - single_gram.get_freq(3, freq); - assert(freq == 32); - - printf("--------------------------------------------------------\n"); - PhraseIndexRange range; - BigramPhraseArray array = g_array_new(FALSE, FALSE, sizeof(BigramPhraseItem)); - range.m_range_begin = 0; range.m_range_end = 8; - single_gram.search(&range,array); - for ( size_t i = 0; i < array->len; ++i){ - BigramPhraseItem * item = &g_array_index(array, BigramPhraseItem, i); - printf("item:%d:%f\n", item->m_token, item->m_freq); - } - - assert(single_gram.get_total_freq(freq)); - assert(freq == total_freq); - - Bigram bigram; - assert(bigram.attach("/tmp/test.db", ATTACH_CREATE|ATTACH_READWRITE)); - bigram.store(1, &single_gram); - assert(single_gram.insert_freq(5, 8)); - assert(single_gram.remove_freq(1, freq)); - single_gram.set_total_freq(32); - - bigram.store(2, &single_gram); - - - SingleGram * gram = NULL; - for ( int m = 1; m <= 2; ++m ){ - printf("--------------------------------------------------------\n"); - bigram.load(m, gram); - g_array_set_size(array, 0); - range.m_range_begin = 0; range.m_range_end = 8; - gram->search(&range,array); - for ( size_t i = 0; i < array->len; ++i){ - BigramPhraseItem * item = &g_array_index(array, BigramPhraseItem, i); - printf("item:%d:%f\n", item->m_token, item->m_freq); - } - delete gram; - } - - printf("--------------------------------------------------------\n"); - assert(single_gram.get_total_freq(freq)); - printf("total_freq:%d\n", freq); - - g_array_free(array, TRUE); - - GArray * items = g_array_new(FALSE, FALSE, sizeof(phrase_token_t)); - bigram.get_all_items(items); - - printf("----------------------system----------------------------\n"); - for ( size_t i = 0; i < items->len; ++i){ - phrase_token_t * token = &g_array_index(items, phrase_token_t, i); - printf("item:%d\n", *token); - } - - assert(bigram.load_db("/tmp/test.db")); - assert(bigram.save_db("/tmp/test.db")); - - g_array_free(items, TRUE); - - /* mask out all index items. */ - bigram.mask_out(0x0, 0x0); - - return 0; -} diff --git a/tests/storage/test_parser2.cpp b/tests/storage/test_parser2.cpp deleted file mode 100644 index 3205e01..0000000 --- a/tests/storage/test_parser2.cpp +++ /dev/null @@ -1,154 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include "timer.h" -#include <errno.h> -#include <stdio.h> -#include <assert.h> -#include <stdlib.h> -#include <string.h> -#include "pinyin_parser2.h" - - -static const gchar * parsername = ""; -static gboolean incomplete = FALSE; -static const gchar * schemename = ""; - -static GOptionEntry entries[] = -{ - {"parser", 'p', 0, G_OPTION_ARG_STRING, &parsername, "parser", "fullpinyin chewing direct"}, - {"incomplete", 'i', 0, G_OPTION_ARG_NONE, &incomplete, "incomplete pinyin", NULL}, - {"scheme", 's', 0, G_OPTION_ARG_STRING, &schemename, "scheme", "standard hsu dachen26"}, - {NULL} -}; - -#if 0 - " -s <scheme> specify scheme for doublepinyin/chewing.\n" - " schemes for doublepinyin: zrm, ms, ziguang, abc, pyjj, xhe.\n" - " schemes for chewing: standard, ibm, ginyieh, eten.\n" -#endif - - -size_t bench_times = 1000; - -using namespace zhuyin; - - -int main(int argc, char * argv[]) { - GError * error = NULL; - GOptionContext * context; - - context = g_option_context_new("- test pinyin parser"); - g_option_context_add_main_entries(context, entries, NULL); - if (!g_option_context_parse(context, &argc, &argv, &error)) { - g_print("option parsing failed:%s\n", error->message); - exit(EINVAL); - } - - pinyin_option_t options = USE_TONE|FORCE_TONE; - if (incomplete) - options |= PINYIN_INCOMPLETE | CHEWING_INCOMPLETE; - - PhoneticParser2 * parser = NULL; - ChewingKeyVector keys = g_array_new(FALSE, FALSE, sizeof(ChewingKey)); - ChewingKeyRestVector key_rests = - g_array_new(FALSE, FALSE, sizeof(ChewingKeyRest)); - - /* create the parser */ - if (strcmp("fullpinyin", parsername) == 0) { - parser = new FullPinyinParser2(); - } else if (strcmp("chewing", parsername) == 0) { - if (strcmp("standard", schemename) == 0) { - parser = new ChewingSimpleParser2(); - } else if (strcmp("hsu", schemename) == 0) { - parser = new ChewingDiscreteParser2(); - } else if (strcmp("dachen26", schemename) == 0) { - parser = new ChewingDaChenCP26Parser2(); - } - } else if (strcmp("direct", parsername) == 0) { - parser = new ChewingDirectParser2(); - } - - if (!parser) - parser = new ChewingSimpleParser2(); - - char* linebuf = NULL; size_t size = 0; ssize_t read; - while( (read = getline(&linebuf, &size, stdin)) != -1 ){ - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - -#if 0 - ChewingKey key; - bool success = parser->parse_one_key(options, key, - linebuf, strlen(linebuf)); - if (success) { - gchar * pinyins = key.get_pinyin_string(); - printf("pinyin:%s\n", pinyins); - g_free(pinyins); - } -#endif - -#if 1 - int len = 0; - guint32 start_time = record_time(); - for ( size_t i = 0; i < bench_times; ++i) - len = parser->parse(options, keys, key_rests, - linebuf, strlen(linebuf)); - - print_time(start_time, bench_times); - - printf("parsed %d chars, %d keys.\n", len, keys->len); - - assert(keys->len == key_rests->len); - - for (size_t i = 0; i < keys->len; ++i) { - ChewingKey * key = - &g_array_index(keys, ChewingKey, i); - ChewingKeyRest * key_rest = - &g_array_index(key_rests, ChewingKeyRest, i); - - gchar * pinyins = key->get_pinyin_string(); - gchar * bopomofos = key->get_bopomofo_string(); - printf("%s %s %d %d\t", pinyins, bopomofos, - key_rest->m_raw_begin, key_rest->m_raw_end); - g_free(bopomofos); - g_free(pinyins); - } - printf("\n"); -#endif - - } - - if (linebuf) - free(linebuf); - - delete parser; - - g_array_free(key_rests, TRUE); - g_array_free(keys, TRUE); - - return 0; -} diff --git a/tests/storage/test_phrase_index.cpp b/tests/storage/test_phrase_index.cpp deleted file mode 100644 index c360c5b..0000000 --- a/tests/storage/test_phrase_index.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "timer.h" -#include <stdio.h> -#include <errno.h> -#include "zhuyin_internal.h" -#include "tests_helper.h" - -size_t bench_times = 100000; - -int main(int argc, char * argv[]){ - PhraseItem phrase_item; - ucs4_t string1 = 2; - ChewingKey key1 = ChewingKey(CHEWING_CH, CHEWING_ZERO_MIDDLE, CHEWING_ENG); - ChewingKey key2 = ChewingKey(CHEWING_SH, CHEWING_ZERO_MIDDLE, CHEWING_ANG); - - - phrase_item.set_phrase_string(1, &string1); - phrase_item.add_pronunciation(&key1, 100); - phrase_item.add_pronunciation(&key2, 300); - - assert(phrase_item.get_phrase_length() == 1); - - ChewingKey key3; - guint32 freq; - phrase_item.get_nth_pronunciation(0, &key3, freq); - assert(key3 == key1); - assert(freq == 100); - phrase_item.get_nth_pronunciation(1, &key3, freq); - assert(key3 == key2); - assert(freq == 300); - - pinyin_option_t options = 0; - gfloat poss = phrase_item.get_pronunciation_possibility(options, &key1); - printf("pinyin possiblitiy:%f\n", poss); - - assert(phrase_item.get_unigram_frequency() == 0); - - ucs4_t string2; - phrase_item.get_phrase_string(&string2); - assert(string1 == string2); - - FacadePhraseIndex phrase_index_test; - assert(!phrase_index_test.add_phrase_item(1, &phrase_item)); - - MemoryChunk* chunk = new MemoryChunk; - assert(phrase_index_test.store(0, chunk)); - assert(phrase_index_test.load(0, chunk)); - - PhraseItem item2; - guint32 time = record_time(); - for ( size_t i = 0; i < bench_times; ++i){ - phrase_index_test.get_phrase_item(1, item2); - assert(item2.get_unigram_frequency() == 0); - assert(item2.get_n_pronunciation() == 2); - assert(item2.get_phrase_length() == 1); - assert(item2.get_pronunciation_possibility(options, &key2) == 0.75); - } - print_time(time, bench_times); - - { - PhraseItem item3; - phrase_index_test.get_phrase_item(1, item3); - item3.increase_pronunciation_possibility(options, &key1, 200); - assert(item3.get_pronunciation_possibility(options, &key1) == 0.5) ; - } - - { - PhraseItem item5; - phrase_index_test.get_phrase_item(1, item5); - gfloat poss = item5.get_pronunciation_possibility(options, &key1); - printf("pinyin poss:%f\n", poss); - assert(poss == 0.5); - } - - SystemTableInfo system_table_info; - - bool retval = system_table_info.load("../../data/table.conf"); - if (!retval) { - fprintf(stderr, "load table.conf failed.\n"); - exit(ENOENT); - } - - FacadePhraseIndex phrase_index; - - const pinyin_table_info_t * phrase_files = - system_table_info.get_table_info(); - - if (!load_phrase_table(phrase_files, NULL, NULL, &phrase_index)) - exit(ENOENT); - - phrase_index.compact(); - - MemoryChunk* store1 = new MemoryChunk; - phrase_index.store(1, store1); - phrase_index.load(1, store1); - - MemoryChunk* store2 = new MemoryChunk; - phrase_index.store(2, store2); - phrase_index.load(2, store2); - - phrase_index.compact(); - - phrase_index.get_phrase_item(16870553, item2); - assert( item2.get_phrase_length() == 14); - assert( item2.get_n_pronunciation() == 1); - - ucs4_t buf[1024]; - item2.get_phrase_string(buf); - char * string = g_ucs4_to_utf8( buf, 14, NULL, NULL, NULL); - printf("%s\n", string); - g_free(string); - - guint32 delta = 3; - phrase_index.add_unigram_frequency(16870553, delta); - phrase_index.get_phrase_item(16870553, item2); - assert( item2.get_unigram_frequency() == 3); - - phrase_index.get_phrase_item(16777222, item2); - assert(item2.get_phrase_length() == 1); - assert(item2.get_n_pronunciation() == 2); - - return 0; -} diff --git a/tests/storage/test_phrase_index_logger.cpp b/tests/storage/test_phrase_index_logger.cpp deleted file mode 100644 index f13f7ca..0000000 --- a/tests/storage/test_phrase_index_logger.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#include "zhuyin_internal.h" - - -/* TODO: check whether tsi.bin and tsi2.bin should be the same. */ - -int main(int argc, char * argv[]){ - FacadePhraseIndex phrase_index; - MemoryChunk * chunk = new MemoryChunk; - chunk->load("../../data/tsi.bin"); - phrase_index.load(1, chunk); - - PhraseIndexRange range; - assert(ERROR_OK == phrase_index.get_range(1, range)); - for (size_t i = range.m_range_begin; i < range.m_range_end; ++i ) { - phrase_index.add_unigram_frequency(i, 1); - } - - printf("total freq:%d\n", phrase_index.get_phrase_index_total_freq()); - - MemoryChunk * new_chunk = new MemoryChunk; - phrase_index.store(1, new_chunk); - new_chunk->save("/tmp/tsi.bin"); - delete new_chunk; - - chunk = new MemoryChunk; - chunk->load("../../data/tsi.bin"); - new_chunk = new MemoryChunk; - assert(phrase_index.diff(1, chunk, new_chunk)); - new_chunk->save("/tmp/tsi.dbin"); - delete new_chunk; - - chunk = new MemoryChunk; - chunk->load("../../data/tsi.bin"); - phrase_index.load(1, chunk); - new_chunk = new MemoryChunk; - new_chunk->load("/tmp/tsi.dbin"); - assert(phrase_index.merge(1, new_chunk)); - chunk = new MemoryChunk; - phrase_index.store(1, chunk); - chunk->save("/tmp/tsi2.bin"); - delete chunk; - - printf("total freq:%d\n", phrase_index.get_phrase_index_total_freq()); - - return 0; -} diff --git a/tests/storage/test_phrase_table.cpp b/tests/storage/test_phrase_table.cpp deleted file mode 100644 index 7fc0a29..0000000 --- a/tests/storage/test_phrase_table.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "timer.h" -#include <string.h> -#include "zhuyin_internal.h" -#include "tests_helper.h" - -size_t bench_times = 1000; - -int main(int argc, char * argv[]){ - SystemTableInfo system_table_info; - - bool retval = system_table_info.load("../../data/table.conf"); - if (!retval) { - fprintf(stderr, "load table.conf failed.\n"); - exit(ENOENT); - } - - PhraseLargeTable2 largetable; - FacadePhraseIndex phrase_index; - - const pinyin_table_info_t * phrase_files = - system_table_info.get_table_info(); - - if (!load_phrase_table(phrase_files, NULL, &largetable, &phrase_index)) - exit(ENOENT); - - MemoryChunk * chunk = new MemoryChunk; - largetable.store(chunk); - largetable.load(chunk); - - char* linebuf = NULL; size_t size = 0; ssize_t read; - while ((read = getline(&linebuf, &size, stdin)) != -1) { - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - glong phrase_len = g_utf8_strlen(linebuf, -1); - ucs4_t * new_phrase = g_utf8_to_ucs4(linebuf, -1, NULL, NULL, NULL); - - if (0 == phrase_len) - continue; - - PhraseTokens tokens; - memset(tokens, 0, sizeof(PhraseTokens)); - phrase_index.prepare_tokens(tokens); - - guint32 start = record_time(); - for (size_t i = 0; i < bench_times; ++i){ - phrase_index.clear_tokens(tokens); - largetable.search(phrase_len, new_phrase, tokens); - } - print_time(start, bench_times); - - phrase_index.clear_tokens(tokens); - int retval = largetable.search(phrase_len, new_phrase, tokens); - - if (retval & SEARCH_OK) { - for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) { - GArray * array = tokens[i]; - if (NULL == array) - continue; - - for (size_t k = 0; k < array->len; ++k) { - phrase_token_t token = g_array_index - (array, phrase_token_t, k); - - printf("token:%d\t", token); - } - } - printf("\n"); - } - - phrase_index.destroy_tokens(tokens); - g_free(new_phrase); - } - - if ( linebuf ) - free(linebuf); - - /* mask out all index items. */ - largetable.mask_out(0x0, 0x0); - - return 0; -} diff --git a/tests/storage/test_table_info.cpp b/tests/storage/test_table_info.cpp deleted file mode 100644 index 6fa09f3..0000000 --- a/tests/storage/test_table_info.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2013 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#include <stdio.h> -#include <locale.h> -#include "zhuyin_internal.h" - - -int main(int argc, char * argv[]) { - setlocale(LC_ALL, ""); - - SystemTableInfo system_table_info; - - bool retval = system_table_info.load("../../data/table.conf"); - if (!retval) { - fprintf(stderr, "load table.conf failed.\n"); - exit(ENOENT); - } - - printf("lambda:%f\n", system_table_info.get_lambda()); - - size_t i; - for (i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) { - const pinyin_table_info_t * table_info = - system_table_info.get_table_info() + i; - - assert(i == table_info->m_dict_index); - printf("table index:%d\n", table_info->m_dict_index); - - switch(table_info->m_file_type) { - case NOT_USED: - printf("not used.\n"); - break; - - case SYSTEM_FILE: - printf("system file:%s %s %s.\n", table_info->m_table_filename, - table_info->m_system_filename, table_info->m_user_filename); - break; - - case DICTIONARY: - printf("dictionary:%s %s %s.\n", table_info->m_table_filename, - table_info->m_system_filename, table_info->m_user_filename); - break; - - case USER_FILE: - printf("user file:%s.\n", table_info->m_user_filename); - break; - - default: - assert(false); - } - } - - UserTableInfo user_table_info; - retval = user_table_info.is_conform(&system_table_info); - assert(!retval); - - user_table_info.make_conform(&system_table_info); - retval = user_table_info.is_conform(&system_table_info); - assert(retval); - - assert(user_table_info.save("/tmp/user.conf")); - assert(user_table_info.load("/tmp/user.conf")); - - retval = user_table_info.is_conform(&system_table_info); - assert(retval); - - return 0; -} diff --git a/tests/test_chewing.cpp b/tests/test_chewing.cpp deleted file mode 100644 index 5a98e2e..0000000 --- a/tests/test_chewing.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include "zhuyin.h" -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -int main(int argc, char * argv[]){ - zhuyin_context_t * context = - zhuyin_init("../data", "../data"); - - zhuyin_instance_t * instance = zhuyin_alloc_instance(context); - - char* linebuf = NULL; - size_t size = 0; - ssize_t read; - while( (read = getline(&linebuf, &size, stdin)) != -1 ){ - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - zhuyin_parse_more_chewings - (instance, linebuf); - zhuyin_guess_sentence(instance); - - char * sentence = NULL; - zhuyin_get_sentence (instance, &sentence); - if (sentence) - printf("%s\n", sentence); - g_free(sentence); - - zhuyin_train(instance); - zhuyin_reset(instance); - zhuyin_save(context); - } - - zhuyin_free_instance(instance); - - zhuyin_mask_out(context, 0x0, 0x0); - zhuyin_save(context); - zhuyin_fini(context); - - free(linebuf); - return 0; -} diff --git a/tests/test_phrase.cpp b/tests/test_phrase.cpp deleted file mode 100644 index acd58d8..0000000 --- a/tests/test_phrase.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include "zhuyin.h" -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -int main(int argc, char * argv[]){ - zhuyin_context_t * context = - zhuyin_init("../data", "../data"); - - zhuyin_instance_t * instance = zhuyin_alloc_instance(context); - - char* linebuf = NULL; - size_t size = 0; - ssize_t read; - while( (read = getline(&linebuf, &size, stdin)) != -1 ){ - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - zhuyin_phrase_segment(instance, linebuf); - guint len = 0; - zhuyin_get_n_phrase(instance, &len); - - for ( size_t i = 0; i < len; ++i ){ - phrase_token_t token = null_token; - zhuyin_get_phrase_token(instance, i, &token); - - if ( null_token == token ) - continue; - - char * word = NULL; - zhuyin_token_get_phrase(instance, token, NULL, &word); - printf("%s\t", word); - g_free(word); - } - printf("\n"); - - zhuyin_save(context); - } - - zhuyin_free_instance(instance); - - zhuyin_mask_out(context, 0x0, 0x0); - zhuyin_save(context); - zhuyin_fini(context); - - free(linebuf); - return 0; -} diff --git a/tests/test_pinyin.cpp b/tests/test_pinyin.cpp deleted file mode 100644 index 6442dcb..0000000 --- a/tests/test_pinyin.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - - -#include "zhuyin.h" -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -int main(int argc, char * argv[]){ - zhuyin_context_t * context = - zhuyin_init("../data", "../data"); - - pinyin_option_t options = DYNAMIC_ADJUST; - zhuyin_set_options(context, options); - - zhuyin_instance_t * instance = zhuyin_alloc_instance(context); - - char * prefixbuf = NULL; size_t prefixsize = 0; - char * linebuf = NULL; size_t linesize = 0; - ssize_t read; - - while( TRUE ){ - fprintf(stdout, "prefix:"); - fflush(stdout); - - if ((read = getline(&prefixbuf, &prefixsize, stdin)) == -1) - break; - - if ( '\n' == prefixbuf[strlen(prefixbuf) - 1] ) { - prefixbuf[strlen(prefixbuf) - 1] = '\0'; - } - - fprintf(stdout, "pinyin:"); - fflush(stdout); - - if ((read = getline(&linebuf, &linesize, stdin)) == -1) - break; - - if ( '\n' == linebuf[strlen(linebuf) - 1] ) { - linebuf[strlen(linebuf) - 1] = '\0'; - } - - if ( strcmp ( linebuf, "quit" ) == 0) - break; - - zhuyin_parse_more_full_pinyins(instance, linebuf); - zhuyin_guess_sentence_with_prefix(instance, prefixbuf); - zhuyin_guess_candidates_after_cursor(instance, 0); - - guint len = 0; - zhuyin_get_n_candidate(instance, &len); - for (size_t i = 0; i < len; ++i) { - lookup_candidate_t * candidate = NULL; - zhuyin_get_candidate(instance, i, &candidate); - - const char * word = NULL; - zhuyin_get_candidate_string(instance, candidate, &word); - - printf("%s\t", word); - } - printf("\n"); - - zhuyin_train(instance); - zhuyin_reset(instance); - zhuyin_save(context); - } - - zhuyin_free_instance(instance); - - zhuyin_mask_out(context, 0x0, 0x0); - zhuyin_save(context); - zhuyin_fini(context); - - free(prefixbuf); free(linebuf); - return 0; -} diff --git a/tests/tests_helper.h b/tests/tests_helper.h deleted file mode 100644 index 7a05037..0000000 --- a/tests/tests_helper.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2012 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef TESTS_HELPER_H -#define TESTS_HELPER_H - -static bool load_phrase_index(const pinyin_table_info_t * phrase_files, - FacadePhraseIndex * phrase_index){ - MemoryChunk * chunk = NULL; - for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) { - const pinyin_table_info_t * table_info = phrase_files + i; - - if (SYSTEM_FILE != table_info->m_file_type) - continue; - - const char * binfile = table_info->m_system_filename; - - gchar * filename = g_build_filename("..", "..", "data", - binfile, NULL); - chunk = new MemoryChunk; - bool retval = chunk->load(filename); - if (!retval) { - fprintf(stderr, "open %s failed!\n", binfile); - delete chunk; - return false; - } - - phrase_index->load(i, chunk); - g_free(filename); - } - return true; -} - -static bool load_phrase_table(const pinyin_table_info_t * phrase_files, - ChewingLargeTable * chewing_table, - PhraseLargeTable2 * phrase_table, - FacadePhraseIndex * phrase_index){ - for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) { - const pinyin_table_info_t * table_info = phrase_files + i; - - if (SYSTEM_FILE != table_info->m_file_type) - continue; - - const char * tablename = table_info->m_table_filename; - - gchar * filename = g_build_filename("..", "..", "data", - tablename, NULL); - FILE * tablefile = fopen(filename, "r"); - if (NULL == tablefile) { - fprintf(stderr, "open %s failed!\n", tablename); - return false; - } - g_free(filename); - - if (chewing_table) - chewing_table->load_text(tablefile); - fseek(tablefile, 0L, SEEK_SET); - if (phrase_table) - phrase_table->load_text(tablefile); - fseek(tablefile, 0L, SEEK_SET); - if (phrase_index) - phrase_index->load_text(i, tablefile); - fclose(tablefile); - } - return true; -} - -#endif diff --git a/tests/timer.h b/tests/timer.h deleted file mode 100644 index e3ae5a2..0000000 --- a/tests/timer.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * libzhuyin - * Library to deal with zhuyin. - * - * Copyright (C) 2011 Peng Wu <alexepico@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef TIMER_H -#define TIMER_H - -#include <sys/time.h> -#include <stdio.h> -#include <glib.h> - - -static guint32 record_time () -{ - timeval tv; - gettimeofday (&tv, NULL); - return (guint32) tv.tv_sec * 1000000 + tv.tv_usec; -} - -static void print_time (guint32 old_time, guint32 times) -{ - timeval tv; - gettimeofday (&tv, NULL); - - guint32 wasted = (guint32) tv.tv_sec * 1000000 + tv.tv_usec - old_time; - - printf("Spent %d us for %d operations, %f us/op, %f times/s.\n\n" , wasted , times , ((double) wasted)/times , times * 1000000.0/wasted ); -} - - -#endif |