From f41d1fdf83408e042ab07925710a8913bad0c27c Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Tue, 3 Aug 2010 10:42:47 +0800 Subject: import from pinyin. --- src/storage/Makefile.am | 35 + src/storage/ngram.cpp | 283 +++++++ src/storage/ngram.h | 119 +++ src/storage/phrase_index.cpp | 340 ++++++++ src/storage/phrase_index.h | 250 ++++++ src/storage/pinyin_base.cpp | 1425 ++++++++++++++++++++++++++++++++++ src/storage/pinyin_base.h | 728 +++++++++++++++++ src/storage/pinyin_large_table.cpp | 690 ++++++++++++++++ src/storage/pinyin_large_table.h | 178 +++++ src/storage/pinyin_phrase.h | 298 +++++++ src/storage/pinyin_zhuyin_map_data.h | 582 ++++++++++++++ 11 files changed, 4928 insertions(+) create mode 100644 src/storage/Makefile.am create mode 100644 src/storage/ngram.cpp create mode 100644 src/storage/ngram.h create mode 100644 src/storage/phrase_index.cpp create mode 100755 src/storage/phrase_index.h create mode 100644 src/storage/pinyin_base.cpp create mode 100644 src/storage/pinyin_base.h create mode 100644 src/storage/pinyin_large_table.cpp create mode 100755 src/storage/pinyin_large_table.h create mode 100644 src/storage/pinyin_phrase.h create mode 100644 src/storage/pinyin_zhuyin_map_data.h (limited to 'src/storage') diff --git a/src/storage/Makefile.am b/src/storage/Makefile.am new file mode 100644 index 0000000..adf2b5c --- /dev/null +++ b/src/storage/Makefile.am @@ -0,0 +1,35 @@ +## Makefile.am -- Process this file with automake to produce Makefile.in +## Copyright (C) 2007 Peng Wu +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 2, or (at your option) +## any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +INCLUDES = -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/storage \ + @GLIB2_CPPFLAGS@ + +noinst_HEADERS = pinyin_large_table.h \ + pinyin_base.h \ + pinyin_phrase.h \ + phrase_index.h \ + pinyin_zhuyin_map_data.h \ + ngram.h + +noinst_LTLIBRARIES = libstorage.la + +libstorage_la_SOURCES = pinyin_base.cpp \ + pinyin_large_table.cpp \ + phrase_index.cpp \ + ngram.cpp + diff --git a/src/storage/ngram.cpp b/src/storage/ngram.cpp new file mode 100644 index 0000000..7fdc58f --- /dev/null +++ b/src/storage/ngram.cpp @@ -0,0 +1,283 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include "memory_chunk.h" +#include "novel_types.h" +#include "ngram.h" + +struct SingleGramItem{ + phrase_token_t m_token; + guint32 m_freq; +}; + +SingleGram::SingleGram(){ + m_chunk.set_size(sizeof(guint32)); + memset(m_chunk.begin(), 0, sizeof(guint32)); +} + +SingleGram::SingleGram(void * buffer, size_t length){ + m_chunk.set_chunk(buffer, length, NULL); +} + +bool SingleGram::set_total_freq(guint32 m_total){ + char * buf_begin = (char *)m_chunk.begin(); + *((guint32 *)buf_begin) = m_total; + return true; +} + +bool SingleGram::get_total_freq(guint32 & m_total){ + char * buf_begin = (char *)m_chunk.begin(); + m_total = *((guint32 *)buf_begin); + return true; +} + +bool SingleGram::prune(){ +#if 1 + SingleGramItem * begin = (SingleGramItem *) + ((const char *)(m_chunk.begin()) + sizeof(guint32)); + SingleGramItem * end = (SingleGramItem *)m_chunk.end(); + + size_t nitem = 0; + for ( SingleGramItem * cur = begin; cur != end; ++cur){ + cur->m_freq--; + nitem++; + if ( cur->m_freq == 0 ){ + size_t offset = sizeof(guint32) + (cur - begin) + * sizeof(SingleGramItem) ; + m_chunk.remove_content(offset, sizeof(SingleGramItem)); + } + } + guint32 total_freq; + assert(get_total_freq(total_freq)); + assert(set_total_freq(total_freq - nitem)); +#endif + return true; +} + +bool token_less_than(const SingleGramItem & lhs,const SingleGramItem & rhs){ + return lhs.m_token < rhs.m_token; +} + +bool SingleGram::search(/* in */ PhraseIndexRange * range, + /* out */ BigramPhraseArray array){ + const SingleGramItem * begin = (const SingleGramItem *) + ((const char *)(m_chunk.begin()) + sizeof(guint32)); + const SingleGramItem * end = (const SingleGramItem *)m_chunk.end(); + SingleGramItem compare_item; + compare_item.m_token = range->m_range_begin; + const SingleGramItem * cur_item = std_lite::lower_bound(begin, end, compare_item, token_less_than); + + guint32 total_freq; + BigramPhraseItem bigram_item; + assert(get_total_freq(total_freq)); + for ( ; cur_item != end; ++cur_item){ + if ( cur_item->m_token >= range->m_range_end ) + break; + bigram_item.m_token = cur_item->m_token; + bigram_item.m_freq = cur_item->m_freq / (gfloat)total_freq; + g_array_append_val(array, bigram_item); + } + return true; +} + +bool SingleGram::get_freq(/* in */ phrase_token_t token, + /* out */ guint32 & freq){ + freq = 0; + const SingleGramItem * begin = (const SingleGramItem *) + ((const char *)(m_chunk.begin()) + sizeof(guint32)); + const SingleGramItem * end = (const SingleGramItem *)m_chunk.end(); + SingleGramItem compare_item; + compare_item.m_token = token; + const SingleGramItem * cur_item = std_lite::lower_bound(begin, end, compare_item, token_less_than); + + for ( ; cur_item != end; ++cur_item){ + if ( cur_item->m_token > token ) + return false; + if ( cur_item->m_token == token ){ + freq = cur_item -> m_freq; + return true; + } + } + return false; +} + +bool SingleGram::set_freq(/* in */ phrase_token_t token, + guint32 freq){ + SingleGramItem * begin = (SingleGramItem *) + ((const char *)(m_chunk.begin()) + sizeof(guint32)); + SingleGramItem * end = (SingleGramItem *)m_chunk.end(); + SingleGramItem compare_item; + compare_item.m_token = token; + SingleGramItem * cur_item = std_lite::lower_bound(begin, end, compare_item, token_less_than); + + SingleGramItem insert_item; + insert_item.m_token = token; + insert_item.m_freq = freq; + for ( ;cur_item != end; ++cur_item){ + if ( cur_item->m_token > token ){ + size_t offset = sizeof(guint32) + + sizeof(SingleGramItem) * (cur_item - begin); + m_chunk.insert_content(offset, &insert_item, + sizeof(SingleGramItem)); + return true; + } + if ( cur_item->m_token == token ){ + cur_item -> m_freq = freq; + return true; + } + } + m_chunk.insert_content(m_chunk.size(), &insert_item, + sizeof(SingleGramItem)); + return true; +} + + +bool Bigram::attach(const char * systemfile, const char * userfile){ + reset(); + if ( systemfile ){ + int ret = db_create(&m_system, NULL, 0); + if ( ret != 0 ) + assert(false); + + m_system->open(m_system, NULL, systemfile, NULL, + DB_HASH, DB_RDONLY, 0664); + if ( ret != 0) + return false; + } + + if ( userfile ){ + int ret = db_create(&m_user, NULL, 0); + if ( ret != 0 ) + assert(false); + + m_user->open(m_user, NULL, userfile, NULL, DB_HASH, DB_CREATE, 0664); + if ( ret != 0) + return false; + } + return true; +} + +bool Bigram::load(phrase_token_t index, SingleGram * & system_gram, SingleGram * & user_gram){ + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + + system_gram = NULL; user_gram = NULL; + if ( m_system ){ + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + int ret = m_system->get(m_system, NULL, &db_key, &db_data, 0); + if ( ret == 0 ) + system_gram = new SingleGram(db_data.data, db_data.size); + } + if ( m_user ){ + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + int ret = m_user->get(m_user, NULL, &db_key, &db_data, 0); + if ( ret == 0 ) + user_gram = new SingleGram(db_data.data, db_data.size); + } + return true; +} + +bool Bigram::store(phrase_token_t index, SingleGram * user_gram){ + if ( !m_user ) + return false; + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.data = user_gram->m_chunk.begin(); + db_data.size = user_gram->m_chunk.size(); + + int ret = m_user->put(m_user, NULL, &db_key, &db_data, 0); + return ret == 0; +} + +bool Bigram::get_all_items(GArray * system, GArray * user){ + bool retval = false; + g_array_set_size(system, 0); + g_array_set_size(user, 0); + if ( m_system ){ + DBC * cursorp; + DBT key, data; + int ret; + /* Get a cursor */ + m_system->cursor(m_system, NULL, &cursorp, 0); + + /* Initialize our DBTs. */ + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + + /* Iterate over the database, retrieving each record in turn. */ + while ((ret = cursorp->c_get(cursorp, &key, &data, DB_NEXT)) == 0) { + assert(key.size == sizeof(phrase_token_t)); + phrase_token_t * token = (phrase_token_t *)key.data; + g_array_append_val(system, *token); + } + + if (ret != DB_NOTFOUND) { + fprintf(stderr, "system db error, exit!"); + exit(1); + } + + /* Cursors must be closed */ + if (cursorp != NULL) + cursorp->c_close(cursorp); + + retval = true; + } + if ( m_user ){ + DBC * cursorp; + DBT key, data; + int ret; + /* Get a cursor */ + m_user->cursor(m_user, NULL, &cursorp, 0); + + /* Initialize out DBTs. */ + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + + /* Iterate over the database, retrieving each record in turn. */ + while((ret = cursorp->c_get(cursorp, &key, &data, DB_NEXT)) == 0) { + assert(key.size == sizeof(phrase_token_t)); + phrase_token_t * token = (phrase_token_t *) key.data; + g_array_append_val(user, *token); + } + + if (ret != DB_NOTFOUND){ + fprintf(stderr, "user db error, exit!"); + exit(1); + } + + /* Cursor must be closed */ + if ( cursorp != NULL) + cursorp->c_close(cursorp); + + retval = true; + } + return retval; +} diff --git a/src/storage/ngram.h b/src/storage/ngram.h new file mode 100644 index 0000000..39a9ecc --- /dev/null +++ b/src/storage/ngram.h @@ -0,0 +1,119 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef NGRAM_H +#define NGRAM_H + +#include + +namespace novel{ + +class Bigram; + +/* Note: + * When transfer from system ngram to user ngram, + * if user ngram doesn't exist, + * copy total freq from system ngram to user ngram, + * so the total freq exists. + * if item freq don't exist, copy item freq from system to user ngram, + * so the item freq exists. + * if user ngram already exists(always true), increases the total freq, + * if item ngram already exists(always true), increases the freq. + */ + +class SingleGram{ + friend class Bigram; +private: + MemoryChunk m_chunk; + SingleGram(void * buffer, size_t length); +public: + /* Null Constructor */ + SingleGram(); + /* search method */ + /* the array result contains many items */ + bool search(/* in */ PhraseIndexRange * range, + /* out */ BigramPhraseArray array); + + bool get_freq(/* in */ phrase_token_t token, + /* out */ guint32 & freq); + + /* set_freq method + */ + bool set_freq(/* in */ phrase_token_t token, + guint32 freq); + + /* set_total_freq method + * used in user bigram table + */ + bool set_total_freq(guint32 m_total); + + /* get_total_freq method + * used in user bigram table + */ + bool get_total_freq(guint32 & m_total); + + /* prune one method + * only used in training + */ + bool prune(); +}; + +class Bigram{ +private: + DB * m_system; + DB * m_user; +public: + Bigram(){ + m_system = NULL; m_user = NULL; + } + + ~Bigram(){ + reset(); + } + + void reset(){ + if ( m_system ){ + m_system->close(m_system, 0); + m_system = NULL; + } + if ( m_user ){ + m_user->close(m_user, 0); + m_user = NULL; + } + } + + /* attach system and user bi-gram */ + /* when with training systemdb is NULL, only user_gram */ + bool attach(const char * systemfile, const char * userfile); + + bool load(phrase_token_t index, SingleGram * & system_gram, SingleGram * & user_gram); + bool store(phrase_token_t index, SingleGram * user_gram); + /* array of phrase_token_t items, for parameter estimation. */ + bool get_all_items(GArray * system, GArray * user); +}; + +}; + +using namespace novel; + + +#endif diff --git a/src/storage/phrase_index.cpp b/src/storage/phrase_index.cpp new file mode 100644 index 0000000..7dbecb3 --- /dev/null +++ b/src/storage/phrase_index.cpp @@ -0,0 +1,340 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "phrase_index.h" + +bool PhraseItem::set_n_pronunciation(guint8 n_prouns){ + m_chunk.set_content(sizeof(guint8), &n_prouns, sizeof(guint8)); + return true; +} + +bool PhraseItem::get_nth_pronunciation(size_t index, PinyinKey * pinyin, guint32 & freq){ + guint8 phrase_length = get_phrase_length(); + table_offset_t offset = phrase_item_header + phrase_length * sizeof( utf16_t) + index * ( phrase_length * sizeof (PinyinKey) + sizeof(guint32)); + bool retval = m_chunk.get_content(offset, pinyin, phrase_length * sizeof(PinyinKey)); + if ( !retval ) + return retval; + return m_chunk.get_content(offset + phrase_length * sizeof(PinyinKey), &freq , sizeof(guint32)); +} + +void PhraseItem::append_pronunciation(PinyinKey * pinyin, guint32 freq){ + guint8 phrase_length = get_phrase_length(); + set_n_pronunciation(get_n_pronunciation() + 1); + m_chunk.set_content(m_chunk.size(), pinyin, phrase_length * sizeof(PinyinKey)); + m_chunk.set_content(m_chunk.size(), &freq, sizeof(guint32)); +} + +void PhraseItem::remove_nth_pronunciation(size_t index){ + guint8 phrase_length = get_phrase_length(); + set_n_pronunciation(get_n_pronunciation() - 1); + size_t offset = phrase_item_header + phrase_length * sizeof ( utf16_t ) + index * (phrase_length * sizeof (PinyinKey) + sizeof(guint32)); + m_chunk.remove_content(offset, phrase_length * sizeof(PinyinKey) + sizeof(guint32)); +} + +bool PhraseItem::get_phrase_string(utf16_t * phrase){ + guint8 phrase_length = get_phrase_length(); + return m_chunk.get_content(phrase_item_header, phrase, phrase_length * sizeof(utf16_t)); +} + +bool PhraseItem::set_phrase_string(guint8 phrase_length, utf16_t * phrase){ + m_chunk.set_content(0, &phrase_length, sizeof(guint8)); + m_chunk.set_content(phrase_item_header, phrase, phrase_length * sizeof(utf16_t)); + return true; +} + +void PhraseItem::increase_pinyin_possibility(PinyinCustomSettings & custom, + PinyinKey * pinyin_keys, + gint32 delta){ + guint8 phrase_length = get_phrase_length(); + guint8 npron = get_n_pronunciation(); + size_t offset = phrase_item_header + phrase_length * sizeof ( utf16_t ); + char * buf_begin = (char *) m_chunk.begin(); + guint32 total_freq = 0; + for ( int i = 0 ; i < npron ; ++i){ + char * pinyin_begin = buf_begin + offset + + i * ( phrase_length * sizeof(PinyinKey) + sizeof(guint32) ); + guint32 * freq = (guint32 *)(pinyin_begin + phrase_length * sizeof(PinyinKey)); + total_freq += *freq; + if ( 0 == pinyin_compare_with_ambiguities(custom, + (PinyinKey *)pinyin_begin, + pinyin_keys, + phrase_length)){ + //protect against total_freq overflow. + if ( delta > 0 && total_freq > total_freq + delta ) + return; + *freq += delta; + total_freq += delta; + } + } +} + + +guint32 SubPhraseIndex::get_phrase_index_total_freq(){ + return m_total_freq; +} + +bool SubPhraseIndex::add_unigram_frequency(phrase_token_t token, guint32 delta){ + table_offset_t offset; + guint32 freq; + bool result = m_phrase_index.get_content + ((token & PHRASE_MASK) + * sizeof(table_offset_t), &offset, sizeof(table_offset_t)); + + if ( !result) + return result; + + if ( 0 == offset ) + return false; + + result = m_phrase_content.get_content + (offset + sizeof(guint8) + sizeof(guint8), &freq, sizeof(guint32)); + //protect total_freq overflow + if ( delta > 0 && m_total_freq > m_total_freq + delta ) + return false; + freq += delta; + m_total_freq += delta; + return m_phrase_content.set_content(offset + sizeof(guint8) + sizeof(guint8), &freq, sizeof(guint32)); +} + +bool SubPhraseIndex::get_phrase_item(phrase_token_t token, PhraseItem & item){ + table_offset_t offset; + guint8 phrase_length; + guint8 n_prons; + + bool result = m_phrase_index.get_content + ((token & PHRASE_MASK) + * sizeof(table_offset_t), &offset, sizeof(table_offset_t)); + + if ( !result ) + return result; + + if ( 0 == offset ) + return false; + + result = m_phrase_content.get_content(offset, &phrase_length, sizeof(guint8)); + if ( !result ) + return result; + + result = m_phrase_content.get_content(offset+sizeof(guint8), &n_prons, sizeof(guint8)); + if ( !result ) + return result; + + size_t length = phrase_item_header + phrase_length * sizeof ( utf16_t ) + n_prons * ( phrase_length * sizeof (PinyinKey) + sizeof(guint32) ); + item.m_chunk.set_chunk((char *)m_phrase_content.begin() + offset, length, NULL); + return true; +} + +bool SubPhraseIndex::add_phrase_item(phrase_token_t token, PhraseItem * item){ + table_offset_t offset = m_phrase_content.size(); + if ( 0 == offset ) + offset = 8; + m_phrase_content.set_content(offset, item->m_chunk.begin(), item->m_chunk.size()); + m_phrase_index.set_content((token & PHRASE_MASK) + * sizeof(table_offset_t), &offset, sizeof(table_offset_t)); + m_total_freq += item->get_unigram_frequency(); + return true; +} + +bool SubPhraseIndex::remove_phrase_item(phrase_token_t token, PhraseItem * & item){ + table_offset_t offset; + guint8 phrase_length; + guint8 n_prons; + + bool result = m_phrase_index.get_content + ((token & PHRASE_MASK) + * sizeof(table_offset_t), &offset, sizeof(table_offset_t)); + + if ( !result ) + return result; + + if ( 0 == offset ) + return false; + + result = m_phrase_content.get_content(offset, &phrase_length, sizeof(guint8)); + if ( !result ) + return result; + + result = m_phrase_content.get_content(offset+sizeof(guint8), &n_prons, sizeof(guint8)); + if ( !result ) + return result; + + size_t length = phrase_item_header + phrase_length * sizeof ( utf16_t ) + n_prons * ( phrase_length * sizeof (PinyinKey) + sizeof(guint32) ); + item = new PhraseItem; + //implictly copy data from m_chunk_content. + item->m_chunk.set_content(0, (char *) m_phrase_content.begin() + offset, length); + + const table_offset_t zero_const = 0; + m_phrase_index.set_content((token & PHRASE_MASK) + * sizeof(table_offset_t), &zero_const, sizeof(table_offset_t)); + m_total_freq -= item->get_unigram_frequency(); + return true; +} + +bool FacadePhraseIndex::load(guint8 phrase_index, MemoryChunk * chunk){ + SubPhraseIndex * & sub_phrases = m_sub_phrase_indices[phrase_index]; + if ( !sub_phrases ){ + sub_phrases = new SubPhraseIndex; + } + + bool retval = sub_phrases->load(chunk, 0, chunk->size()); + if ( !retval ) + return retval; + m_total_freq += sub_phrases->get_phrase_index_total_freq(); + return retval; +} + +bool FacadePhraseIndex::store(guint8 phrase_index, MemoryChunk * new_chunk){ + table_offset_t end; + SubPhraseIndex * & sub_phrases = m_sub_phrase_indices[phrase_index]; + if ( !sub_phrases ) + return false; + + sub_phrases->store(new_chunk, 0, end); + return true; +} + +bool FacadePhraseIndex::unload(guint8 phrase_index){ + SubPhraseIndex * & sub_phrases = m_sub_phrase_indices[phrase_index]; + if ( !sub_phrases ) + return false; + m_total_freq -= sub_phrases->get_phrase_index_total_freq(); + delete sub_phrases; + sub_phrases = NULL; + return true; +} + +bool SubPhraseIndex::load(MemoryChunk * chunk, + table_offset_t offset, table_offset_t end){ + //save the memory chunk + if ( m_chunk ){ + delete m_chunk; + m_chunk = NULL; + } + m_chunk = chunk; + + char * buf_begin = (char *)chunk->begin(); + chunk->get_content(offset, &m_total_freq, sizeof(guint32)); + offset += sizeof(guint32); + table_offset_t index_one, index_two, index_three; + chunk->get_content(offset, &index_one, sizeof(table_offset_t)); + offset += sizeof(table_offset_t); + chunk->get_content(offset, &index_two, sizeof(table_offset_t)); + offset += sizeof(table_offset_t); + chunk->get_content(offset, &index_three, sizeof(table_offset_t)); + offset += sizeof(table_offset_t); + g_return_val_if_fail(*(buf_begin + offset) == c_separate, FALSE); + g_return_val_if_fail(*(buf_begin + index_two - 1) == c_separate, FALSE); + g_return_val_if_fail(*(buf_begin + index_three - 1) == c_separate, FALSE); + m_phrase_index.set_chunk(buf_begin + index_one, + index_two - 1 - index_one, NULL); + m_phrase_content.set_chunk(buf_begin + index_two, + index_three - 1 - index_two, NULL); + g_return_val_if_fail( index_three <= end, FALSE); + return true; +} + +bool SubPhraseIndex::store(MemoryChunk * new_chunk, + table_offset_t offset, table_offset_t& end){ + new_chunk->set_content(offset, &m_total_freq, sizeof(guint32)); + table_offset_t index = offset + sizeof(guint32); + + offset = index + sizeof(table_offset_t) * 3 ; + new_chunk->set_content(offset, &c_separate, sizeof(char)); + offset += sizeof(char); + + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + index += sizeof(table_offset_t); + new_chunk->set_content(offset, m_phrase_index.begin(), m_phrase_index.size()); + offset += m_phrase_index.size(); + new_chunk->set_content(offset, &c_separate, sizeof(char)); + offset += sizeof(char); + + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + index += sizeof(table_offset_t); + + new_chunk->set_content(offset, m_phrase_content.begin(), m_phrase_content.size()); + offset += m_phrase_content.size(); + new_chunk->set_content(offset, &c_separate, sizeof(char)); + offset += sizeof(char); + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + return true; +} + +bool FacadePhraseIndex::load_text(guint8 phrase_index, FILE * infile){ + SubPhraseIndex * & sub_phrases = m_sub_phrase_indices[phrase_index]; + if ( !sub_phrases ){ + sub_phrases = new SubPhraseIndex; + } + + char pinyin[256]; + char phrase[256]; + phrase_token_t token; + size_t freq; + PhraseItem * item_ptr = new PhraseItem; + phrase_token_t cur_token = 0; + while ( !feof(infile)){ + fscanf(infile, "%s", pinyin); + fscanf(infile, "%s", phrase); + fscanf(infile, "%ld", &token); + fscanf(infile, "%ld", &freq); + if ( feof(infile) ) + break; + + glong written; + utf16_t * phrase_utf16 = g_utf8_to_utf16(phrase, -1, NULL, + &written, NULL); + + if ( 0 == cur_token ){ + cur_token = token; + item_ptr->set_phrase_string(written, phrase_utf16); + } + + if ( cur_token != token ){ + add_phrase_item( cur_token, item_ptr); + delete item_ptr; + item_ptr = new PhraseItem; + cur_token = token; + item_ptr->set_phrase_string(written, phrase_utf16); + } + + PinyinDefaultParser parser; + NullPinyinValidator validator; + PinyinKeyVector keys; + PinyinKeyPosVector poses; + + keys = g_array_new(FALSE, FALSE, sizeof( PinyinKey)); + poses = g_array_new(FALSE, FALSE, sizeof( PinyinKeyPos)); + parser.parse(validator, keys, poses, pinyin); + + assert ( item_ptr->get_phrase_length() == keys->len ); + item_ptr->append_pronunciation((PinyinKey *)keys->data, freq); + + g_array_free(keys, TRUE); + g_array_free(poses, TRUE); + g_free(phrase_utf16); + } + + add_phrase_item( cur_token, item_ptr); + delete item_ptr; + m_total_freq += m_sub_phrase_indices[phrase_index]->get_phrase_index_total_freq(); + return true; +} diff --git a/src/storage/phrase_index.h b/src/storage/phrase_index.h new file mode 100755 index 0000000..e635453 --- /dev/null +++ b/src/storage/phrase_index.h @@ -0,0 +1,250 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef PHRASE_INDEX_H +#define PHRASE_INDEX_H + +#include +#include +#include "novel_types.h" +#include "pinyin_base.h" +#include "pinyin_phrase.h" +#include "memory_chunk.h" + +class PinyinLookup; + +namespace novel{ + +/* Because this is not large, + * Store this in user home directory. + */ + +const int phrase_item_header = sizeof(guint8) + sizeof(guint8) + sizeof(guint32); + +class PhraseItem{ + friend class SubPhraseIndex; +private: + MemoryChunk m_chunk; + bool set_n_pronunciation(guint8 n_prouns); +public: + /* Null Constructor */ + PhraseItem(){ + m_chunk.set_size(phrase_item_header); + memset(m_chunk.begin(), 0, m_chunk.size()); + } + + PhraseItem(MemoryChunk chunk){ + m_chunk = chunk; + assert ( m_chunk.size() >= phrase_item_header); + } + + /* functions */ + guint8 get_phrase_length(){ + char * buf_begin = (char *)m_chunk.begin(); + return (*(guint8 *)buf_begin); + } + + guint8 get_n_pronunciation(){ + char * buf_begin = ( char *) m_chunk.begin(); + return (*(guint8 *)(buf_begin + sizeof(guint8))); + } + + guint32 get_unigram_frequency(){ + char * buf_begin = (char *)m_chunk.begin(); + return (*(guint32 *)(buf_begin + sizeof(guint8) + sizeof(guint8))); + } + + gfloat get_pinyin_possibility(PinyinCustomSettings & custom, + PinyinKey * pinyin_keys){ + guint8 phrase_length = get_phrase_length(); + guint8 npron = get_n_pronunciation(); + size_t offset = phrase_item_header + phrase_length * sizeof ( utf16_t ); + char * buf_begin = (char *)m_chunk.begin(); + guint32 matched = 0, total_freq =0; + for ( int i = 0 ; i < npron ; ++i){ + char * pinyin_begin = buf_begin + offset + + i * ( phrase_length * sizeof(PinyinKey) + sizeof(guint32) ); + guint32 * freq = (guint32 *)(pinyin_begin + phrase_length * sizeof(PinyinKey)); + total_freq += *freq; + if ( 0 == pinyin_compare_with_ambiguities(custom, + (PinyinKey *)pinyin_begin, + pinyin_keys, + phrase_length)){ + matched += *freq; + } + } + // use preprocessor to avoid zero freq, in gen_pinyin_table. + /* + if ( 0 == total_freq ) + return 0.1; + */ + gfloat retval = matched / (gfloat) total_freq; + /* + if ( 0 == retval ) + return 0.03; + */ + return retval; + } + + void increase_pinyin_possibility(PinyinCustomSettings & custom, + PinyinKey * pinyin_keys, + gint32 delta); + + bool get_phrase_string(utf16_t * phrase); + bool set_phrase_string(guint8 phrase_length, utf16_t * phrase); + bool get_nth_pronunciation(size_t index, + /* out */ PinyinKey * pinyin, + /* out */ guint32 & freq); + /* Normally don't change the first pronunciation, + * which decides the token number. + */ + void append_pronunciation(PinyinKey * pinyin, guint32 freq); + void remove_nth_pronunciation(size_t index); +}; + +/* + * In Sub Phrase Index, token == (token & PHRASE_MASK). + */ + +class SubPhraseIndex{ +private: + guint32 m_total_freq; + MemoryChunk m_phrase_index; + MemoryChunk m_phrase_content; + MemoryChunk * m_chunk; +public: + SubPhraseIndex():m_total_freq(0){ + m_chunk = NULL; + } + + ~SubPhraseIndex(){ + reset(); + } + + void reset(){ + if ( m_chunk ){ + delete m_chunk; + m_chunk = NULL; + } + } + + bool load(MemoryChunk * chunk, + table_offset_t offset, table_offset_t end); + bool store(MemoryChunk * new_chunk, + table_offset_t offset, table_offset_t & end); + + /* Zero-gram */ + guint32 get_phrase_index_total_freq(); + bool add_unigram_frequency(phrase_token_t token, guint32 delta); + /* get_phrase_item function can't modify the phrase item, + * but can increment the freq of the special pronunciation. + */ + bool get_phrase_item(phrase_token_t token, PhraseItem & item); + bool add_phrase_item(phrase_token_t token, PhraseItem * item); + /* remove_phrase_item will substract item->get_unigram_frequency() + * from m_total_freq + */ + bool remove_phrase_item(phrase_token_t token, /* out */ PhraseItem * & item); +}; + +class FacadePhraseIndex{ + friend class ::PinyinLookup; +private: + guint32 m_total_freq; + SubPhraseIndex * m_sub_phrase_indices[PHRASE_INDEX_LIBRARY_COUNT]; +public: + FacadePhraseIndex(){ + m_total_freq = 0; + memset(m_sub_phrase_indices, 0, sizeof(m_sub_phrase_indices)); + } + + ~FacadePhraseIndex(){ + for ( size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i){ + if ( m_sub_phrase_indices[i] ){ + delete m_sub_phrase_indices[i]; + m_sub_phrase_indices[i] = NULL; + } + } + } + + /* load/store single sub phrase index, according to the config files. */ + bool load_text(guint8 phrase_index, FILE * infile); + bool load(guint8 phrase_index, MemoryChunk * chunk); + bool store(guint8 phrase_index, MemoryChunk * new_chunk); + bool unload(guint8 phrase_index); + + /* Zero-gram */ + guint32 get_phrase_index_total_freq(){ + return m_total_freq; + } + + bool add_unigram_frequency(phrase_token_t token, guint32 delta){ + guint8 index = PHRASE_INDEX_LIBRARY_INDEX(token); + SubPhraseIndex * sub_phrase = m_sub_phrase_indices[index]; + if ( !sub_phrase ) + return false; + m_total_freq += delta; + return sub_phrase->add_unigram_frequency(token, delta); + } + + /* get_phrase_item function can't modify the phrase item */ + bool get_phrase_item(phrase_token_t token, PhraseItem & item){ + guint8 index = PHRASE_INDEX_LIBRARY_INDEX(token); + SubPhraseIndex * sub_phrase = m_sub_phrase_indices[index]; + if ( !sub_phrase ) + return false; + return sub_phrase->get_phrase_item(token, item); + } + + bool add_phrase_item(phrase_token_t token, PhraseItem * item){ + guint8 index = PHRASE_INDEX_LIBRARY_INDEX(token); + SubPhraseIndex * & sub_phrase = m_sub_phrase_indices[index]; + if ( !sub_phrase ){ + sub_phrase = new SubPhraseIndex; + } + m_total_freq += item->get_unigram_frequency(); + return sub_phrase->add_phrase_item(token, item); + } + + bool remove_phrase_item(phrase_token_t token, PhraseItem * & item){ + guint8 index = PHRASE_INDEX_LIBRARY_INDEX(token); + SubPhraseIndex * & sub_phrase = m_sub_phrase_indices[index]; + if ( !sub_phrase ){ + return false; + } + bool result = sub_phrase->remove_phrase_item(token, item); + if ( !result ) + return result; + m_total_freq -= item->get_unigram_frequency(); + return result; + } +}; + +}; + +using namespace novel; + + + + + +#endif diff --git a/src/storage/pinyin_base.cpp b/src/storage/pinyin_base.cpp new file mode 100644 index 0000000..cffee3c --- /dev/null +++ b/src/storage/pinyin_base.cpp @@ -0,0 +1,1425 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2002,2003,2006 James Su + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "stl_lite.h" +#include "novel_types.h" +#include "pinyin_base.h" +#include "pinyin_phrase.h" +#include "pinyin_large_table.h" + +// Internal data definition + +/** + * struct of pinyin token. + * + * this struct store the informations of a pinyin token + * (an initial or final) + */ +struct PinyinToken +{ + const char *latin; /**< Latin name of the token. */ + const char *zhuyin; /**< Zhuyin name in UTF-8. */ + int latin_len; /**< length of Latin name. */ + int zhuyin_len; /**< length of Chinese name. */ +}; + +/** + * struct to index PinyinToken list. + */ +struct PinyinTokenIndex +{ + int start; + int num; +}; + +static const PinyinToken __pinyin_initials[] = +{ + {"", "", 0, 0}, + {"b", "ㄅ", 1, 1}, + {"c", "ㄘ", 1, 1}, + {"ch","ㄔ", 2, 1}, + {"d", "ㄉ", 1, 1}, + {"f", "ㄈ", 1, 1}, + {"h", "ㄏ", 1, 1}, + {"g", "ㄍ", 1, 1}, + {"j", "ㄐ", 1, 1}, + {"k", "ㄎ", 1, 1}, + {"m", "ㄇ", 1, 1}, + {"n", "ㄋ", 1, 1}, + {"l", "ㄌ", 1, 1}, + {"r", "ㄖ", 1, 1}, + {"p", "ㄆ", 1, 1}, + {"q", "ㄑ", 1, 1}, + {"s", "ㄙ", 1, 1}, + {"sh","ㄕ", 2, 1}, + {"t", "ㄊ", 1, 1}, + {"w", "ㄨ", 1, 1}, //Should be omitted in some case. + {"x", "ㄒ", 1, 1}, + {"y", "ㄧ", 1, 1}, //Should be omitted in some case. + {"z", "ㄗ", 1, 1}, + {"zh","ㄓ", 2, 1} +}; + +static const PinyinToken __pinyin_finals[] = +{ + {"", "", 0, 0}, + {"a", "ㄚ", 1, 1}, + {"ai", "ㄞ", 2, 1}, + {"an", "ㄢ", 2, 1}, + {"ang", "ㄤ", 3, 1}, + {"ao", "ㄠ", 2, 1}, + {"e", "ㄜ", 1, 1}, + {"ea", "ㄝ", 2, 1}, + {"ei", "ㄟ", 2, 1}, + {"en", "ㄣ", 2, 1}, + {"eng", "ㄥ", 3, 1}, + {"er", "ㄦ", 2, 1}, + {"i", "ㄧ", 1, 1}, + {"ia", "ㄧㄚ", 2, 2}, + {"ian", "ㄧㄢ", 3, 2}, + {"iang","ㄧㄤ", 4, 2}, + {"iao", "ㄧㄠ", 3, 2}, + {"ie", "ㄧㄝ", 2, 2}, + {"in", "ㄧㄣ", 2, 2}, + {"ing", "ㄧㄥ", 3, 2}, + {"iong","ㄩㄥ", 4, 2}, + {"iu", "ㄧㄡ", 2, 2}, + {"ng", "ㄣ", 2, 1}, + {"o", "ㄛ", 1, 1}, + {"ong", "ㄨㄥ", 3, 2}, + {"ou", "ㄡ", 2, 1}, + {"u", "ㄨ", 1, 1}, + {"ua", "ㄨㄚ", 2, 2}, + {"uai", "ㄨㄞ", 3, 2}, + {"uan", "ㄨㄢ", 3, 2}, + {"uang","ㄨㄤ", 4, 2}, + {"ue", "ㄩㄝ", 2, 2}, + {"ueng","ㄨㄥ", 4, 2}, + {"ui", "ㄨㄟ", 2, 2}, + {"un", "ㄨㄣ", 2, 2}, + {"uo", "ㄨㄛ", 2, 2}, + {"v", "ㄩ", 1, 1}, + {"van", "ㄩㄢ", 3, 2}, + {"ve", "ㄩㄝ", 2, 2}, + {"vn", "ㄩㄣ", 2, 2} +}; + +static const PinyinToken __pinyin_tones [] = +{ + {"", "", 0, 0}, + {"1", "ˉ", 1, 1}, + {"2", "ˊ", 1, 1}, + {"3", "ˇ", 1, 1}, + {"4", "ˋ", 1, 1}, + {"5", "˙", 1, 1} +}; + +static const PinyinTokenIndex __pinyin_initials_index[] = +{ + //a b c d e f g h i j k l m + {-1,0},{1,1}, {2,2}, {4,1}, {-1,0},{5,1}, {7,1}, {6,1}, {-1,0},{8,1}, {9,1}, {12,1},{10,1}, + //n o p q r s t u v w x y z + {11,1},{-1,0},{14,1},{15,1},{13,1},{16,2},{18,1},{-1,0},{-1,0},{19,1},{20,1},{21,1},{22,2} +}; + +static const PinyinTokenIndex __pinyin_finals_index[] = +{ + //a b c d e f g h i j k l m + {1,5}, {-1,0},{-1,0},{-1,0},{6,6},{-1,0},{-1,0},{-1,0},{12,10},{-1,0},{-1,0},{-1,0},{-1,0}, + //n o p q r s t u v w x y z + {22,1},{23,3},{-1,0},{-1,0},{-1,0},{-1,0},{-1,0},{26,10},{36,4},{-1,0},{-1,0},{-1,0},{-1,0} +}; + + + +static const PinyinInitial __shuang_pin_stone_initial_map [] = +{ + PINYIN_ZeroInitial, // A + PINYIN_Bo, // B + PINYIN_Ci, // C + PINYIN_De, // D + PINYIN_ZeroInitial, // E + PINYIN_Fo, // F + PINYIN_Ge, // G + PINYIN_He, // H + PINYIN_Shi, // I + PINYIN_Ji, // J + PINYIN_Ke, // K + PINYIN_Le, // L + PINYIN_Mo, // M + PINYIN_Ne, // N + PINYIN_ZeroInitial, // O + PINYIN_Po, // P + PINYIN_Qi, // Q + PINYIN_Ri, // R + PINYIN_Si, // S + PINYIN_Te, // T + PINYIN_Chi, // U + PINYIN_Zhi, // V + PINYIN_Wu, // W + PINYIN_Xi, // X + PINYIN_Yi, // Y + PINYIN_Zi, // Z + PINYIN_ZeroInitial, // ; +}; + +static const PinyinFinal __shuang_pin_stone_final_map [][2] = +{ + { PINYIN_A, PINYIN_ZeroFinal }, // A + { PINYIN_Ia, PINYIN_Ua }, // B + { PINYIN_Uan, PINYIN_ZeroFinal }, // C + { PINYIN_Ao, PINYIN_ZeroFinal }, // D + { PINYIN_E, PINYIN_ZeroFinal }, // E + { PINYIN_An, PINYIN_ZeroFinal }, // F + { PINYIN_Ang, PINYIN_ZeroFinal }, // G + { PINYIN_Uang,PINYIN_Iang }, // H + { PINYIN_I, PINYIN_ZeroFinal }, // I + { PINYIN_Ian, PINYIN_ZeroFinal }, // J + { PINYIN_Iao, PINYIN_ZeroFinal }, // K + { PINYIN_In, PINYIN_ZeroFinal }, // L + { PINYIN_Ie, PINYIN_ZeroFinal }, // M + { PINYIN_Iu, PINYIN_ZeroFinal }, // N + { PINYIN_Uo, PINYIN_O }, // O + { PINYIN_Ou, PINYIN_ZeroFinal }, // P + { PINYIN_Ing, PINYIN_Er }, // Q + { PINYIN_En, PINYIN_ZeroFinal }, // R + { PINYIN_Ai, PINYIN_ZeroFinal }, // S + { PINYIN_Ng, PINYIN_Eng }, // T + { PINYIN_U, PINYIN_ZeroFinal }, // U + { PINYIN_V, PINYIN_Ui }, // V + { PINYIN_Ei, PINYIN_ZeroFinal }, // W + { PINYIN_Uai, PINYIN_Ue }, // X + { PINYIN_Ong, PINYIN_Iong }, // Y + { PINYIN_Un, PINYIN_ZeroFinal }, // Z + { PINYIN_ZeroFinal, PINYIN_ZeroFinal }, // ; +}; + + +static const PinyinInitial __shuang_pin_zrm_initial_map [] = +{ + PINYIN_ZeroInitial, // A + PINYIN_Bo, // B + PINYIN_Ci, // C + PINYIN_De, // D + PINYIN_ZeroInitial, // E + PINYIN_Fo, // F + PINYIN_Ge, // G + PINYIN_He, // H + PINYIN_Chi, // I + PINYIN_Ji, // J + PINYIN_Ke, // K + PINYIN_Le, // L + PINYIN_Mo, // M + PINYIN_Ne, // N + PINYIN_ZeroInitial, // O + PINYIN_Po, // P + PINYIN_Qi, // Q + PINYIN_Ri, // R + PINYIN_Si, // S + PINYIN_Te, // T + PINYIN_Shi, // U + PINYIN_Zhi, // V + PINYIN_Wu, // W + PINYIN_Xi, // X + PINYIN_Yi, // Y + PINYIN_Zi, // Z + PINYIN_ZeroInitial, // ; +}; + +static const PinyinFinal __shuang_pin_zrm_final_map [][2] = +{ + { PINYIN_A, PINYIN_ZeroFinal }, // A + { PINYIN_Ou, PINYIN_ZeroFinal }, // B + { PINYIN_Iao, PINYIN_ZeroFinal }, // C + { PINYIN_Uang,PINYIN_Iang }, // D + { PINYIN_E, PINYIN_ZeroFinal }, // E + { PINYIN_En, PINYIN_ZeroFinal }, // F + { PINYIN_Ng, PINYIN_Eng }, // G + { PINYIN_Ang, PINYIN_ZeroFinal }, // H + { PINYIN_I, PINYIN_ZeroFinal }, // I + { PINYIN_An, PINYIN_ZeroFinal }, // J + { PINYIN_Ao, PINYIN_ZeroFinal }, // K + { PINYIN_Ai, PINYIN_ZeroFinal }, // L + { PINYIN_Ian, PINYIN_ZeroFinal }, // M + { PINYIN_In, PINYIN_ZeroFinal }, // N + { PINYIN_Uo, PINYIN_O }, // O + { PINYIN_Un, PINYIN_ZeroFinal }, // P + { PINYIN_Iu, PINYIN_ZeroFinal }, // Q + { PINYIN_Uan, PINYIN_Er }, // R + { PINYIN_Ong, PINYIN_Iong }, // S + { PINYIN_Ue, PINYIN_ZeroFinal }, // T + { PINYIN_U, PINYIN_ZeroFinal }, // U + { PINYIN_V, PINYIN_Ui }, // V + { PINYIN_Ia, PINYIN_Ua }, // W + { PINYIN_Ie, PINYIN_ZeroFinal }, // X + { PINYIN_Ing, PINYIN_Uai }, // Y + { PINYIN_Ei, PINYIN_ZeroFinal }, // Z + { PINYIN_ZeroFinal, PINYIN_ZeroFinal }, // ; +}; + + +static const PinyinInitial __shuang_pin_ms_initial_map [] = +{ + PINYIN_ZeroInitial, // A + PINYIN_Bo, // B + PINYIN_Ci, // C + PINYIN_De, // D + PINYIN_ZeroInitial, // E + PINYIN_Fo, // F + PINYIN_Ge, // G + PINYIN_He, // H + PINYIN_Chi, // I + PINYIN_Ji, // J + PINYIN_Ke, // K + PINYIN_Le, // L + PINYIN_Mo, // M + PINYIN_Ne, // N + PINYIN_ZeroInitial, // O + PINYIN_Po, // P + PINYIN_Qi, // Q + PINYIN_Ri, // R + PINYIN_Si, // S + PINYIN_Te, // T + PINYIN_Shi, // U + PINYIN_Zhi, // V + PINYIN_Wu, // W + PINYIN_Xi, // X + PINYIN_Yi, // Y + PINYIN_Zi, // Z + PINYIN_ZeroInitial, // ; +}; + +static const PinyinFinal __shuang_pin_ms_final_map [][2] = +{ + { PINYIN_A, PINYIN_ZeroFinal }, // A + { PINYIN_Ou, PINYIN_ZeroFinal }, // B + { PINYIN_Iao, PINYIN_ZeroFinal }, // C + { PINYIN_Uang,PINYIN_Iang }, // D + { PINYIN_E, PINYIN_ZeroFinal }, // E + { PINYIN_En, PINYIN_ZeroFinal }, // F + { PINYIN_Ng, PINYIN_Eng }, // G + { PINYIN_Ang, PINYIN_ZeroFinal }, // H + { PINYIN_I, PINYIN_ZeroFinal }, // I + { PINYIN_An, PINYIN_ZeroFinal }, // J + { PINYIN_Ao, PINYIN_ZeroFinal }, // K + { PINYIN_Ai, PINYIN_ZeroFinal }, // L + { PINYIN_Ian, PINYIN_ZeroFinal }, // M + { PINYIN_In, PINYIN_ZeroFinal }, // N + { PINYIN_Uo, PINYIN_O }, // O + { PINYIN_Un, PINYIN_ZeroFinal }, // P + { PINYIN_Iu, PINYIN_ZeroFinal }, // Q + { PINYIN_Uan, PINYIN_Er }, // R + { PINYIN_Ong, PINYIN_Iong }, // S + { PINYIN_Ue, PINYIN_ZeroFinal }, // T + { PINYIN_U, PINYIN_ZeroFinal }, // U + { PINYIN_V, PINYIN_Ui }, // V + { PINYIN_Ia, PINYIN_Ua }, // W + { PINYIN_Ie, PINYIN_ZeroFinal }, // X + { PINYIN_Uai, PINYIN_V }, // Y + { PINYIN_Ei, PINYIN_ZeroFinal }, // Z + { PINYIN_Ing, PINYIN_ZeroFinal }, // ; +}; + + +static const PinyinInitial __shuang_pin_ziguang_initial_map [] = +{ + PINYIN_Chi, // A + PINYIN_Bo, // B + PINYIN_Ci, // C + PINYIN_De, // D + PINYIN_ZeroInitial, // E + PINYIN_Fo, // F + PINYIN_Ge, // G + PINYIN_He, // H + PINYIN_Shi, // I + PINYIN_Ji, // J + PINYIN_Ke, // K + PINYIN_Le, // L + PINYIN_Mo, // M + PINYIN_Ne, // N + PINYIN_ZeroInitial, // O + PINYIN_Po, // P + PINYIN_Qi, // Q + PINYIN_Ri, // R + PINYIN_Si, // S + PINYIN_Te, // T + PINYIN_Zhi, // U + PINYIN_ZeroInitial, // V + PINYIN_Wu, // W + PINYIN_Xi, // X + PINYIN_Yi, // Y + PINYIN_Zi, // Z + PINYIN_ZeroInitial, // ; +}; + +static const PinyinFinal __shuang_pin_ziguang_final_map [][2] = +{ + { PINYIN_A, PINYIN_ZeroFinal }, // A + { PINYIN_Iao, PINYIN_ZeroFinal }, // B + { PINYIN_Ing, PINYIN_ZeroFinal }, // C + { PINYIN_Ie, PINYIN_ZeroFinal }, // D + { PINYIN_E, PINYIN_ZeroFinal }, // E + { PINYIN_Ian, PINYIN_ZeroFinal }, // F + { PINYIN_Uang,PINYIN_Iang }, // G + { PINYIN_Ong, PINYIN_Iong }, // H + { PINYIN_I, PINYIN_ZeroFinal }, // I + { PINYIN_Iu, PINYIN_Er }, // J + { PINYIN_Ei, PINYIN_ZeroFinal }, // K + { PINYIN_Uan, PINYIN_ZeroFinal }, // L + { PINYIN_Un, PINYIN_ZeroFinal }, // M + { PINYIN_Ui, PINYIN_Ue }, // N + { PINYIN_Uo, PINYIN_O }, // O + { PINYIN_Ai, PINYIN_ZeroFinal }, // P + { PINYIN_Ao, PINYIN_ZeroFinal }, // Q + { PINYIN_An, PINYIN_ZeroFinal }, // R + { PINYIN_Ang, PINYIN_ZeroFinal }, // S + { PINYIN_Ng, PINYIN_Eng }, // T + { PINYIN_U, PINYIN_ZeroFinal }, // U + { PINYIN_V, PINYIN_ZeroFinal }, // V + { PINYIN_En, PINYIN_ZeroFinal }, // W + { PINYIN_Ia, PINYIN_Ua }, // X + { PINYIN_In, PINYIN_Uai }, // Y + { PINYIN_Ou, PINYIN_ZeroFinal }, // Z + { PINYIN_ZeroFinal, PINYIN_ZeroFinal }, // ; +}; + + +static const PinyinInitial __shuang_pin_abc_initial_map [] = +{ + PINYIN_Zhi, // A + PINYIN_Bo, // B + PINYIN_Ci, // C + PINYIN_De, // D + PINYIN_Chi, // E + PINYIN_Fo, // F + PINYIN_Ge, // G + PINYIN_He, // H + PINYIN_ZeroInitial, // I + PINYIN_Ji, // J + PINYIN_Ke, // K + PINYIN_Le, // L + PINYIN_Mo, // M + PINYIN_Ne, // N + PINYIN_ZeroInitial, // O + PINYIN_Po, // P + PINYIN_Qi, // Q + PINYIN_Ri, // R + PINYIN_Si, // S + PINYIN_Te, // T + PINYIN_ZeroInitial, // U + PINYIN_Shi, // V + PINYIN_Wu, // W + PINYIN_Xi, // X + PINYIN_Yi, // Y + PINYIN_Zi, // Z + PINYIN_ZeroInitial, // ; +}; + +static const PinyinFinal __shuang_pin_abc_final_map [][2] = +{ + { PINYIN_A, PINYIN_ZeroFinal }, // A + { PINYIN_Ou, PINYIN_ZeroFinal }, // B + { PINYIN_In, PINYIN_Uai }, // C + { PINYIN_Ia, PINYIN_Ua }, // D + { PINYIN_E, PINYIN_ZeroFinal }, // E + { PINYIN_En, PINYIN_ZeroFinal }, // F + { PINYIN_Ng, PINYIN_Eng }, // G + { PINYIN_Ang, PINYIN_ZeroFinal }, // H + { PINYIN_I, PINYIN_ZeroFinal }, // I + { PINYIN_An, PINYIN_ZeroFinal }, // J + { PINYIN_Ao, PINYIN_ZeroFinal }, // K + { PINYIN_Ai, PINYIN_ZeroFinal }, // L + { PINYIN_Ui, PINYIN_Ue }, // M + { PINYIN_Un, PINYIN_ZeroFinal }, // N + { PINYIN_Uo, PINYIN_O }, // O + { PINYIN_Uan, PINYIN_ZeroFinal }, // P + { PINYIN_Ei, PINYIN_ZeroFinal }, // Q + { PINYIN_Iu, PINYIN_Er }, // R + { PINYIN_Ong, PINYIN_Iong }, // S + { PINYIN_Uang,PINYIN_Iang }, // T + { PINYIN_U, PINYIN_ZeroFinal }, // U + { PINYIN_V, PINYIN_ZeroFinal }, // V + { PINYIN_Ian, PINYIN_ZeroFinal }, // W + { PINYIN_Ie, PINYIN_ZeroFinal }, // X + { PINYIN_Ing, PINYIN_ZeroFinal }, // Y + { PINYIN_Iao, PINYIN_ZeroFinal }, // Z + { PINYIN_ZeroFinal, PINYIN_ZeroFinal }, // ; +}; + + +static const PinyinInitial __shuang_pin_liushi_initial_map [] = +{ + PINYIN_ZeroInitial, // A + PINYIN_Bo, // B + PINYIN_Ci, // C + PINYIN_De, // D + PINYIN_ZeroInitial, // E + PINYIN_Fo, // F + PINYIN_Ge, // G + PINYIN_He, // H + PINYIN_Chi, // I + PINYIN_Ji, // J + PINYIN_Ke, // K + PINYIN_Le, // L + PINYIN_Mo, // M + PINYIN_Ne, // N + PINYIN_ZeroInitial, // O + PINYIN_Po, // P + PINYIN_Qi, // Q + PINYIN_Ri, // R + PINYIN_Si, // S + PINYIN_Te, // T + PINYIN_Shi, // U + PINYIN_Zhi, // V + PINYIN_Wu, // W + PINYIN_Xi, // X + PINYIN_Yi, // Y + PINYIN_Zi, // Z + PINYIN_ZeroInitial, // ; +}; + +static const PinyinFinal __shuang_pin_liushi_final_map [][2] = +{ + { PINYIN_A, PINYIN_ZeroFinal }, // A + { PINYIN_Ao, PINYIN_ZeroFinal }, // B + { PINYIN_Ang, PINYIN_ZeroFinal }, // C + { PINYIN_Uan, PINYIN_ZeroFinal }, // D + { PINYIN_E, PINYIN_ZeroFinal }, // E + { PINYIN_An, PINYIN_ZeroFinal }, // F + { PINYIN_Ong, PINYIN_Iong }, // G + { PINYIN_Ui, PINYIN_Ue }, // H + { PINYIN_I, PINYIN_ZeroFinal }, // I + { PINYIN_Ia, PINYIN_Ua }, // J + { PINYIN_Un, PINYIN_ZeroFinal }, // K + { PINYIN_Iu, PINYIN_ZeroFinal }, // L + { PINYIN_In, PINYIN_ZeroFinal }, // M + { PINYIN_Uang,PINYIN_Iang }, // N + { PINYIN_Uo, PINYIN_O }, // O + { PINYIN_Ng, PINYIN_Eng }, // P + { PINYIN_Ing, PINYIN_ZeroFinal }, // Q + { PINYIN_Ou, PINYIN_Er }, // R + { PINYIN_Ai, PINYIN_ZeroFinal }, // S + { PINYIN_Ian, PINYIN_ZeroFinal }, // T + { PINYIN_U, PINYIN_ZeroFinal }, // U + { PINYIN_V, PINYIN_En }, // V + { PINYIN_Ei, PINYIN_ZeroFinal }, // W + { PINYIN_Ie, PINYIN_ZeroFinal }, // X + { PINYIN_Uai, PINYIN_ZeroFinal }, // Y + { PINYIN_Iao, PINYIN_ZeroFinal }, // Z + { PINYIN_ZeroFinal, PINYIN_ZeroFinal }, // ; +}; + +static const size_t __zhuyin_zhuyin_map_start_char = 0x3105; +static const size_t __zhuyin_zhuyin_map_tone_start_idx = 37; +static const PinyinKey __zhuyin_zhuyin_map [][3] = +{ + {PinyinKey(PINYIN_Bo),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Po),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Mo),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Fo),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_De),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Te),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Ne),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Le),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Ge),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Ke),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_He),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Ji),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Qi),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Xi),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Zhi),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Chi),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Shi),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Ri),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Zi),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Ci),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_Si),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_A),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_O),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_E),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Ea),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Ai),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Ei),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Ao),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Ou),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_An),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_En),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Ang),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Eng),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_Er),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_I),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_U),PinyinKey(),PinyinKey()}, + {PinyinKey(PINYIN_ZeroInitial,PINYIN_V),PinyinKey(),PinyinKey()}, +}; + +static const size_t __zhuyin_map_start_char = 0x20; +#include "pinyin_zhuyin_map_data.h" + +static const PinyinKey (*__zhuyin_maps []) [3] = { + __zhuyin_zhuyin_map, + __zhuyin_standard_map, + __zhuyin_hsu_map, + __zhuyin_ibm_map, + __zhuyin_gin_yieh_map, + __zhuyin_et_map, + __zhuyin_et26_map, + 0 +}; + + +////////////////////////////////////////////////////////////////////////////// +// implementation of PinyinCustomSettings + +PinyinCustomSettings::PinyinCustomSettings () + : use_incomplete (true) +{ + for (size_t i=0; i<=PINYIN_AmbLast; ++i) + use_ambiguities [i] = false; +} + +////////////////////////////////////////////////////////////////////////////// +// implementation of PinyinKey + +const guint16 PinyinKey::min_value = 0; +const guint16 PinyinKey::max_value = PINYIN_Number_Of_Initials * PINYIN_Number_Of_Finals * PINYIN_Number_Of_Tones - 1; + +const char* +PinyinKey::get_initial_string () const +{ + return __pinyin_initials [m_initial].latin; +} + +const char* +PinyinKey::get_initial_zhuyin_string () const +{ + if ((m_initial == PINYIN_Wu && m_final == PINYIN_U) || + (m_initial == PINYIN_Yi && + (m_final == PINYIN_I || m_final == PINYIN_In || m_final == PINYIN_Ing || m_final == PINYIN_Ong || + m_final == PINYIN_U || m_final == PINYIN_Ue || m_final == PINYIN_Uan || m_final == PINYIN_Un))) + return ""; + + return __pinyin_initials [m_initial].zhuyin; +} + +const char* +PinyinKey::get_final_string () const +{ + return __pinyin_finals [m_final].latin; +} + +const char* +PinyinKey::get_final_zhuyin_string () const +{ + if (m_initial == PINYIN_Yi && m_final == PINYIN_Ong) { + return __pinyin_finals [PINYIN_Iong].zhuyin; + } else if (m_initial == PINYIN_Yi || m_initial == PINYIN_Ji || m_initial == PINYIN_Qi || m_initial == PINYIN_Xi) { + switch (m_final) { + case PINYIN_U: + return __pinyin_finals [PINYIN_V].zhuyin; + case PINYIN_Ue: + return __pinyin_finals [PINYIN_Ve].zhuyin; + case PINYIN_Uan: + return __pinyin_finals [PINYIN_Van].zhuyin; + case PINYIN_Un: + return __pinyin_finals [PINYIN_Vn].zhuyin; + } + if (m_initial == PINYIN_Yi && m_final == PINYIN_E) + return __pinyin_finals [PINYIN_Ea].zhuyin; + } else if ((m_initial == PINYIN_Ne || m_initial == PINYIN_Le) && m_final == PINYIN_Ue) { + return __pinyin_finals [PINYIN_Ve].zhuyin; + } else if ((m_initial == PINYIN_Zhi || m_initial == PINYIN_Chi || m_initial == PINYIN_Shi || + m_initial == PINYIN_Zi || m_initial == PINYIN_Ci || m_initial == PINYIN_Si || + m_initial == PINYIN_Ri) && m_final == PINYIN_I) { + return ""; + } + + return __pinyin_finals [m_final].zhuyin; +} + +const char* +PinyinKey::get_tone_string () const +{ + return __pinyin_tones [m_tone].latin; +} + +const char* +PinyinKey::get_tone_zhuyin_string () const +{ + return __pinyin_tones [m_tone].zhuyin; +} + +const char * +PinyinKey::get_key_string () const +{ + char key [16]; + g_snprintf (key, 15, "%s%s%s", get_initial_string(), get_final_string(), get_tone_string ()); + + return g_strdup(key); +} + +const char * +PinyinKey::get_key_zhuyin_string () const +{ + char key [32]; + g_snprintf (key, 31, "%s%s%s", get_initial_zhuyin_string(), get_final_zhuyin_string(), get_tone_zhuyin_string ()); + + return g_strdup (key); +} + +int +PinyinKey::set (const PinyinValidator &validator, const char *str, int len) +{ + if (!str || ! (*str)) + return 0; + + PinyinDefaultParser parser; + + return parser.parse_one_key (validator, *this, str, len); +} + +////////////////////////////////////////////////////////////////////////////// +// implementation of PinyinValidator +BitmapPinyinValidator::BitmapPinyinValidator (const PinyinLargeTable *table) +{ + initialize (table); +} + +void +BitmapPinyinValidator::initialize (const PinyinLargeTable *table) +{ + memset (m_bitmap, 0, sizeof (m_bitmap)); + + if (!table) return; + + for (guint16 val=0; val<=PinyinKey::max_value; ++val) + if (!table->has_key (PinyinKey (val))) + m_bitmap [val >> 3] |= (1 << (val % 8)); +} + +bool +BitmapPinyinValidator::operator () (PinyinKey key) const +{ + if (key.is_empty ()) return false; + + guint16 val = key.get_value (); + + return (m_bitmap [ val >> 3 ] & (1 << (val % 8))) == 0; +} + +////////////////////////////////////////////////////////////////////////////// +// implementation of PinyinParser +PinyinParser::~PinyinParser () +{ +} + +struct PinyinReplaceRulePair +{ + PinyinInitial initial; + PinyinFinal final; + PinyinInitial new_initial; + PinyinFinal new_final; +}; + +class PinyinReplaceRulePairLessThan +{ +public: + bool operator () (const PinyinReplaceRulePair &lhs, const PinyinReplaceRulePair &rhs) const { + if (lhs.initial < rhs.initial) return true; + if (lhs.initial > rhs.initial) return false; + return lhs.final < rhs.final; + } +}; + +void +PinyinParser::normalize (PinyinKey &key) +{ + static const PinyinReplaceRulePair rules [] = + { +#if 0 + {PINYIN_ZeroInitial, PINYIN_I, PINYIN_Yi, PINYIN_I}, + {PINYIN_ZeroInitial, PINYIN_Ia, PINYIN_Yi, PINYIN_A}, + {PINYIN_ZeroInitial, PINYIN_Ian, PINYIN_Yi, PINYIN_An}, + {PINYIN_ZeroInitial, PINYIN_Iang, PINYIN_Yi, PINYIN_Ang}, + {PINYIN_ZeroInitial, PINYIN_Iao, PINYIN_Yi, PINYIN_Ao}, + {PINYIN_ZeroInitial, PINYIN_Ie, PINYIN_Yi, PINYIN_E}, + {PINYIN_ZeroInitial, PINYIN_In, PINYIN_Yi, PINYIN_In}, + {PINYIN_ZeroInitial, PINYIN_Ing, PINYIN_Yi, PINYIN_Ing}, + {PINYIN_ZeroInitial, PINYIN_Iong, PINYIN_Yi, PINYIN_Ong}, + {PINYIN_ZeroInitial, PINYIN_Iu, PINYIN_Yi, PINYIN_Ou}, + {PINYIN_ZeroInitial, PINYIN_U, PINYIN_Wu, PINYIN_U}, + {PINYIN_ZeroInitial, PINYIN_Ua, PINYIN_Wu, PINYIN_A}, + {PINYIN_ZeroInitial, PINYIN_Uai, PINYIN_Wu, PINYIN_Ai}, + {PINYIN_ZeroInitial, PINYIN_Uan, PINYIN_Wu, PINYIN_An}, + {PINYIN_ZeroInitial, PINYIN_Uang, PINYIN_Wu, PINYIN_Ang}, + {PINYIN_ZeroInitial, PINYIN_Ue, PINYIN_Wu, PINYIN_E}, + {PINYIN_ZeroInitial, PINYIN_Ueng, PINYIN_Wu, PINYIN_Eng}, + {PINYIN_ZeroInitial, PINYIN_Ui, PINYIN_Wu, PINYIN_Ei}, + {PINYIN_ZeroInitial, PINYIN_Un, PINYIN_Wu, PINYIN_En}, + {PINYIN_ZeroInitial, PINYIN_Uo, PINYIN_Wu, PINYIN_O}, + {PINYIN_ZeroInitial, PINYIN_V, PINYIN_Yi, PINYIN_U}, + {PINYIN_ZeroInitial, PINYIN_Van, PINYIN_Yi, PINYIN_Uan}, + {PINYIN_ZeroInitial, PINYIN_Ve, PINYIN_Yi, PINYIN_Ue}, + {PINYIN_ZeroInitial, PINYIN_Vn, PINYIN_Yi, PINYIN_Un}, +#endif + {PINYIN_Ji, PINYIN_V, PINYIN_Ji, PINYIN_U}, + {PINYIN_Ji, PINYIN_Van, PINYIN_Ji, PINYIN_Uan}, + {PINYIN_Ji, PINYIN_Ve, PINYIN_Ji, PINYIN_Ue}, + {PINYIN_Ji, PINYIN_Vn, PINYIN_Ji, PINYIN_Un}, + {PINYIN_Ne, PINYIN_Ve, PINYIN_Ne, PINYIN_Ue}, + {PINYIN_Le, PINYIN_Ve, PINYIN_Le, PINYIN_Ue}, + {PINYIN_Qi, PINYIN_V, PINYIN_Qi, PINYIN_U}, + {PINYIN_Qi, PINYIN_Van, PINYIN_Qi, PINYIN_Uan}, + {PINYIN_Qi, PINYIN_Ve, PINYIN_Qi, PINYIN_Ue}, + {PINYIN_Qi, PINYIN_Vn, PINYIN_Qi, PINYIN_Un}, + {PINYIN_Xi, PINYIN_V, PINYIN_Xi, PINYIN_U}, + {PINYIN_Xi, PINYIN_Van, PINYIN_Xi, PINYIN_Uan}, + {PINYIN_Xi, PINYIN_Ve, PINYIN_Xi, PINYIN_Ue}, + {PINYIN_Xi, PINYIN_Vn, PINYIN_Xi, PINYIN_Un} + }; + static const PinyinReplaceRulePair *rules_start = rules; + static const PinyinReplaceRulePair *rules_end = rules + sizeof(rules)/sizeof(PinyinReplaceRulePair); + + PinyinReplaceRulePair kp; + + kp.initial = key.get_initial (); + kp.final = key.get_final (); + + const PinyinReplaceRulePair *p = std_lite::lower_bound (rules_start, rules_end, kp, PinyinReplaceRulePairLessThan ()); + + if (p->initial == kp.initial && p->final == kp.final) { + key.set_initial (p->new_initial); + key.set_final (p->new_final); + } +} + +//============== Internal functions used by PinyinDefaultParser ============== +static int +__default_parser_parse_initial (PinyinInitial &initial, const char *str, int len) +{ + int lastlen = 0; + + initial = PINYIN_ZeroInitial; + + if (str && *str >= 'a' && *str <= 'z') { + int start = __pinyin_initials_index [*str - 'a'].start; + int end = __pinyin_initials_index [*str - 'a'].num + start; + + if (start > 0) { + for (int i = start; i < end; ++i) { + if ((len < 0 || len >= __pinyin_initials [i].latin_len) && __pinyin_initials [i].latin_len >= lastlen) { + int j; + for (j = 1; j < __pinyin_initials [i].latin_len; ++j) { + if (str [j] != __pinyin_initials [i].latin [j]) + break; + } + if (j == __pinyin_initials [i].latin_len) { + initial = static_cast(i); + lastlen = __pinyin_initials [i].latin_len; + } + } + } + } + } + + return lastlen; +} +static int +__default_parser_parse_final (PinyinFinal &final, const char *str, int len) +{ + int lastlen = 0; + + final = PINYIN_ZeroFinal; + + if (str && *str >= 'a' && *str <= 'z') { + int start = __pinyin_finals_index [*str - 'a'].start; + int end = __pinyin_finals_index [*str - 'a'].num + start; + + if (start > 0) { + for (int i = start; i < end; ++i) { + if ((len < 0 || len >= __pinyin_finals [i].latin_len) && __pinyin_finals [i].latin_len >= lastlen) { + int j; + for (j = 1; j < __pinyin_finals [i].latin_len; ++j) { + if (str [j] != __pinyin_finals [i].latin [j]) + break; + } + if (j == __pinyin_finals [i].latin_len) { + final = static_cast(i); + lastlen = __pinyin_finals [i].latin_len; + } + } + } + } + } + + return lastlen; +} +static int +__default_parser_parse_tone (PinyinTone &tone, const char *str, int len) +{ + tone = PINYIN_ZeroTone; + + if (str && (len >= 1 || len < 0)) { + int kt = (*str) - '0'; + if (kt >= PINYIN_First && kt <= PINYIN_LastTone) { + tone = static_cast(kt); + return 1; + } + } + return 0; +} + +static int +__default_parser_parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char *str, int len = -1) +{ + int initial_len = 0; + int final_len = 0; + int tone_len = 0; + + const char *ptr; + + PinyinInitial initial; + PinyinFinal final; + PinyinTone tone; + + key.clear (); + + if (!str || !len) return 0; + + if (len < 0) len = strlen (str); + + while (len > 0) { + ptr = str; + + initial = PINYIN_ZeroInitial; + final = PINYIN_ZeroFinal; + tone = PINYIN_ZeroTone; + + final_len = __default_parser_parse_final (final, ptr, len); + ptr += final_len; + len -= final_len; + + // An initial is present + if (final == PINYIN_ZeroFinal) { + initial_len = __default_parser_parse_initial (initial, ptr, len); + ptr += initial_len; + len -= initial_len; + if (len){ + final_len = __default_parser_parse_final (final, ptr, len); + ptr += final_len; + len -= final_len; + } + } + + if (len) + tone_len = __default_parser_parse_tone (tone, ptr, len); + + key.set (initial, final, tone); + + PinyinParser::normalize (key); + + // A valid key was found, return. + if (validator (key)) break; + + // The key is invalid, reduce the len and find again. + len = initial_len + final_len + tone_len - 1; + + initial_len = final_len = tone_len = 0; + + key.clear (); + } + + len = initial_len + final_len + tone_len; + + return len; +} + +struct DefaultParserCacheElement +{ + PinyinKey key; + PinyinKeyPos pos; + int num_keys; + int parsed_len; + int next_start; +}; + +typedef GArray* DefaultParserCache; /* Array of DefaultParserCacheElement */ + +static int +__default_parser_parse_recursive (const PinyinValidator &validator, + DefaultParserCache &cache, + int &real_start, + int &num_keys, + const char *str, + int len, + int start) +{ + if (*str == 0 || len == 0) return 0; + + int used_len = 0; + + real_start = 0; + num_keys = 0; + + if (*str == '\'' || *str == ' ') { + ++used_len; + ++str; + ++start; + --len; + } + + if (!isalpha (*str) || !len) + return 0; + + real_start = start; + + // The best keys start from this position have been found, just return the result. + DefaultParserCacheElement* element = &g_array_index + (cache, DefaultParserCacheElement, start); + + + if (element->num_keys >=0) { + num_keys = element->num_keys; + return element->parsed_len; + } + + PinyinKey first_key; + PinyinKey best_first_key; + PinyinKeyPos pos; + + int first_len = 0; + int best_first_len = 0; + + int remained_len = 0; + int best_remained_len = 0; + + int remained_keys = 0; + int best_remained_keys = 0; + + int remained_start = 0; + int best_remained_start = 0; + + first_len = __default_parser_parse_one_key (validator, first_key, str, len); + + if (!first_len) { + element = &g_array_index(cache, DefaultParserCacheElement, start); + + element->key = PinyinKey (); + element->num_keys = 0; + element->parsed_len = 0; + element->next_start = start; + return 0; + } + + best_first_key = first_key; + best_first_len = first_len; + + if (len > first_len) { + char ch1 = str [first_len -1]; + char ch2 = str [first_len]; + + best_remained_len = __default_parser_parse_recursive (validator, + cache, + best_remained_start, + best_remained_keys, + str + first_len, + len - first_len, + start + first_len); + + // For those keys which the last char is 'g' or 'n' or 'r', try put the end char into the next key. + if (first_len > 1 && + (((ch1=='g' || ch1=='n' || ch1=='r') && (ch2=='a' || ch2=='e' || ch2=='i' || ch2=='o' || ch2=='u' || ch2=='v')) || + ((ch1=='a' || ch1=='e' || ch1=='o') && (ch2=='i' || ch2=='n' || ch2=='o' || ch2=='r' || ch2=='u')))) { + + first_len = __default_parser_parse_one_key (validator, first_key, str, first_len - 1); + + if (first_len) { + remained_len = __default_parser_parse_recursive (validator, + cache, + remained_start, + remained_keys, + str + first_len, + len - first_len, + start + first_len); + + + DefaultParserCacheElement* best_remained_element = &g_array_index + (cache, DefaultParserCacheElement, best_remained_start); + + // A better seq was found. + if (remained_len != 0 && (remained_len + first_len) >= (best_remained_len + best_first_len) && + (remained_keys <= best_remained_keys || best_remained_keys == 0)) { +#if 0 + if ((remained_len + first_len) > (best_remained_len + best_first_len) || + remained_keys < best_remained_keys || + best_remained_element->key.get_final () == PINYIN_ZeroFinal || + best_remained_element->key.get_initial () == PINYIN_Wu || + best_remained_element->key.get_initial () == PINYIN_Yi) { +#endif + best_first_len = first_len; + best_first_key = first_key; + best_remained_len = remained_len; + best_remained_keys = remained_keys; + best_remained_start = remained_start; +#if 0 + } +#endif + } + } + } + } + + num_keys = best_remained_keys + 1; + + + element = &g_array_index + (cache, DefaultParserCacheElement, start); + + pos.set_pos(start); + pos.set_length(best_first_len); + + element->key = best_first_key; + element->pos = pos; + element->num_keys = num_keys; + element->parsed_len = used_len + best_first_len + best_remained_len; + element->next_start = best_remained_start; + + return element->parsed_len; +} +//============================================================================ + +PinyinDefaultParser::~PinyinDefaultParser () +{ +} + +int +PinyinDefaultParser::parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char *str, int len) const +{ + return __default_parser_parse_one_key (validator, key, str, len); +} + +int +PinyinDefaultParser::parse (const PinyinValidator &validator, PinyinKeyVector & keys, PinyinKeyPosVector & poses, const char *str, int len) const +{ + g_array_set_size(keys, 0); + g_array_set_size(poses, 0); + + if (!str || !len) return 0; + + if (len < 0) len = strlen (str); + + DefaultParserCacheElement elm; + + elm.num_keys = -1L; + elm.parsed_len = 0; + elm.next_start = 0; + + DefaultParserCache cache = g_array_new (FALSE, TRUE, sizeof (DefaultParserCacheElement)); + g_array_set_size(cache, len); + for ( size_t index = 0 ; index < len ; index++){ + DefaultParserCacheElement * element = + &g_array_index(cache,DefaultParserCacheElement, index); + *element = elm; + } + int start = 0; + int num_keys = 0; + + len = __default_parser_parse_recursive (validator, cache, start, num_keys, str, len, 0); + + for (size_t i=0; i<(size_t)num_keys; ++i) { + DefaultParserCacheElement* element = &g_array_index + (cache, DefaultParserCacheElement, start); + g_array_append_val(keys, element->key); + g_array_append_val(poses, element->pos); + start = element->next_start; + } + + return len; +} + +PinyinShuangPinParser::PinyinShuangPinParser (PinyinShuangPinScheme scheme) +{ + set_scheme (scheme); +} + +PinyinShuangPinParser::PinyinShuangPinParser (const PinyinInitial initial_map[27], const PinyinFinal final_map[27][2]) +{ + set_scheme (initial_map, final_map); +} + +PinyinShuangPinParser::~PinyinShuangPinParser () +{ +} + +int +PinyinShuangPinParser::parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char *str, int len) const +{ + key.clear (); + + if (!str || !len || ! (*str)) return 0; + + if (len < 0) len = strlen (str); + + PinyinInitial initial = PINYIN_ZeroInitial; + PinyinFinal final = PINYIN_ZeroFinal; + PinyinFinal final_cands [4] = { PINYIN_ZeroFinal, PINYIN_ZeroFinal, PINYIN_ZeroFinal, PINYIN_ZeroFinal }; + + PinyinTone tone = PINYIN_ZeroTone; + + int idx [2] = {-1, -1}; + int used_len = 0; + + size_t i; + bool matched = false; + + for (i = 0; i < 2 && i < (size_t) len; ++i) { + if (str [i] >= 'a' && str [i] <= 'z') idx [i] = str [i] - 'a'; + else if (str [i] == ';') idx [i] = 26; + } + + // parse initial or final + if (idx [0] >= 0) { + initial = m_initial_map [idx[0]]; + final_cands [0] = m_final_map [idx[0]][0]; + final_cands [1] = m_final_map [idx[0]][1]; + } + + if (initial == PINYIN_ZeroInitial && final_cands [0] == PINYIN_ZeroFinal) + return 0; + + // parse final, if str [0] == 'o' (idx [0] == 14) then just skip to parse final. + if (idx [1] >= 0 && (initial != PINYIN_ZeroInitial || idx[0] == 14)) { + final_cands [2] = m_final_map [idx [1]][0]; + final_cands [3] = m_final_map [idx [1]][1]; + + for (i = 2; i < 4; ++i) { + if (final_cands [i] != PINYIN_ZeroFinal) { + key.set (initial, final_cands [i]); + PinyinParser::normalize (key); + + if (validator (key)) { + final = final_cands [i]; + matched = true; + used_len = 2; + str += 2; + len -= 2; + break; + } + } + } + } + + if (!matched) { + initial = PINYIN_ZeroInitial; + for (i = 0; i < 2; ++i) { + key.set (initial, final_cands [i]); + PinyinParser::normalize (key); + + if (validator (key)) { + final = final_cands [i]; + matched = true; + used_len = 1; + ++str; + --len; + break; + } + } + } + + if (!matched) return 0; + + // parse tone + if (len) { + int kt = (*str) - '0'; + if (kt >= PINYIN_First && kt <= PINYIN_LastTone) { + tone = static_cast(kt); + + key.set (initial, final, tone); + + if (validator (key)) { + return used_len + 1; + } + } + } + + return used_len; +} + +int +PinyinShuangPinParser::parse (const PinyinValidator &validator, PinyinKeyVector &keys, PinyinKeyPosVector & poses, const char *str, int len) const +{ + g_array_set_size(keys, 0); + g_array_set_size(poses, 0); + + if (!str || !len || ! (*str)) return 0; + + if (len < 0) len = strlen (str); + + int used_len = 0; + + PinyinKey key; + PinyinKeyPos pos; + + while (used_len < len) { + if (*str == '\'' || *str == ' ') { + ++str; + ++used_len; + continue; + } + + int one_len = parse_one_key (validator, key, str, len); + + if (one_len) { + pos.set_pos(used_len); + pos.set_length(one_len); + g_array_append_val(keys, key); + g_array_append_val(poses, pos); + } else { + break; + } + + str += one_len; + used_len += one_len; + } + + return used_len; +} + +void +PinyinShuangPinParser::set_scheme (PinyinShuangPinScheme scheme) +{ + switch (scheme) { + case SHUANG_PIN_STONE: + set_scheme (__shuang_pin_stone_initial_map, __shuang_pin_stone_final_map); + break; + case SHUANG_PIN_ZRM: + set_scheme (__shuang_pin_zrm_initial_map, __shuang_pin_zrm_final_map); + break; + case SHUANG_PIN_MS: + set_scheme (__shuang_pin_ms_initial_map, __shuang_pin_ms_final_map); + break; + case SHUANG_PIN_ZIGUANG: + set_scheme (__shuang_pin_ziguang_initial_map, __shuang_pin_ziguang_final_map); + break; + case SHUANG_PIN_ABC: + set_scheme (__shuang_pin_abc_initial_map, __shuang_pin_abc_final_map); + break; + case SHUANG_PIN_LIUSHI: + set_scheme (__shuang_pin_liushi_initial_map, __shuang_pin_liushi_final_map); + break; + default: + set_scheme (__shuang_pin_zrm_initial_map, __shuang_pin_zrm_final_map); + return; + } +} + +void +PinyinShuangPinParser::set_scheme (const PinyinInitial initial_map[27], const PinyinFinal final_map[27][2]) +{ + for (size_t i = 0; i < 27; ++i) { + m_initial_map [i] = initial_map [i]; + m_final_map [i][0] = final_map [i][0]; + m_final_map [i][1] = final_map [i][1]; + } +} + +void +PinyinShuangPinParser::get_scheme (PinyinInitial initial_map[27], PinyinFinal final_map[27][2]) +{ + for (size_t i = 0; i < 27; ++i) { + initial_map [i] = m_initial_map [i]; + final_map [i][0] = m_final_map [i][0]; + final_map [i][1] = m_final_map [i][1]; + } +} + +namespace novel{ + +////////////////////////////////////////////////////////////////////////////// +// implementation of PinyinKey comparision classe +int pinyin_compare_initial (const PinyinCustomSettings &custom, + PinyinInitial lhs, + PinyinInitial rhs) +{ + if ((lhs == rhs) || + (custom.use_ambiguities [PINYIN_AmbZhiZi] && + ((lhs == PINYIN_Zhi && rhs == PINYIN_Zi) || + (lhs == PINYIN_Zi && rhs == PINYIN_Zhi))) || + + (custom.use_ambiguities [PINYIN_AmbChiCi] && + ((lhs == PINYIN_Chi && rhs == PINYIN_Ci) || + (lhs == PINYIN_Ci && rhs == PINYIN_Chi))) || + + (custom.use_ambiguities [PINYIN_AmbShiSi] && + ((lhs == PINYIN_Shi && rhs == PINYIN_Si) || + (lhs == PINYIN_Si && rhs == PINYIN_Shi))) || + + (custom.use_ambiguities [PINYIN_AmbLeRi] && + ((lhs == PINYIN_Le && rhs == PINYIN_Ri) || + (lhs == PINYIN_Ri && rhs == PINYIN_Le))) || + + (custom.use_ambiguities [PINYIN_AmbNeLe] && + ((lhs == PINYIN_Ne && rhs == PINYIN_Le) || + (lhs == PINYIN_Le && rhs == PINYIN_Ne))) || + + (custom.use_ambiguities [PINYIN_AmbFoHe] && + ((lhs == PINYIN_Fo && rhs == PINYIN_He) || + (lhs == PINYIN_He && rhs == PINYIN_Fo))) + ) + return 0; + else if (lhs < rhs) return -1; + return 1; +} + +int pinyin_compare_final (const PinyinCustomSettings &custom, + PinyinFinal lhs, + PinyinFinal rhs) +{ + if(((lhs == rhs) || + (custom.use_ambiguities [PINYIN_AmbAnAng] && + ((lhs == PINYIN_An && rhs == PINYIN_Ang) || + (lhs == PINYIN_Ang && rhs == PINYIN_An))) || + + (custom.use_ambiguities [PINYIN_AmbEnEng] && + ((lhs == PINYIN_En && rhs == PINYIN_Eng) || + (lhs == PINYIN_Eng && rhs == PINYIN_En))) || + + (custom.use_ambiguities [PINYIN_AmbInIng] && + ((lhs == PINYIN_In && rhs == PINYIN_Ing) || + (lhs == PINYIN_Ing && rhs == PINYIN_In))))) + return 0; + else if (custom.use_incomplete && (lhs == PINYIN_ZeroFinal || rhs == PINYIN_ZeroFinal)) + return 0; + else if (lhs < rhs) return -1; + return 1; +} + +int pinyin_compare_tone (const PinyinCustomSettings &custom, + PinyinTone lhs, + PinyinTone rhs) +{ + if(lhs == rhs || !lhs || !rhs) + return 0; + else if (lhs < rhs) return -1; + return 1; +} + +}; diff --git a/src/storage/pinyin_base.h b/src/storage/pinyin_base.h new file mode 100644 index 0000000..374cc53 --- /dev/null +++ b/src/storage/pinyin_base.h @@ -0,0 +1,728 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2002,2003,2006 James Su + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/** @file pinyin_base.h + * @brief the definitions of pinyin related classes and structs. + */ + +#ifndef PINYIN_BASE_H +#define PINYIN_BASE_H + +#include + +namespace novel{ + +// Predefinition of some classes and structs +struct PinyinKey; + +class PinyinValidator; +class PinyinParser; + +struct PinyinKeyPos{ + int m_pos; + size_t m_len; + PinyinKeyPos(){ + m_pos = 0; + m_len = 0; + } + void set_pos(int pos){ + m_pos = pos; + } + void set_length(size_t len){ + m_len = len; + } + int get_pos(){ + return m_pos; + } + int get_end_pos(){ + return m_pos + m_len; + } + size_t get_length(){ + return m_len; + } +}; + +typedef GArray* PinyinKeyVector; /* Array of PinyinKey */ +typedef GArray* PinyinKeyPosVector; /* Array of PinyinKeyPos */ + + +struct PinyinCustomSettings; + +/** + * @brief enums of pinyin initial element. + * + * A pinyin key can be divided into three tokens: + * Initial -- such as B P M F D T N L etc. + * Final -- such as A O E I U V etc. + * Tone -- can be 1, 2, 3, 4 and 5. + */ +enum PinyinInitial +{ + PINYIN_ZeroInitial = 0, /**< zero initial. indicates invaild initial */ + PINYIN_Bo = 1, + PINYIN_Ci = 2, + PINYIN_Chi = 3, + PINYIN_De = 4, + PINYIN_Fo = 5, + PINYIN_He = 6, + PINYIN_Ge = 7, + PINYIN_Ji = 8, + PINYIN_Ke = 9, + PINYIN_Mo =10, + PINYIN_Ne =11, + PINYIN_Le =12, + PINYIN_Ri =13, + PINYIN_Po =14, + PINYIN_Qi =15, + PINYIN_Si =16, + PINYIN_Shi =17, + PINYIN_Te =18, + PINYIN_Wu =19, + PINYIN_Xi =20, + PINYIN_Yi =21, + PINYIN_Zi =22, + PINYIN_Zhi =23, + PINYIN_LastInitial = PINYIN_Zhi, /**< the last initial */ + PINYIN_Number_Of_Initials = PINYIN_LastInitial + 1 +}; + +/** + * @brief enums of pinyin final element. + */ +enum PinyinFinal +{ + PINYIN_ZeroFinal = 0, /**< zero final. indicates invalid final */ + PINYIN_A = 1, + PINYIN_Ai = 2, + PINYIN_An = 3, + PINYIN_Ang = 4, + PINYIN_Ao = 5, + PINYIN_E = 6, + PINYIN_Ea = 7, + PINYIN_Ei = 8, + PINYIN_En = 9, + PINYIN_Eng =10, + PINYIN_Er =11, + PINYIN_I =12, + PINYIN_Ia =13, + PINYIN_Ian =14, + PINYIN_Iang =15, + PINYIN_Iao =16, + PINYIN_Ie =17, + PINYIN_In =18, + PINYIN_Ing =19, + PINYIN_Iong =20, + PINYIN_Iu =21, + PINYIN_Ng =22, + PINYIN_O =23, + PINYIN_Ong =24, + PINYIN_Ou =25, + PINYIN_U =26, + PINYIN_Ua =27, + PINYIN_Uai =28, + PINYIN_Uan =29, + PINYIN_Uang =30, + PINYIN_Ue =31, + PINYIN_Ueng =32, + PINYIN_Ui =33, + PINYIN_Un =34, + PINYIN_Uo =35, + PINYIN_V =36, + PINYIN_Van =37, + PINYIN_Ve =38, + PINYIN_Vn =39, + PINYIN_LastFinal = PINYIN_Vn, /**< the last final */ + PINYIN_Number_Of_Finals = PINYIN_LastFinal + 1 +}; + +/** + * @brief enums of pinyin tone element. + */ +enum PinyinTone +{ + PINYIN_ZeroTone = 0, /**< zero tone. this will be matched with all other tones. */ + PINYIN_First = 1, + PINYIN_Second = 2, + PINYIN_Third = 3, + PINYIN_Fourth = 4, + PINYIN_Fifth = 5, + PINYIN_LastTone = PINYIN_Fifth, /**< the last tone */ + PINYIN_Number_Of_Tones = PINYIN_LastTone + 1 +}; + +/** + * @brief enums of Shuang Pin Schemes. + */ +enum PinyinShuangPinScheme +{ + SHUANG_PIN_STONE = 0, + SHUANG_PIN_ZRM = 1, + SHUANG_PIN_MS = 2, + SHUANG_PIN_ZIGUANG = 3, + SHUANG_PIN_ABC = 4, + SHUANG_PIN_LIUSHI = 5, + SHUANG_PIN_CUSTOMIZED = 6, + SHUANG_PIN_DEFAULT = SHUANG_PIN_ZRM +}; + +/** + * @brief enums of ZhuYin Schemes. + */ +enum PinyinZhuYinScheme +{ + ZHUYIN_ZHUYIN = 0, + ZHUYIN_STANDARD = 1, + ZHUYIN_HSU = 2, + ZHUYIN_IBM = 3, + ZHUYIN_GIN_YIEH = 4, + ZHUYIN_ET = 5, + ZHUYIN_ET26 = 6, + ZHUYIN_DEFAULT = ZHUYIN_STANDARD +}; + +/** + * @brief enums of pinyin ambiguities. + * + * Some pinyin element maybe confused by somebody, + * We allow these ambiguities. + */ +enum PinyinAmbiguity +{ + PINYIN_AmbAny= 0, + PINYIN_AmbZhiZi, + PINYIN_AmbChiCi, + PINYIN_AmbShiSi, + PINYIN_AmbNeLe, + PINYIN_AmbLeRi, + PINYIN_AmbFoHe, + PINYIN_AmbAnAng, + PINYIN_AmbEnEng, + PINYIN_AmbInIng, + PINYIN_AmbLast = PINYIN_AmbInIng +}; + +/** + * @brief Structure to hold pinyin custom settings. + * + * user can custom the behavor of libpinyin by these settings. + */ +struct PinyinCustomSettings +{ + bool use_incomplete; + /**< allow incomplete pinyin key which only has inital. */ + + bool use_ambiguities [PINYIN_AmbLast + 1]; + /**< allow ambiguous pinyin elements or not. */ + + PinyinCustomSettings (); + + void set_use_incomplete (bool use) { use_incomplete = use; } + void set_use_ambiguities (PinyinAmbiguity amb, bool use) + { + if (amb == PINYIN_AmbAny) + for (size_t i=0; i<=PINYIN_AmbLast; ++i) use_ambiguities [i] = use; + else { + use_ambiguities [0] = false; + use_ambiguities [static_cast(amb)] = use; + for (size_t i=1; i<=PINYIN_AmbLast; ++i) + if (use_ambiguities [i]) { + use_ambiguities [0] = true; + break; + } + } + } + + bool operator == (const PinyinCustomSettings &rhs) const + { + if (use_incomplete != rhs.use_incomplete) + return false; + + for (size_t i=0; i <= PINYIN_AmbLast; ++i) + if (use_ambiguities [i] != rhs.use_ambiguities [i]) + return false; + + return true; + } + + bool operator != (const PinyinCustomSettings &rhs) const + { + return !(*this == rhs); + } + + guint32 to_value () const + { + guint32 val = 0; + + if (use_incomplete) val |= 1; + + for (size_t i=0; i <= PINYIN_AmbLast; ++i) + if (use_ambiguities [i]) + val |= (1 << (i+1)); + + return val; + } + + void from_value (guint32 val) + { + use_incomplete = ((val & 1) != 0); + + for (size_t i=0; i <= PINYIN_AmbLast; ++i) + use_ambiguities [i] = ((val & (1 << (i+1))) != 0); + } +}; + +/** + * @brief Pinyin key class. + * + * A pinyin key is a composed element of an initial, a final and a tone, + * which represents one or several Chinese ideographs + * + * The position and length information for the portion of string, from which + * the PinyinKey is parsed, are also stored in this structure. + */ +struct PinyinKey +{ + friend class PinyinBitmapIndexLevel; + friend inline int pinyin_exact_compare(const PinyinKey key_lhs[], + const PinyinKey key_rhs[], + int word_length); + friend inline int pinyin_compare_with_ambiguities + (const PinyinCustomSettings &custom, + const PinyinKey* key_lhs, + const PinyinKey* key_rhs, + int word_length); + friend inline void compute_lower_value(const PinyinCustomSettings &custom, + PinyinKey in_keys[], + PinyinKey out_keys[], + int word_length); + friend inline void compute_upper_value(const PinyinCustomSettings &custom, + PinyinKey in_keys[], + PinyinKey out_keys[], + int word_length); + +private: + guint16 m_initial : 5; /**< pinyin initial */ + guint16 m_final : 6; /**< pinyin final */ + guint16 m_tone : 3; /**< pinyin tone */ +public: + /** + * @brief Minimal numerical value of a PinyinKey + * @sa get_value(); + */ + static const guint16 min_value; + + /** + * @brief Maximal numerical value of a PinyinKey + * @sa get_value(); + */ + static const guint16 max_value; + +public: + /** + * Constructor. + * + * The default constructor of class PinyinKey. + */ + PinyinKey (PinyinInitial initial = PINYIN_ZeroInitial, + PinyinFinal final = PINYIN_ZeroFinal, + PinyinTone tone = PINYIN_ZeroTone) + : m_initial (initial), m_final (final), m_tone (tone) + { + } + + /** + * Constructor. + * + * Construct a PinyinKey object from a key string, with + * specified validator. + * + * @sa PinyinValidator + */ + PinyinKey (const PinyinValidator &validator, const char *str, int len = -1) + { + set (validator, str, len); + } + + PinyinKey (guint16 value) + { + set (value); + } + /** + * Clear the PinyinKey object. + */ + + void clear () + { + m_initial = PINYIN_ZeroInitial; + m_final = PINYIN_ZeroFinal; + m_tone = PINYIN_ZeroTone; + } + + /** + * Read PinyinKey value from a key string. + * + * @param validator a PinyinValidator object to validate the key. + * @param key a Latin string including one or more pinyin keys. + * @return the number of characters used by this pinyin key. + */ + int set (const PinyinValidator &validator, const char *str, int len = -1); + + /** + * Set PinyinKey's value to initial, final and tone. + */ + void set (PinyinInitial initial = PINYIN_ZeroInitial, + PinyinFinal final = PINYIN_ZeroFinal, + PinyinTone tone = PINYIN_ZeroTone) + { + m_initial = initial; + m_final = final; + m_tone = tone; + } + + /** + * @brief Set this PinyinKey from its numerical value. + */ + void set (guint16 value) + { + m_tone = value % PINYIN_Number_Of_Tones; + value /= PINYIN_Number_Of_Tones; + m_final = value % PINYIN_Number_Of_Finals; + m_initial = value / PINYIN_Number_Of_Finals; + } + + /** + * @brief Get numerical value of this PinyinKey + */ + guint16 get_value () const + { + return (m_initial * PINYIN_Number_Of_Finals + m_final) * PINYIN_Number_Of_Tones + m_tone; + } + + /** + * Set PinyinKey's initial value to initial. + */ + void set_initial (PinyinInitial initial = PINYIN_ZeroInitial) + { + m_initial = initial; + } + + /** + * Set PinyinKey's final value to final. + */ + void set_final (PinyinFinal final = PINYIN_ZeroFinal) + { + m_final = final; + } + + /** + * Set PinyinKey's tone value to tone. + */ + void set_tone (PinyinTone tone = PINYIN_ZeroTone) + { + m_tone = tone; + } + + /** + * Get initial value of this key. + */ + PinyinInitial get_initial () const + { + return static_cast(m_initial); + } + + /** + * Get final value of this key. + */ + PinyinFinal get_final () const + { + return static_cast(m_final); + } + + /** + * Get tone value of this key. + */ + PinyinTone get_tone () const + { + return static_cast(m_tone); + } + + /** + * Get Latin name of this key's initial. + */ + const char* get_initial_string () const; + + /** + * Get Chinese ZhuYin name of this key's initial, in UTF-8 encoding. + */ + const char* get_initial_zhuyin_string () const; + + /** + * Get Latin name of this key's final. + */ + const char* get_final_string () const; + + /** + * Get Chinese ZhuYin name of this key's final, in UTF-8 encoding. + */ + const char* get_final_zhuyin_string () const; + + /** + * Get Latin name of this key's tone. + */ + const char* get_tone_string () const; + + /** + * Get Chinese ZhuYin name of this key's tone, in UTF-8 encoding. + */ + const char* get_tone_zhuyin_string () const; + + /** + * Get Latin name of this key. + */ + const char * get_key_string () const; + + /** + * Get Chinese ZhuYin name of this key, in UTF-8 encoding. + */ + const char * get_key_zhuyin_string () const; + + /** + * Check if this key is empty. + */ + bool is_empty () const + { + return m_initial == PINYIN_ZeroInitial && m_final == PINYIN_ZeroFinal && m_tone == PINYIN_ZeroTone; + } + + /** + * Check if this key has both initial, final and tone. + */ + bool is_complete () const + { + return m_initial != PINYIN_ZeroInitial && m_final != PINYIN_ZeroFinal && m_tone != PINYIN_ZeroTone; + } + + bool operator == (PinyinKey rhs) const + { + return m_initial == rhs.m_initial && m_final == rhs.m_final && m_tone == rhs.m_tone; + } + + bool operator != (PinyinKey rhs) const + { + return m_initial != rhs.m_initial || m_final != rhs.m_final || m_tone != rhs.m_tone; + } + + bool operator < (PinyinKey rhs) const + { + if (m_initial < rhs.m_initial) return true; + if (m_initial > rhs.m_initial) return false; + if (m_final < rhs.m_final) return true; + if (m_final > rhs.m_final) return false; + return m_tone < rhs.m_tone; + } + + bool operator > (PinyinKey rhs) const + { + if (m_initial > rhs.m_initial) return true; + if (m_initial < rhs.m_initial) return false; + if (m_final > rhs.m_final) return true; + if (m_final < rhs.m_final) return false; + return m_tone > rhs.m_tone; + } +}; + +/** + * NULL Validator of PinyinKey object. + * + * This class is for validating a PinyinKey object. + */ +class PinyinValidator +{ +public: + /** + * Overloaded operator () function to validate a pinyin key. + * + * @param key The key to be validated. + * @return true if the key is valid. + */ + virtual bool operator () (PinyinKey key) const = 0; +}; + +class PinyinLargeTable; +/** + * Validator of PinyinKey object. + * + * This class is for validating a PinyinKey object. + */ +class BitmapPinyinValidator:public PinyinValidator +{ + char m_bitmap [(PINYIN_Number_Of_Initials * PINYIN_Number_Of_Finals * PINYIN_Number_Of_Tones + 7) / 8]; + +public: + BitmapPinyinValidator (const PinyinLargeTable *table = 0); + + /** + * initialize the validator with specified custom settings + * and PinyinLargeTable. + */ + void initialize (const PinyinLargeTable *table = 0); + + /** + * Overloaded operator () function to validate a pinyin key. + * + * @param key The key to be validated. + * @return true if the key is valid. + */ + virtual bool operator () (PinyinKey key) const; +}; + +/** + * NULL Validator of PinyinKey object. + * + * This class is for validating a PinyinKey object. + */ +class NullPinyinValidator:public PinyinValidator +{ +public: + /** + * Overloaded operator () function to validate a pinyin key. + * + * @param key The key to be validated. + * @return true if the key is valid. + */ + virtual bool operator () (PinyinKey key) const{ + return true; + } +}; + +/** + * @brief Class to translate string into PinyinKey. + */ +class PinyinParser +{ +public: + virtual ~PinyinParser (); + + /** + * @brief Translate only one PinyinKey from a string. + * + * @param validator PinyinValidator object to valid result. + * @param key Stores result PinyinKey. + * @param str Input string in UTF-8 encoding, in most case this string is just a plain ASCII string, + * but for ZhuYin Parser works in ZHUYIN_ZHUYIN scheme, + * it's an UTF-8 string which contains ZhuYin chars. + * @param len The length of str, in number of chars rather than bytes. + * + * @return the number of chars were actually used. + */ + virtual int parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char *str, int len = -1) const = 0; + + /** + * @brief Handy wrapper function of parse_one_key(), which accept a String object instead of char *. + */ + int parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char * &str) const + { + return parse_one_key (validator, key, str, g_utf8_strlen (str, -1)); + } + + /** + * @brief Translate the source string into a set of PinyinKeys. + * + * @param validator PinyinValidator object to valid result. + * @param keys Stores result PinyinKeys. + * @param str Input string in UTF-8 encoding, in most case this string is just a plain ASCII string, + * but for ZhuYin Parser works in ZHUYIN_ZHUYIN scheme, + * it's an UTF-8 string which contains ZhuYin chars. + * @param len The length of str, in number of chars rather than bytes. + * + * @return the number of chars were actually used. + */ + virtual int parse (const PinyinValidator &validator, PinyinKeyVector & keys,PinyinKeyPosVector & poses, const char *str, int len = -1) const = 0; + +public: + static void normalize (PinyinKey &key); +}; + +/** + * The default Pinyin Parser which parses full pinyin string into PinyinKeys. + */ +class PinyinDefaultParser : public PinyinParser +{ +public: + virtual ~PinyinDefaultParser (); + + virtual int parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char *str, int len = -1) const; + virtual int parse (const PinyinValidator &validator, PinyinKeyVector & keys, PinyinKeyPosVector & poses, const char *str, int len = -1) const; + +public: + using PinyinParser::parse_one_key; + using PinyinParser::parse; +}; + +/* The valid input chars of ShuangPin is a-z and ';' + */ +class PinyinShuangPinParser : public PinyinParser +{ + PinyinInitial m_initial_map [27]; + PinyinFinal m_final_map [27][2]; + +public: + /** + * Constructor + * + * @param scheme the predefined ShuangPin scheme to be used. + */ + PinyinShuangPinParser (PinyinShuangPinScheme scheme = SHUANG_PIN_DEFAULT); + PinyinShuangPinParser (const PinyinInitial initial_map[27], const PinyinFinal final_map[27][2]); + + virtual ~PinyinShuangPinParser (); + + virtual int parse_one_key (const PinyinValidator &validator, PinyinKey &key, const char *str, int len = -1) const; + virtual int parse (const PinyinValidator &validator, PinyinKeyVector &keys, PinyinKeyPosVector & poses, const char *str, int len = -1) const; + +public: + void set_scheme (PinyinShuangPinScheme scheme); + void set_scheme (const PinyinInitial initial_map[27], const PinyinFinal final_map[27][2]); + + void get_scheme (PinyinInitial initial_map[27], PinyinFinal final_map[27][2]); + +public: + using PinyinParser::parse_one_key; + using PinyinParser::parse; +}; + +int pinyin_compare_initial (const PinyinCustomSettings &custom, + PinyinInitial lhs, + PinyinInitial rhs); + +int pinyin_compare_final (const PinyinCustomSettings &custom, + PinyinFinal lhs, + PinyinFinal rhs); + +int pinyin_compare_tone (const PinyinCustomSettings &custom, + PinyinTone lhs, + PinyinTone rhs); +}; + +using namespace novel; + +#endif diff --git a/src/storage/pinyin_large_table.cpp b/src/storage/pinyin_large_table.cpp new file mode 100644 index 0000000..794cca5 --- /dev/null +++ b/src/storage/pinyin_large_table.cpp @@ -0,0 +1,690 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include "novel_types.h" +#include "pinyin_base.h" +#include "pinyin_phrase.h" +#include "pinyin_large_table.h" + + +PinyinBitmapIndexLevel::PinyinBitmapIndexLevel(PinyinCustomSettings * custom) + :m_custom(custom){ + memset(m_pinyin_length_indexes, 0 , sizeof(m_pinyin_length_indexes)); +} + +void PinyinBitmapIndexLevel::reset(){ + for ( int k = PINYIN_ZeroInitial; k < PINYIN_Number_Of_Initials; k++) + for ( int m = PINYIN_ZeroFinal; m < PINYIN_Number_Of_Finals; m++) + for ( int n = PINYIN_ZeroTone; n < PINYIN_Number_Of_Tones; n++){ + PinyinLengthIndexLevel * length_array = + m_pinyin_length_indexes[k][m][n]; + if ( length_array ) + delete length_array; + } +} + +int PinyinBitmapIndexLevel::search( int phrase_length, /* in */ PinyinKey keys[], + /* out */ PhraseIndexRanges ranges) const{ + return initial_level_search(phrase_length, keys, ranges); +} + +int PinyinBitmapIndexLevel::initial_level_search(int phrase_length, + /* in */PinyinKey keys[], + /* out */ PhraseIndexRanges ranges) const{ + +#define MATCH(AMBIGUITY, ORIGIN, ANOTHER) case ORIGIN: \ + { \ + result |= final_level_search((PinyinInitial)first_key.m_initial,\ + phrase_length, keys, ranges); \ + if ( custom.use_ambiguities [AMBIGUITY] ){ \ + result |= final_level_search(ANOTHER, \ + phrase_length, keys, ranges); \ + } \ + return result; \ + } + + //deal with the ambiguities + + int result = 0; + PinyinKey& first_key = keys[0]; + PinyinCustomSettings & custom= *m_custom; + + switch(first_key.m_initial){ + + MATCH(PINYIN_AmbZhiZi, PINYIN_Zi, PINYIN_Zhi); + MATCH(PINYIN_AmbZhiZi, PINYIN_Zhi, PINYIN_Zi); + MATCH(PINYIN_AmbChiCi, PINYIN_Ci, PINYIN_Chi); + MATCH(PINYIN_AmbChiCi, PINYIN_Chi, PINYIN_Ci); + MATCH(PINYIN_AmbShiSi, PINYIN_Si, PINYIN_Shi); + MATCH(PINYIN_AmbShiSi, PINYIN_Shi, PINYIN_Si); + MATCH(PINYIN_AmbLeRi, PINYIN_Ri, PINYIN_Le); + MATCH(PINYIN_AmbNeLe, PINYIN_Ne, PINYIN_Le); + MATCH(PINYIN_AmbFoHe, PINYIN_Fo, PINYIN_He); + MATCH(PINYIN_AmbFoHe, PINYIN_He, PINYIN_Fo); + + case PINYIN_Le: + { + result |= final_level_search((PinyinInitial)first_key.m_initial, + phrase_length, keys, ranges); + if ( custom.use_ambiguities [PINYIN_AmbLeRi] ) + result |= final_level_search(PINYIN_Ri, phrase_length, + keys, ranges); + if ( custom.use_ambiguities [PINYIN_AmbNeLe] ) + result |= final_level_search(PINYIN_Ne, phrase_length, + keys, ranges); + return result; + } + default: + { + return final_level_search((PinyinInitial)first_key.m_initial, + phrase_length, + keys, ranges); + } + } +#undef MATCH +} + +int PinyinBitmapIndexLevel::final_level_search(PinyinInitial initial, + int phrase_length, + /* in */PinyinKey keys[], + /* out */ PhraseIndexRanges ranges) const{ +#define MATCH(AMBIGUITY, ORIGIN, ANOTHER) case ORIGIN: \ + { \ + result = tone_level_search(initial,(PinyinFinal) first_key.m_final,\ + phrase_length, keys, ranges); \ + if ( custom.use_ambiguities [AMBIGUITY] ){ \ + result |= tone_level_search(initial, ANOTHER, \ + phrase_length, keys, ranges); \ + } \ + return result; \ + } + + int result = 0; + PinyinKey& first_key = keys[0]; + PinyinCustomSettings & custom= *m_custom; + + switch(first_key.m_final){ + case PINYIN_ZeroFinal: + { + if (!custom.use_incomplete ) + return result; + for ( int i = PINYIN_A; i < PINYIN_Number_Of_Finals; ++i){ + result |= tone_level_search(initial,(PinyinFinal)i , + phrase_length, keys, ranges); + } + return result; + } + + MATCH(PINYIN_AmbAnAng, PINYIN_An, PINYIN_Ang); + MATCH(PINYIN_AmbAnAng, PINYIN_Ang, PINYIN_An); + MATCH(PINYIN_AmbEnEng, PINYIN_En, PINYIN_Eng); + MATCH(PINYIN_AmbEnEng, PINYIN_Eng, PINYIN_En); + MATCH(PINYIN_AmbInIng, PINYIN_In, PINYIN_Ing); + MATCH(PINYIN_AmbInIng, PINYIN_Ing, PINYIN_In); + + default: + { + return tone_level_search(initial,(PinyinFinal)first_key.m_final, + phrase_length, keys, ranges); + } + } +#undef MATCH +} + +int PinyinBitmapIndexLevel::tone_level_search(PinyinInitial initial, + PinyinFinal final, + int phrase_length, + /* in */PinyinKey keys[], + /* out */ PhraseIndexRanges ranges) const{ + int result = 0; + PinyinKey& first_key = keys[0]; + PinyinCustomSettings & custom= *m_custom; + + switch ( first_key.m_tone ){ + case PINYIN_ZeroTone: + { + //deal with ZeroTone in pinyin table files. + for ( int i = PINYIN_ZeroTone; i < PINYIN_Number_Of_Tones; ++i){ + PinyinLengthIndexLevel * phrases = + m_pinyin_length_indexes[initial][final][(PinyinTone)i]; + if ( phrases ) + result |= phrases->search(phrase_length - 1, &custom, + keys + 1, ranges); + } + return result; + } + default: + { + PinyinLengthIndexLevel * phrases = + m_pinyin_length_indexes[initial][final] + [PINYIN_ZeroTone]; + if ( phrases ) + result = phrases->search(phrase_length - 1, &custom, + keys + 1, ranges); + phrases = m_pinyin_length_indexes[initial][final] + [(PinyinTone) first_key.m_tone]; + if ( phrases ) + result |= phrases->search(phrase_length - 1, &custom, + keys + 1, ranges); + return result; + } + } + return result; +} + +PinyinLengthIndexLevel::PinyinLengthIndexLevel(){ + m_pinyin_array_indexes = g_array_new(FALSE, TRUE, sizeof(void *)); +} + +PinyinLengthIndexLevel::~PinyinLengthIndexLevel(){ +#define CASE(x) case x: \ + { \ + PinyinArrayIndexLevel * array = g_array_index \ + (m_pinyin_array_indexes, PinyinArrayIndexLevel *, x); \ + if (array) \ + delete array; \ + break; \ + } + for ( int i = 0 ; i < m_pinyin_array_indexes->len; ++i){ + switch (i){ + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + default: + assert(false); + } + } + g_array_free(m_pinyin_array_indexes, TRUE); +#undef CASE +} + +int PinyinLengthIndexLevel::search( int phrase_length, + /* in */ PinyinCustomSettings * custom, + /* in */ PinyinKey keys[], + /* out */ PhraseIndexRanges ranges){ + int result = SEARCH_NONE; + if(m_pinyin_array_indexes->len < phrase_length + 1) + return result; + if (m_pinyin_array_indexes->len > phrase_length + 1) + result |= SEARCH_CONTINUED; + +#define CASE(len) case len: \ + { \ + PinyinArrayIndexLevel * array = g_array_index \ + (m_pinyin_array_indexes, PinyinArrayIndexLevel *, len); \ + if ( !array ) \ + return result; \ + result |= array->search(custom, keys, ranges); \ + return result; \ + } + + switch ( phrase_length ){ + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + default: + assert(false); + } +#undef CASE +} + +template +int PinyinArrayIndexLevel::search(/* in */ PinyinCustomSettings * custom, /* in */ PinyinKey keys[], /* out */ PhraseIndexRanges ranges){ + PhraseExactLessThan m_lessthan; + PinyinIndexItem * chunk_begin, * chunk_end; + chunk_begin = (PinyinIndexItem *)m_chunk.begin(); + chunk_end = (PinyinIndexItem *)m_chunk.end(); + //do the search + PinyinKey left_keys[phrase_length], right_keys[phrase_length]; + compute_lower_value(*custom, keys, left_keys, phrase_length); + compute_upper_value(*custom, keys, right_keys, phrase_length); + PinyinIndexItem left(left_keys, -1), right(right_keys, -1); + + PinyinIndexItem * begin = std_lite::lower_bound(chunk_begin, chunk_end, left, m_lessthan); + PinyinIndexItem * end = std_lite::upper_bound(chunk_begin, chunk_end, right, m_lessthan); + + return convert(custom, keys, begin, end, ranges); +} + +template +int PinyinArrayIndexLevel::convert(PinyinCustomSettings * custom, PinyinKey keys[], PinyinIndexItem * begin, PinyinIndexItem * end, PhraseIndexRanges ranges){ + PinyinIndexItem * iter; + PhraseIndexRange cursor; + GArray * head, *cursor_head = NULL; + int result = SEARCH_NONE; + cursor.m_range_begin = -1; cursor.m_range_end = -1; + for ( iter = begin; iter != end; ++iter){ + if ( ! 0 == + pinyin_compare_with_ambiguities + (*custom, keys, iter->m_keys, phrase_length)) + continue; + phrase_token_t token = iter->m_token; + head = ranges[PHRASE_INDEX_LIBRARY_INDEX(token)]; + if ( NULL == head ) + continue; + + result |= SEARCH_OK; + + if ( cursor.m_range_begin == -1 ){ + cursor.m_range_begin = token; + cursor.m_range_end = token + 1; + cursor_head = head; + }else if (cursor.m_range_end == token && + PHRASE_INDEX_LIBRARY_INDEX(cursor.m_range_end) == + PHRASE_INDEX_LIBRARY_INDEX(token) ){ + cursor.m_range_end++; + }else { + g_array_append_val(cursor_head, cursor); + cursor.m_range_begin = token; cursor.m_range_end = token + 1; + cursor_head = head; + } + } + if ( cursor.m_range_begin == -1 ) + return result; + + g_array_append_val(cursor_head, cursor); + return result; +} + +int PinyinBitmapIndexLevel::add_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + PinyinKey firstkey = keys[0]; + PinyinLengthIndexLevel * &length_array = + m_pinyin_length_indexes[firstkey.m_initial][firstkey.m_final][firstkey.m_tone]; + if ( ! length_array ){ + length_array = new PinyinLengthIndexLevel(); + } + return length_array->add_index(phrase_length - 1, keys + 1, token); +} + +int PinyinBitmapIndexLevel::remove_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + PinyinKey firstkey = keys[0]; + PinyinLengthIndexLevel * &length_array = + m_pinyin_length_indexes[firstkey.m_initial][firstkey.m_final][firstkey.m_tone]; + if ( length_array ) + return length_array->add_index(phrase_length - 1, keys + 1, token); + return REMOVE_ITEM_DONOT_EXISTS; +} + +int PinyinLengthIndexLevel::add_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + assert(phrase_length + 1 < MAX_PHRASE_LENGTH); + if ( m_pinyin_array_indexes -> len <= phrase_length ) + g_array_set_size(m_pinyin_array_indexes, phrase_length + 1); +#define CASE(x) case x: \ + { \ + PinyinArrayIndexLevel * &array = g_array_index \ + (m_pinyin_array_indexes, PinyinArrayIndexLevel *, x); \ + if ( !array ) \ + array = new PinyinArrayIndexLevel; \ + return array->add_index(keys, token); \ + } + switch(phrase_length){ + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + default: + assert(false); + } +#undef CASE +} + +int PinyinLengthIndexLevel::remove_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + assert(phrase_length + 1 < MAX_PHRASE_LENGTH); + if ( m_pinyin_array_indexes -> len <= phrase_length ) + return false; +#define CASE(x) case x: \ + { \ + PinyinArrayIndexLevel * &array = g_array_index \ + (m_pinyin_array_indexes, PinyinArrayIndexLevel *, x); \ + if ( !array ) \ + return false; \ + return array->remove_index(keys, token); \ + } + switch(phrase_length){ + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + default: + assert(false); + } +#undef CASE +} + +template +int PinyinArrayIndexLevel::add_index(/* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + PhraseExactLessThan m_lessthan; + PinyinIndexItem * buf_begin, * buf_end; + + PinyinIndexItem new_elem(keys, token); + buf_begin = (PinyinIndexItem *) m_chunk.begin(); + buf_end = (PinyinIndexItem *) m_chunk.end(); + + std_lite::pair *, PinyinIndexItem *> range; + range = std_lite::equal_range(buf_begin, buf_end, new_elem, m_lessthan); + + PinyinIndexItem * cur_elem; + for ( cur_elem = range.first; + cur_elem != range.second; ++cur_elem){ + if ( cur_elem->m_token == token ) + return INSERT_ITEM_EXISTS; + if ( cur_elem->m_token > token ) + break; + } + + int offset = (cur_elem - buf_begin) * + sizeof(PinyinIndexItem); + m_chunk.insert_content(offset, &new_elem, + sizeof ( PinyinIndexItem )); + return INSERT_OK; +} + +template +int PinyinArrayIndexLevel::remove_index(/* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + PhraseExactLessThan m_lessthan; + PinyinIndexItem * buf_begin, * buf_end; + + PinyinIndexItem new_elem(keys, token); + buf_begin = (PinyinIndexItem *) m_chunk.begin(); + buf_end = (PinyinIndexItem *) m_chunk.end(); + + std_lite::pair *, PinyinIndexItem *> range; + range = std_lite::equal_range(buf_begin, buf_end, new_elem, m_lessthan); + + PinyinIndexItem * cur_elem; + for ( cur_elem = range.first; + cur_elem != range.second; ++cur_elem){ + if ( cur_elem->m_token == token ) + break; + } + if (cur_elem->m_token != token ) + return REMOVE_ITEM_DONOT_EXISTS; + + int offset = (cur_elem - buf_begin) * + sizeof(PinyinIndexItem); + m_chunk.remove_content(offset, sizeof (PinyinIndexItem)); + return REMOVE_OK; +} + +bool PinyinLargeTable::load_text(FILE * infile){ + char pinyin[256]; + char phrase[256]; + phrase_token_t token; + size_t freq; + while ( !feof(infile)){ + fscanf(infile, "%s", pinyin); + fscanf(infile, "%s", phrase); + fscanf(infile, "%ld", &token); + fscanf(infile, "%ld", &freq); + + PinyinDefaultParser parser; + NullPinyinValidator validator; + PinyinKeyVector keys; + PinyinKeyPosVector poses; + + keys = g_array_new(FALSE, FALSE, sizeof( PinyinKey)); + poses = g_array_new(FALSE, FALSE, sizeof( PinyinKeyPos)); + parser.parse(validator, keys, poses, pinyin); + + add_index( keys->len, (PinyinKey *)keys->data, token); + + g_array_free(keys, true); + g_array_free(poses, true); + } + return true; +} + +bool PinyinBitmapIndexLevel::load(MemoryChunk * chunk, table_offset_t offset, + table_offset_t end){ + reset(); + char * buf_begin = (char *) chunk->begin(); + table_offset_t phrase_begin, phrase_end; + table_offset_t * index = (table_offset_t *) (buf_begin + offset); + phrase_end = *index; + for ( int m = 0; m < PINYIN_Number_Of_Initials; ++m ) + for ( int n = 0; n < PINYIN_Number_Of_Finals; ++n) + for ( int k = 0; k < PINYIN_Number_Of_Tones; ++k){ + phrase_begin = phrase_end; + index++; + phrase_end = *index; + if ( phrase_begin == phrase_end ) //null pointer + continue; + PinyinLengthIndexLevel * phrases = new PinyinLengthIndexLevel; + m_pinyin_length_indexes[m][n][k] = phrases; + phrases->load(chunk, phrase_begin, phrase_end - 1); + assert( phrase_end <= end ); + assert( *(buf_begin + phrase_end - 1) == c_separate); + } + offset += (PINYIN_Number_Of_Initials * PINYIN_Number_Of_Finals * PINYIN_Number_Of_Tones + 1) * sizeof ( table_offset_t); + assert( c_separate == *(buf_begin + offset)); + return true; +} + +bool PinyinBitmapIndexLevel::store(MemoryChunk * new_chunk, + table_offset_t offset, + table_offset_t & end){ + table_offset_t phrase_end; + table_offset_t index = offset; + offset += (PINYIN_Number_Of_Initials * PINYIN_Number_Of_Finals * PINYIN_Number_Of_Tones + 1) * sizeof ( table_offset_t); + //add '#' + new_chunk->set_content(offset, &c_separate, sizeof(char)); + offset += sizeof(char); + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + index += sizeof(table_offset_t); + for ( int m = 0; m < PINYIN_Number_Of_Initials; ++m ) + for ( int n = 0; n < PINYIN_Number_Of_Finals; ++n) + for ( int k = 0; k < PINYIN_Number_Of_Tones; ++k){ + PinyinLengthIndexLevel * phrases = m_pinyin_length_indexes[m][n][k]; + if ( !phrases ){ //null pointer + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + index += sizeof(table_offset_t); + continue; + } + phrases->store(new_chunk, offset, phrase_end); //has a end '#' + offset = phrase_end; + //add '#' + new_chunk->set_content(offset, &c_separate, sizeof(char)); + offset += sizeof(char); + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + index += sizeof(table_offset_t); + } + end = offset; + return true; +} + +bool PinyinLengthIndexLevel::load(MemoryChunk * chunk, table_offset_t offset, table_offset_t end){ + char * buf_begin = (char *) chunk->begin(); + guint32 nindex = *((guint32 *)(buf_begin + offset)); + table_offset_t * index = (table_offset_t *) + (buf_begin + offset + sizeof(guint32)); + + table_offset_t phrase_begin, phrase_end = *index; + m_pinyin_array_indexes = g_array_new(FALSE, TRUE, sizeof(void *)); + for ( size_t i = 1; i <= nindex; i++){ + phrase_begin = phrase_end; + index++; + phrase_end = *index; + if ( phrase_begin == phrase_end ){ + void * null = NULL; + g_array_append_val(m_pinyin_array_indexes , null); + continue; + } + +#define CASE(x) case x - 1: \ + { \ + PinyinArrayIndexLevel * phrase = new PinyinArrayIndexLevel; \ + phrase->load(chunk, phrase_begin, phrase_end - 1); \ + assert( *(buf_begin + phrase_end - 1) == c_separate); \ + assert( phrase_end <= end ); \ + g_array_append_val(m_pinyin_array_indexes, phrase); \ + break; \ + } + switch ( i ){ + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + default: + assert(false); + } +#undef CASE + } + offset += sizeof(guint32) + (nindex + 1) * sizeof(table_offset_t); + assert ( c_separate == * (buf_begin + offset) ); + return true; +} + +bool PinyinLengthIndexLevel::store(MemoryChunk * new_chunk, table_offset_t offset, table_offset_t& end){ + guint32 nindex = m_pinyin_array_indexes->len; + new_chunk->set_content(offset, &nindex, sizeof(guint32)); + table_offset_t index = offset + sizeof(guint32); + + offset += sizeof(guint32) + (nindex + 1) * sizeof(table_offset_t); + new_chunk->set_content(offset, &c_separate, sizeof(char)); + offset += sizeof(char); + new_chunk->set_content(index, &offset, sizeof(table_offset_t)); + index += sizeof(table_offset_t); + table_offset_t phrase_end; + for ( size_t i = 0 ; i < m_pinyin_array_indexes->len; ++i){ +#define CASE(x) case x: { \ + PinyinArrayIndexLevel * phrase = g_array_index \ + (m_pinyin_array_indexes, PinyinArrayIndexLevel * , i); \ + if ( !phrase ){ \ + new_chunk->set_content \ + (index, &offset, sizeof(table_offset_t)); \ + index += sizeof(table_offset_t); \ + continue; \ + } \ + phrase->store(new_chunk, offset, phrase_end); \ + offset = phrase_end; \ + /*add '#'*/ \ + new_chunk->set_content(offset, &c_separate, sizeof(char)); \ + offset += sizeof(char); \ + new_chunk->set_content(index, &offset, sizeof(table_offset_t));\ + index += sizeof(table_offset_t); \ + break; \ + } + switch ( i ){ + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + default: + assert(false); + } +#undef CASE + } + end = offset; + return true; +} + +template +bool PinyinArrayIndexLevel:: +load(MemoryChunk * chunk, table_offset_t offset, table_offset_t end){ + char * buf_begin = (char *) chunk->begin(); + m_chunk.set_chunk(buf_begin + offset, end - offset, NULL); + return true; +} + +template +bool PinyinArrayIndexLevel:: +store(MemoryChunk * new_chunk, table_offset_t offset, table_offset_t& end){ + new_chunk->set_content(offset, m_chunk.begin(), m_chunk.size()); + end = offset + m_chunk.size(); + return true; +} diff --git a/src/storage/pinyin_large_table.h b/src/storage/pinyin_large_table.h new file mode 100755 index 0000000..71b3640 --- /dev/null +++ b/src/storage/pinyin_large_table.h @@ -0,0 +1,178 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef PINYIN_LARGE_TABLE_H +#define PINYIN_LARGE_TABLE_H + +#include +#include "novel_types.h" +#include "memory_chunk.h" + +namespace novel{ + +/* Because this is not large, + * Store this in user home directory. + */ + +class PinyinLengthIndexLevel; + +class PinyinBitmapIndexLevel{ + friend class PinyinLargeTable; + PinyinCustomSettings * m_custom; +protected: + PinyinLengthIndexLevel * m_pinyin_length_indexes[PINYIN_Number_Of_Initials] + [PINYIN_Number_Of_Finals] + [PINYIN_Number_Of_Tones]; + //search function + int initial_level_search(int word_length, /* in */PinyinKey keys[], + /* out */ PhraseIndexRanges ranges) const; + int final_level_search(PinyinInitial initial, int word_length, /* in */PinyinKey keys[], /* out */ PhraseIndexRanges ranges) const; + int tone_level_search(PinyinInitial initial, PinyinFinal final, int word_length, /* in */PinyinKey keys[], /* out */ PhraseIndexRanges ranges) const; + void reset(); +public: + PinyinBitmapIndexLevel(PinyinCustomSettings * custom); + ~PinyinBitmapIndexLevel(){ + reset(); + } + + bool load(MemoryChunk * chunk, table_offset_t offset, table_offset_t end); + bool store(MemoryChunk * new_chunk, table_offset_t offset, table_offset_t &end); + + /*bool load_text(FILE * file);*/ + /*bool save_text(FILE * file);*/ + + /*search/add_index method */ + int search( int phrase_length, /* in */ PinyinKey keys[], + /* out */ PhraseIndexRanges ranges) const; + int add_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token); + int remove_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token); +}; + +class PinyinLengthIndexLevel{ +protected: + GArray* m_pinyin_array_indexes; +public: + PinyinLengthIndexLevel(); + ~PinyinLengthIndexLevel(); + bool load(MemoryChunk * chunk, table_offset_t offset, table_offset_t end); + bool store(MemoryChunk * new_chunk, table_offset_t offset, table_offset_t& end); + + /*search/add_index method */ + int search( int phrase_length, /* in */ PinyinCustomSettings * custom, + /* in */ PinyinKey keys[], + /* out */ PhraseIndexRanges ranges); + int add_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token); + int remove_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token); +}; + +template +class PinyinArrayIndexLevel{ +protected: + MemoryChunk m_chunk; + int convert(PinyinCustomSettings * custom, + PinyinKey keys[], + PinyinIndexItem * begin, + PinyinIndexItem * end, + PhraseIndexRanges ranges); +public: + bool load(MemoryChunk * chunk, table_offset_t offset, table_offset_t end); + bool store(MemoryChunk * new_chunk, table_offset_t offset, table_offset_t& end); + + /*search/add_index method */ + int search(/* in */ PinyinCustomSettings * custom, + /* in */ PinyinKey keys[], + /* out */ PhraseIndexRanges ranges); + int add_index(/* in */ PinyinKey keys[] ,/* in */ phrase_token_t token); + int remove_index(/* in */ PinyinKey keys[] ,/* in */ phrase_token_t token); +}; + + +/* TODO: add file version check */ +class PinyinLargeTable{ +protected: + PinyinBitmapIndexLevel m_bitmap_table; + MemoryChunk * m_chunk; + + void reset(){ + if ( m_chunk ){ + delete m_chunk; + m_chunk = NULL; + } + } + +public: + PinyinLargeTable(PinyinCustomSettings * custom): + m_bitmap_table(custom){ + m_chunk = NULL; + } + + ~PinyinLargeTable(){ + reset(); + } + + /*load/save method*/ + bool load(MemoryChunk * chunk){ + reset(); + m_chunk = chunk; + return m_bitmap_table.load(chunk, 0 , chunk->size()); + } + + bool store(MemoryChunk * new_chunk){ + table_offset_t end; + return m_bitmap_table.store(new_chunk, 0, end); + } + + bool load_text(FILE * file); +/* + bool save_text(FILE * file){ + return m_bitmap_table.save_text(file); + } +*/ + + /*search/add_index method */ + int search( int phrase_length, /* in */ PinyinKey keys[], + /* out */ PhraseIndexRanges ranges){ + return m_bitmap_table.search(phrase_length, keys, ranges); + } + + int add_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + return m_bitmap_table.add_index(phrase_length, keys, token); + } + int remove_index( int phrase_length, /* in */ PinyinKey keys[] ,/* in */ phrase_token_t token){ + return m_bitmap_table.remove_index(phrase_length, keys, token); + } + + bool has_key(PinyinKey key) const { + PhraseIndexRanges ranges; + memset(ranges, 0, sizeof(ranges)); + ranges[1] = g_array_new(FALSE, FALSE, sizeof(PhraseIndexRange)); + int result = m_bitmap_table.search(1, &key, ranges); + g_array_free(ranges[1], TRUE); + ranges[1] = NULL; + return result & SEARCH_OK; + } +}; + +}; + +using namespace novel; +#endif diff --git a/src/storage/pinyin_phrase.h b/src/storage/pinyin_phrase.h new file mode 100644 index 0000000..07ee0de --- /dev/null +++ b/src/storage/pinyin_phrase.h @@ -0,0 +1,298 @@ +/* + * novel-pinyin, + * A Simplified Chinese Sentence-Based Pinyin Input Method Engine + * Based On Markov Model. + * + * Copyright (C) 2006-2007 Peng Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef PINYIN_PHRASE_H +#define PINYIN_PHRASE_H + +#include +#include "stl_lite.h" + +namespace novel{ + +static inline int pinyin_utility_sign(int value){ + if(value > 0) + return 1; + else if (value < 0) + return -1; + else return 0; +} + +inline int pinyin_exact_compare(const PinyinKey key_lhs[], + const PinyinKey key_rhs[], + int phrase_length){ + int i; + int result; + for ( i = 0 ; i < phrase_length ; i++){ + result = key_lhs[i].m_initial - key_rhs[i].m_initial; + if ( result != 0 ) + return pinyin_utility_sign(result); + } + for( i = 0 ; i < phrase_length ; i++){ + result = key_lhs[i].m_final - key_rhs[i].m_final; + if ( result != 0 ) + return pinyin_utility_sign(result); + } + for( i = 0 ; i < phrase_length ; i++){ + result = key_lhs[i].m_tone - key_rhs[i].m_tone; + if ( result != 0 ) + return pinyin_utility_sign(result); + } + return 0; +} + + +inline int pinyin_compare_with_ambiguities(const PinyinCustomSettings &custom, + const PinyinKey* key_lhs, + const PinyinKey* key_rhs, + int phrase_length){ + int i; + int result; + for ( i = 0 ; i < phrase_length ; i++){ + result = pinyin_compare_initial + (custom, + (PinyinInitial)key_lhs[i].m_initial, + (PinyinInitial)key_rhs[i].m_initial); + if ( result != 0 ) + return result; + } + for( i = 0 ; i < phrase_length ; i++){ + result = pinyin_compare_final + (custom, + (PinyinFinal)key_lhs[i].m_final, + (PinyinFinal)key_rhs[i].m_final); + if ( result != 0 ) + return result; + } + for( i = 0 ; i < phrase_length ; i++){ + result = pinyin_compare_tone + (custom, + (PinyinTone)key_lhs[i].m_tone, + (PinyinTone)key_rhs[i].m_tone); + if ( result != 0 ) + return result; + } + return 0; +} + +//compute pinyin lower bound +//maybe replace by table lookup +inline void compute_lower_value(const PinyinCustomSettings &custom, + PinyinKey in_keys[], + PinyinKey out_keys[], + int phrase_length){ + PinyinKey aKey = in_keys[0]; + + for ( int i = 0; i < phrase_length; i++){ + int k; int sel; + aKey = in_keys[i]; + //deal with initial + sel = aKey.m_initial; + for( k = aKey.m_initial - 1; k >= PINYIN_ZeroInitial; k--){ + if ( 0 != pinyin_compare_initial(custom, + (PinyinInitial)k, + (PinyinInitial)aKey.m_initial) ) + break; + else + sel = k; + } + aKey.m_initial = (PinyinInitial)sel; + //deal with final + sel = aKey.m_final; + for( k = aKey.m_final - 1; k >= PINYIN_ZeroFinal; k--){ + if ( 0 != pinyin_compare_final(custom, + (PinyinFinal)k, + (PinyinFinal)aKey.m_final) ) + break; + else + sel = k; + } + aKey.m_final = (PinyinFinal)sel; + //deal with tone + sel = aKey.m_tone; + for( k = aKey.m_tone - 1; k >= PINYIN_ZeroTone; k--){ + if ( 0 != pinyin_compare_tone(custom, + (PinyinTone)k, + (PinyinTone)aKey.m_tone) ) + break; + else + sel = k; + } + aKey.m_tone = (PinyinTone)sel; + //save the result + out_keys[i] = aKey; + } +} + +//compute pinyin upper bound +//maybe replace by table lookup +inline void compute_upper_value(const PinyinCustomSettings &custom, + PinyinKey in_keys[], + PinyinKey out_keys[], + int phrase_length){ + PinyinKey aKey = in_keys[0]; + + for ( int i = 0; i < phrase_length; i++){ + int k; int sel; + aKey = in_keys[i]; + //deal with initial + sel = aKey.m_initial; + for( k = aKey.m_initial + 1; k <= PINYIN_LastInitial; k++){ + if ( 0 != pinyin_compare_initial(custom, (PinyinInitial)k, (PinyinInitial)aKey.m_initial) ) + break; + else + sel = k; + } + aKey.m_initial = (PinyinInitial)sel; + //deal with final + sel = aKey.m_final; + for( k = aKey.m_final + 1; k <= PINYIN_LastFinal; k++){ + if ( 0 != pinyin_compare_final(custom, (PinyinFinal)k, (PinyinFinal)aKey.m_final) ) + break; + else + sel = k; + } + aKey.m_final = (PinyinFinal)sel; + //deal with tone + sel = aKey.m_tone; + for( k = aKey.m_tone + 1; k <= PINYIN_LastTone; k++){ + if ( 0 != pinyin_compare_tone(custom, (PinyinTone)k, (PinyinTone)aKey.m_tone) ) + break; + else + sel = k; + } + aKey.m_tone = (PinyinTone)sel; + //save the result + out_keys[i] = aKey; + } +} + +template +struct PinyinIndexItem{ + phrase_token_t m_token; + PinyinKey m_keys[phrase_length]; +public: + PinyinIndexItem(PinyinKey * keys, phrase_token_t token){ + memmove(m_keys, keys, sizeof(PinyinKey) * phrase_length); + m_token = token; + } +}; + +/* +//just need less than mode +//this method mainly used in pinyin lookup +template +class PhraseCompareWithAmbiguities + : public std_lite::binary_function , + const PinyinIndexItem , int> +{ + const PinyinCustomSettings & m_custom; +public: + PhraseCompareWithAmbiguities + (const PinyinCustomSettings & custom):m_custom(custom){} + + int operator () (const PinyinIndexItem &lhs, + const PinyinIndexItem &rhs) const{ + PinyinKey * key_lhs = (PinyinKey *) lhs.m_keys; + PinyinKey * key_rhs = (PinyinKey *) rhs.m_keys; + return pinyin_compare_with_ambiguities(m_custom, + key_lhs, key_rhs, phrase_length); + } +}; +*/ + +//for find the element in the phrase array +template +class PhraseExactCompare + : public std_lite::binary_function + ,const PinyinIndexItem, int> +{ +public: + int operator () (const PinyinIndexItem &lhs, + const PinyinIndexItem &rhs) const{ + PinyinKey * key_lhs = (PinyinKey *) lhs.m_keys; + PinyinKey * key_rhs = (PinyinKey *) rhs.m_keys; + + return pinyin_exact_compare(key_lhs, key_rhs, phrase_length); + } +}; + +/* +//for find the element in the phrase array +template +class PhraseExactCompareWithToken + : public std_lite::binary_function + ,const PinyinIndexItem, int> +{ +public: + int operator () (const PinyinIndexItem &lhs, + const PinyinIndexItem &rhs) const{ + PinyinKey * key_lhs = (PinyinKey *) lhs.m_keys; + PinyinKey * key_rhs = (PinyinKey *) rhs.m_keys; + + phrase_token_t token_lhs = lhs.m_token; + phrase_token_t token_rhs = rhs.m_token; + + int result = pinyin_exact_compare(key_lhs, key_rhs, phrase_length); + if ( !result ) + return result; + return pinyin_utility_sign(token_lhs - token_rhs); + } +}; +*/ + +template +class PhraseExactLessThan + : public std_lite::binary_function + ,const PinyinIndexItem, + bool> +{ + private: + PhraseExactCompare m_compare; + public: + bool operator () (const PinyinIndexItem &lhs, + const PinyinIndexItem &rhs) const{ + return -1 == m_compare(lhs, rhs); + } +}; + +/* +template +class PhraseExactLessThanWithToken + : public std_lite::binary_function + ,const PinyinIndexItem, + bool> +{ + private: + PhraseExactCompareWithToken m_compare; + public: + bool operator () (const PinyinIndexItem &lhs, + const PinyinIndexItem &rhs) const{ + return -1 == m_compare(lhs, rhs); + } +}; +*/ + +}; + +using namespace novel; + +#endif diff --git a/src/storage/pinyin_zhuyin_map_data.h b/src/storage/pinyin_zhuyin_map_data.h new file mode 100644 index 0000000..7557c5e --- /dev/null +++ b/src/storage/pinyin_zhuyin_map_data.h @@ -0,0 +1,582 @@ +static const PinyinKey __zhuyin_standard_map [][3] = +{ +/* */{PinyinKey(1) /* 1 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ! */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* " */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* # */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* $ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* % */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* & */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ' */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ( */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ) */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* * */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* + */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* , */{PinyinKey(42) /* eh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* - */{PinyinKey(66) /* er */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* . */{PinyinKey(150) /* ou */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* / */{PinyinKey(60) /* eng */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 0 */{PinyinKey(18) /* an */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 1 */{PinyinKey(240) /* b */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 2 */{PinyinKey(960) /* d */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 3 */{PinyinKey(3) /* 3 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 4 */{PinyinKey(4) /* 4 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 5 */{PinyinKey(5520) /* zh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 6 */{PinyinKey(2) /* 2 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 7 */{PinyinKey(5) /* 5 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 8 */{PinyinKey(6) /* a */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 9 */{PinyinKey(12) /* ai */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* : */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ; */{PinyinKey(24) /* ang */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* < */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* = */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* > */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ? */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* @ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* A */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* B */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* C */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* D */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* E */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* F */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* G */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* H */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* I */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* J */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* K */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* L */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* M */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* N */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* O */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* P */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* R */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* S */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* T */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* U */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* V */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* W */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* X */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Y */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Z */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* [ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* \ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ] */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ^ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* _ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ` */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* a */{PinyinKey(2640) /* m */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* b */{PinyinKey(3600) /* r */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* c */{PinyinKey(1680) /* h */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* d */{PinyinKey(2160) /* k */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* e */{PinyinKey(1440) /* g */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* f */{PinyinKey(3360) /* q */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* g */{PinyinKey(4080) /* sh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* h */{PinyinKey(480) /* c */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* i */{PinyinKey(138) /* o */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* j */{PinyinKey(156) /* u */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* k */{PinyinKey(36) /* e */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* l */{PinyinKey(30) /* ao */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* m */{PinyinKey(216) /* v */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* n */{PinyinKey(3840) /* s */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* o */{PinyinKey(48) /* ei */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* p */{PinyinKey(54) /* en */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* q */{PinyinKey(3120) /* p */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* r */{PinyinKey(1920) /* j */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* s */{PinyinKey(2880) /* n */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* t */{PinyinKey(720) /* ch */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* u */{PinyinKey(72) /* i */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* v */{PinyinKey(4800) /* x */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* w */{PinyinKey(4320) /* t */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* x */{PinyinKey(2400) /* l */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* y */{PinyinKey(5280) /* z */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* z */{PinyinKey(1200) /* f */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* { */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* | */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +}; + +static const PinyinKey __zhuyin_hsu_map [][3] = +{ +/* */{PinyinKey(1) /* 1 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ! */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* " */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* # */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* $ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* % */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* & */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ' */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ( */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ) */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* * */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* + */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* , */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* - */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* . */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* / */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 0 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 1 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 2 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 3 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 4 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 5 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 6 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 7 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 8 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 9 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* : */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ; */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* < */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* = */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* > */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ? */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* @ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* A */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* B */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* C */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* D */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* E */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* F */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* G */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* H */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* I */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* J */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* K */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* L */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* M */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* N */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* O */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* P */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* R */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* S */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* T */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* U */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* V */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* W */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* X */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Y */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Z */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* [ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* \ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ] */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ^ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* _ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ` */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* a */{PinyinKey(480) /* c */, PinyinKey(48) /* ei */, PinyinKey(0) /* */}, +/* b */{PinyinKey(240) /* b */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* c */{PinyinKey(4800) /* x */, PinyinKey(4080) /* sh */, PinyinKey(0) /* */}, +/* d */{PinyinKey(960) /* d */, PinyinKey(2) /* 2 */, PinyinKey(0) /* */}, +/* e */{PinyinKey(72) /* i */, PinyinKey(42) /* eh */, PinyinKey(0) /* */}, +/* f */{PinyinKey(1200) /* f */, PinyinKey(3) /* 3 */, PinyinKey(0) /* */}, +/* g */{PinyinKey(1440) /* g */, PinyinKey(36) /* e */, PinyinKey(0) /* */}, +/* h */{PinyinKey(1680) /* h */, PinyinKey(138) /* o */, PinyinKey(0) /* */}, +/* i */{PinyinKey(12) /* ai */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* j */{PinyinKey(1920) /* j */, PinyinKey(5520) /* zh */, PinyinKey(4) /* 4 */}, +/* k */{PinyinKey(2160) /* k */, PinyinKey(24) /* ang */, PinyinKey(0) /* */}, +/* l */{PinyinKey(2400) /* l */, PinyinKey(60) /* eng */, PinyinKey(66) /* er */}, +/* m */{PinyinKey(2640) /* m */, PinyinKey(18) /* an */, PinyinKey(0) /* */}, +/* n */{PinyinKey(2880) /* n */, PinyinKey(54) /* en */, PinyinKey(0) /* */}, +/* o */{PinyinKey(150) /* ou */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* p */{PinyinKey(3120) /* p */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* r */{PinyinKey(3600) /* r */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* s */{PinyinKey(3840) /* s */, PinyinKey(5) /* 5 */, PinyinKey(0) /* */}, +/* t */{PinyinKey(4320) /* t */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* u */{PinyinKey(216) /* v */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* v */{PinyinKey(3360) /* q */, PinyinKey(720) /* ch */, PinyinKey(0) /* */}, +/* w */{PinyinKey(30) /* ao */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* x */{PinyinKey(156) /* u */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* y */{PinyinKey(6) /* a */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* z */{PinyinKey(5280) /* z */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* { */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* | */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +}; + +static const PinyinKey __zhuyin_ibm_map [][3] = +{ +/* */{PinyinKey(1) /* 1 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ! */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* " */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* # */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* $ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* % */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* & */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ' */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ( */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ) */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* * */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* + */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* , */{PinyinKey(3) /* 3 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* - */{PinyinKey(1680) /* h */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* . */{PinyinKey(4) /* 4 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* / */{PinyinKey(5) /* 5 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 0 */{PinyinKey(2160) /* k */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 1 */{PinyinKey(240) /* b */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 2 */{PinyinKey(3120) /* p */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 3 */{PinyinKey(2640) /* m */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 4 */{PinyinKey(1200) /* f */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 5 */{PinyinKey(960) /* d */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 6 */{PinyinKey(4320) /* t */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 7 */{PinyinKey(2880) /* n */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 8 */{PinyinKey(2400) /* l */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 9 */{PinyinKey(1440) /* g */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* : */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ; */{PinyinKey(30) /* ao */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* < */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* = */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* > */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ? */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* @ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* A */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* B */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* C */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* D */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* E */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* F */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* G */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* H */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* I */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* J */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* K */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* L */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* M */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* N */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* O */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* P */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* R */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* S */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* T */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* U */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* V */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* W */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* X */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Y */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Z */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* [ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* \ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ] */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ^ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* _ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ` */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* a */{PinyinKey(72) /* i */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* b */{PinyinKey(60) /* eng */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* c */{PinyinKey(54) /* en */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* d */{PinyinKey(216) /* v */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* e */{PinyinKey(4800) /* x */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* f */{PinyinKey(6) /* a */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* g */{PinyinKey(138) /* o */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* h */{PinyinKey(36) /* e */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* i */{PinyinKey(5280) /* z */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* j */{PinyinKey(42) /* eh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* k */{PinyinKey(12) /* ai */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* l */{PinyinKey(48) /* ei */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* m */{PinyinKey(2) /* 2 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* n */{PinyinKey(66) /* er */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* o */{PinyinKey(480) /* c */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* p */{PinyinKey(3840) /* s */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* q */{PinyinKey(1920) /* j */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* r */{PinyinKey(5520) /* zh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* s */{PinyinKey(156) /* u */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* t */{PinyinKey(720) /* ch */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* u */{PinyinKey(3600) /* r */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* v */{PinyinKey(24) /* ang */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* w */{PinyinKey(3360) /* q */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* x */{PinyinKey(18) /* an */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* y */{PinyinKey(4080) /* sh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* z */{PinyinKey(150) /* ou */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* { */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* | */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +}; + +static const PinyinKey __zhuyin_gin_yieh_map [][3] = +{ +/* */{PinyinKey(1) /* 1 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ! */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* " */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* # */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* $ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* % */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* & */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ' */{PinyinKey(60) /* eng */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ( */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ) */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* * */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* + */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* , */{PinyinKey(6) /* a */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* - */{PinyinKey(54) /* en */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* . */{PinyinKey(12) /* ai */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* / */{PinyinKey(18) /* an */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 0 */{PinyinKey(48) /* ei */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 1 */{PinyinKey(5) /* 5 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 2 */{PinyinKey(240) /* b */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 3 */{PinyinKey(960) /* d */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 4 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 5 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 6 */{PinyinKey(5520) /* zh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 7 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 8 */{PinyinKey(72) /* i */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 9 */{PinyinKey(138) /* o */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* : */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ; */{PinyinKey(150) /* ou */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* < */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* = */{PinyinKey(66) /* er */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* > */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ? */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* @ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* A */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* B */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* C */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* D */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* E */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* F */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* G */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* H */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* I */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* J */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* K */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* L */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* M */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* N */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* O */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* P */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* R */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* S */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* T */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* U */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* V */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* W */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* X */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Y */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Z */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* [ */{PinyinKey(24) /* ang */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* \ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ] */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ^ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* _ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ` */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* a */{PinyinKey(3) /* 3 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* b */{PinyinKey(4800) /* x */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* c */{PinyinKey(2400) /* l */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* d */{PinyinKey(2880) /* n */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* e */{PinyinKey(4320) /* t */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* f */{PinyinKey(2160) /* k */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* g */{PinyinKey(3360) /* q */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* h */{PinyinKey(4080) /* sh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* i */{PinyinKey(156) /* u */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* j */{PinyinKey(480) /* c */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* k */{PinyinKey(216) /* v */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* l */{PinyinKey(42) /* eh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* m */{PinyinKey(3840) /* s */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* n */{PinyinKey(3600) /* r */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* o */{PinyinKey(36) /* e */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* p */{PinyinKey(30) /* ao */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* q */{PinyinKey(2) /* 2 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* r */{PinyinKey(1440) /* g */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* s */{PinyinKey(2640) /* m */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* t */{PinyinKey(1920) /* j */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* u */{PinyinKey(5280) /* z */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* v */{PinyinKey(1680) /* h */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* w */{PinyinKey(3120) /* p */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* x */{PinyinKey(1200) /* f */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* y */{PinyinKey(720) /* ch */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* z */{PinyinKey(4) /* 4 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* { */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* | */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +}; + +static const PinyinKey __zhuyin_et_map [][3] = +{ +/* */{PinyinKey(1) /* 1 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ! */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* " */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* # */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* $ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* % */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* & */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ' */{PinyinKey(480) /* c */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ( */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ) */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* * */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* + */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* , */{PinyinKey(5520) /* zh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* - */{PinyinKey(60) /* eng */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* . */{PinyinKey(720) /* ch */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* / */{PinyinKey(4080) /* sh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 0 */{PinyinKey(24) /* ang */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 1 */{PinyinKey(5) /* 5 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 2 */{PinyinKey(2) /* 2 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 3 */{PinyinKey(3) /* 3 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 4 */{PinyinKey(4) /* 4 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 5 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 6 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 7 */{PinyinKey(3360) /* q */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 8 */{PinyinKey(18) /* an */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 9 */{PinyinKey(54) /* en */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* : */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ; */{PinyinKey(5280) /* z */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* < */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* = */{PinyinKey(66) /* er */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* > */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ? */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* @ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* A */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* B */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* C */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* D */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* E */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* F */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* G */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* H */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* I */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* J */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* K */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* L */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* M */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* N */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* O */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* P */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* R */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* S */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* T */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* U */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* V */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* W */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* X */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Y */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Z */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* [ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* \ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ] */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ^ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* _ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ` */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* a */{PinyinKey(6) /* a */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* b */{PinyinKey(240) /* b */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* c */{PinyinKey(4800) /* x */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* d */{PinyinKey(960) /* d */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* e */{PinyinKey(72) /* i */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* f */{PinyinKey(1200) /* f */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* g */{PinyinKey(1920) /* j */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* h */{PinyinKey(1680) /* h */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* i */{PinyinKey(12) /* ai */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* j */{PinyinKey(3600) /* r */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* k */{PinyinKey(2160) /* k */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* l */{PinyinKey(2400) /* l */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* m */{PinyinKey(2640) /* m */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* n */{PinyinKey(2880) /* n */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* o */{PinyinKey(138) /* o */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* p */{PinyinKey(3120) /* p */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* q */{PinyinKey(48) /* ei */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* r */{PinyinKey(36) /* e */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* s */{PinyinKey(3840) /* s */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* t */{PinyinKey(4320) /* t */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* u */{PinyinKey(216) /* v */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* v */{PinyinKey(1440) /* g */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* w */{PinyinKey(42) /* eh */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* x */{PinyinKey(156) /* u */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* y */{PinyinKey(150) /* ou */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* z */{PinyinKey(30) /* ao */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* { */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* | */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +}; + +static const PinyinKey __zhuyin_et26_map [][3] = +{ +/* */{PinyinKey(1) /* 1 */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ! */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* " */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* # */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* $ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* % */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* & */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ' */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ( */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ) */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* * */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* + */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* , */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* - */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* . */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* / */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 0 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 1 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 2 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 3 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 4 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 5 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 6 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 7 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 8 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* 9 */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* : */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ; */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* < */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* = */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* > */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ? */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* @ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* A */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* B */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* C */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* D */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* E */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* F */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* G */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* H */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* I */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* J */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* K */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* L */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* M */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* N */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* O */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* P */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Q */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* R */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* S */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* T */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* U */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* V */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* W */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* X */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Y */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* Z */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* [ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* \ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ] */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ^ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* _ */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* ` */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* a */{PinyinKey(6) /* a */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* b */{PinyinKey(240) /* b */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* c */{PinyinKey(4800) /* x */, PinyinKey(4080) /* sh */, PinyinKey(0) /* */}, +/* d */{PinyinKey(960) /* d */, PinyinKey(5) /* 5 */, PinyinKey(0) /* */}, +/* e */{PinyinKey(72) /* i */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* f */{PinyinKey(1200) /* f */, PinyinKey(2) /* 2 */, PinyinKey(0) /* */}, +/* g */{PinyinKey(1920) /* j */, PinyinKey(5520) /* zh */, PinyinKey(0) /* */}, +/* h */{PinyinKey(1680) /* h */, PinyinKey(66) /* er */, PinyinKey(0) /* */}, +/* i */{PinyinKey(12) /* ai */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* j */{PinyinKey(3600) /* r */, PinyinKey(3) /* 3 */, PinyinKey(0) /* */}, +/* k */{PinyinKey(2160) /* k */, PinyinKey(4) /* 4 */, PinyinKey(0) /* */}, +/* l */{PinyinKey(2400) /* l */, PinyinKey(60) /* eng */, PinyinKey(0) /* */}, +/* m */{PinyinKey(2640) /* m */, PinyinKey(18) /* an */, PinyinKey(0) /* */}, +/* n */{PinyinKey(2880) /* n */, PinyinKey(54) /* en */, PinyinKey(0) /* */}, +/* o */{PinyinKey(138) /* o */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* p */{PinyinKey(3120) /* p */, PinyinKey(150) /* ou */, PinyinKey(0) /* */}, +/* q */{PinyinKey(5280) /* z */, PinyinKey(48) /* ei */, PinyinKey(0) /* */}, +/* r */{PinyinKey(36) /* e */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* s */{PinyinKey(3840) /* s */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* t */{PinyinKey(4320) /* t */, PinyinKey(24) /* ang */, PinyinKey(0) /* */}, +/* u */{PinyinKey(216) /* v */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* v */{PinyinKey(1440) /* g */, PinyinKey(3360) /* q */, PinyinKey(0) /* */}, +/* w */{PinyinKey(480) /* c */, PinyinKey(42) /* eh */, PinyinKey(0) /* */}, +/* x */{PinyinKey(156) /* u */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* y */{PinyinKey(720) /* ch */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* z */{PinyinKey(30) /* ao */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* { */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +/* | */{PinyinKey(0) /* */, PinyinKey(0) /* */, PinyinKey(0) /* */}, +}; + -- cgit