diff options
Diffstat (limited to 'src/storage')
-rw-r--r-- | src/storage/flexible_ngram.h | 391 | ||||
-rw-r--r-- | src/storage/flexible_ngram_bdb.h | 421 |
2 files changed, 424 insertions, 388 deletions
diff --git a/src/storage/flexible_ngram.h b/src/storage/flexible_ngram.h index 6cff7ff..9589d45 100644 --- a/src/storage/flexible_ngram.h +++ b/src/storage/flexible_ngram.h @@ -24,8 +24,6 @@ #ifndef FLEXIBLE_NGRAM_H #define FLEXIBLE_NGRAM_H -#include <db.h> -#include <errno.h> /* Note: the signature of the template parameters. * struct MagicHeader, ArrayHeader, ArrayItem. @@ -327,393 +325,10 @@ public: } }; -/** - * FlexibleBigram: - * @MagicHeader: the struct type of the magic header. - * @ArrayHeader: the struct type of the array header. - * @ArrayItem: the struct type of the array item. - * - * The flexible bi-gram is mainly used for training purpose. - * - */ -template<typename MagicHeader, typename ArrayHeader, - typename ArrayItem> -class FlexibleBigram{ - /* Note: some flexible bi-gram file format check should be here. */ -private: - DB * m_db; - - phrase_token_t m_magic_header_index[2]; - - char m_magic_number[4]; - - void reset(){ - if ( m_db ){ - m_db->sync(m_db, 0); - m_db->close(m_db, 0); - m_db = NULL; - } - } - -public: - /** - * FlexibleBigram::FlexibleBigram: - * @magic_number: the 4 bytes magic number of the flexible bi-gram. - * - * The constructor of the FlexibleBigram. - * - */ - FlexibleBigram(const char * magic_number){ - m_db = NULL; - m_magic_header_index[0] = null_token; - m_magic_header_index[1] = null_token; - - memcpy(m_magic_number, magic_number, sizeof(m_magic_number)); - } - - /** - * FlexibleBigram::~FlexibleBigram: - * - * The destructor of the FlexibleBigram. - * - */ - ~FlexibleBigram(){ - reset(); - } - - /** - * FlexibleBigram::attach: - * @dbfile: the path name of the flexible bi-gram. - * @flags: the attach flags for the Berkeley DB. - * @returns: whether the attach operation is successful. - * - * Attach Berkeley DB on filesystem for training purpose. - * - */ - bool attach(const char * dbfile, guint32 flags){ - reset(); - u_int32_t db_flags = 0; - - if ( flags & ATTACH_READONLY ) - db_flags |= DB_RDONLY; - if ( flags & ATTACH_READWRITE ) - assert( !(flags & ATTACH_READONLY ) ); - - if ( !dbfile ) - return false; - int ret = db_create(&m_db, NULL, 0); - if ( ret != 0 ) - assert(false); - - ret = m_db->open(m_db, NULL, dbfile, NULL, DB_HASH, db_flags, 0644); - if ( ret != 0 && (flags & ATTACH_CREATE) ) { - db_flags |= DB_CREATE; - /* Create database file here, and write the signature. */ - ret = m_db->open(m_db, NULL, dbfile, NULL, DB_HASH, db_flags, 0644); - if ( ret != 0 ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = m_magic_header_index; - db_key.size = sizeof(m_magic_header_index); - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.data = m_magic_number; - db_data.size = sizeof(m_magic_number); - db_data.flags = DB_DBT_PARTIAL; - db_data.doff = 0; - db_data.dlen = sizeof(m_magic_number); - - ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); - return ret == 0; - } - - /* check the signature. */ - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = m_magic_header_index; - db_key.size = sizeof(m_magic_header_index); - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.flags = DB_DBT_PARTIAL; - db_data.doff = 0; - db_data.dlen = sizeof(m_magic_number); - ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); - if ( ret != 0 ) - return false; - if ( sizeof(m_magic_number) != db_data.size ) - return false; - if ( memcmp(db_data.data, m_magic_number, - sizeof(m_magic_number)) == 0 ) - return true; - return false; - } - - /** - * FlexibleBigram::load: - * @index: the previous token in the flexible bi-gram. - * @single_gram: the single gram of the previous token. - * @returns: whether the load operation is successful. - * - * Load the single gram of the previous token. - * - */ - bool load(phrase_token_t index, - FlexibleSingleGram<ArrayHeader, ArrayItem> * & single_gram){ - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = &index; - db_key.size = sizeof(phrase_token_t); - - single_gram = NULL; - - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - int ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); - if ( ret != 0) - return false; - - single_gram = new FlexibleSingleGram<ArrayHeader, ArrayItem> - (db_data.data, db_data.size); - - return true; - } - - /** - * FlexibleBigram::store: - * @index: the previous token in the flexible bi-gram. - * @single_gram: the single gram of the previous token. - * @returns: whether the store operation is successful. - * - * Store the single gram of the previous token. - * - */ - bool store(phrase_token_t index, - FlexibleSingleGram<ArrayHeader, ArrayItem> * single_gram){ - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = &index; - db_key.size = sizeof(phrase_token_t); - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.data = single_gram->m_chunk.begin(); - db_data.size = single_gram->m_chunk.size(); - - int ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); - return ret == 0; - } - - /** - * FlexibleBigram::remove: - * @index: the previous token in the flexible bi-gram. - * @returns: whether the remove operation is successful. - * - * Remove the single gram of the previous token. - * - */ - bool remove(phrase_token_t index){ - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = &index; - db_key.size = sizeof(phrase_token_t); - - int ret = m_db->del(m_db, NULL, &db_key, 0); - return ret == 0; - } - - /** - * FlexibleBigram::get_all_items: - * @items: the GArray to store all previous tokens. - * @returns: whether the get operation is successful. - * - * Get the array of all previous tokens for parameter estimation. - * - */ - bool get_all_items(GArray * items){ - g_array_set_size(items, 0); - - if ( !m_db ) - return false; - - DBC * cursorp; - DBT key, data; - int ret; - - /* Get a cursor */ - m_db->cursor(m_db, NULL, &cursorp, 0); - - if (NULL == cursorp) - return false; - - /* Initialize our DBTs. */ - memset(&key, 0, sizeof(DBT)); - memset(&data, 0, sizeof(DBT)); - - /* Iterate over the database, retrieving each record in turn. */ - while ((ret = cursorp->c_get(cursorp, &key, &data, DB_NEXT)) == 0 ){ - if (key.size != sizeof(phrase_token_t)){ - /* skip magic header. */ - continue; - } - phrase_token_t * token = (phrase_token_t *) key.data; - g_array_append_val(items, *token); - } - - if ( ret != DB_NOTFOUND ){ - fprintf(stderr, "training db error, exit!"); - - if (cursorp != NULL) - cursorp->c_close(cursorp); - - exit(EIO); - } - - /* Cursors must be closed */ - if (cursorp != NULL) - cursorp->c_close(cursorp); - return true; - } - - /** - * FlexibleBigram::get_magic_header: - * @header: the magic header. - * @returns: whether the get operation is successful. - * - * Get the magic header of the flexible bi-gram. - * - */ - bool get_magic_header(MagicHeader & header){ - /* clear retval */ - memset(&header, 0, sizeof(MagicHeader)); - - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = m_magic_header_index; - db_key.size = sizeof(m_magic_header_index); - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.flags = DB_DBT_PARTIAL; - db_data.doff = sizeof(m_magic_number); - db_data.dlen = sizeof(MagicHeader); - - int ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); - if ( ret != 0 ) - return false; - - if ( sizeof(MagicHeader) != db_data.size ) - return false; - - memcpy(&header, db_data.data, sizeof(MagicHeader)); - return true; - } - - /** - * FlexibleBigram::set_magic_header: - * @header: the magic header. - * @returns: whether the set operation is successful. - * - * Set the magic header of the flexible bi-gram. - * - */ - bool set_magic_header(const MagicHeader & header){ - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = m_magic_header_index; - db_key.size = sizeof(m_magic_header_index); - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.data = (void *) &header; - db_data.size = sizeof(MagicHeader); - db_data.flags = DB_DBT_PARTIAL; - db_data.doff = sizeof(m_magic_number); - db_data.dlen = sizeof(MagicHeader); - - int ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); - return ret == 0; - } - - /** - * FlexibleBigram::get_array_header: - * @index: the previous token in the flexible bi-gram. - * @header: the array header in the single gram of the previous token. - * @returns: whether the get operation is successful. - * - * Get the array header in the single gram of the previous token. - * - */ - bool get_array_header(phrase_token_t index, ArrayHeader & header){ - /* clear retval */ - memset(&header, 0, sizeof(ArrayHeader)); - - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = &index; - db_key.size = sizeof(phrase_token_t); - - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.flags = DB_DBT_PARTIAL; - db_data.doff = 0; - db_data.dlen = sizeof(ArrayHeader); - int ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); - if ( ret != 0 ) - return false; - - assert(db_data.size == sizeof(ArrayHeader)); - memcpy(&header, db_data.data, sizeof(ArrayHeader)); - return true; - } - - /** - * FlexibleBigram::set_array_header: - * @index: the previous token of the flexible bi-gram. - * @header: the array header in the single gram of the previous token. - * @returns: whether the set operation is successful. - * - * Set the array header in the single gram of the previous token. - * - */ - bool set_array_header(phrase_token_t index, const ArrayHeader & header){ - if ( !m_db ) - return false; - - DBT db_key; - memset(&db_key, 0, sizeof(DBT)); - db_key.data = &index; - db_key.size = sizeof(phrase_token_t); - DBT db_data; - memset(&db_data, 0, sizeof(DBT)); - db_data.data = (void *)&header; - db_data.size = sizeof(ArrayHeader); - db_data.flags = DB_DBT_PARTIAL; - db_data.doff = 0; - db_data.dlen = sizeof(ArrayHeader); - - int ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); - return ret == 0; - } - }; -}; +#ifdef HAVE_BERKELEY_DB +#include "flexible_ngram_bdb.h" +#endif #endif diff --git a/src/storage/flexible_ngram_bdb.h b/src/storage/flexible_ngram_bdb.h new file mode 100644 index 0000000..861667e --- /dev/null +++ b/src/storage/flexible_ngram_bdb.h @@ -0,0 +1,421 @@ +/* + * libpinyin + * Library to deal with pinyin. + * + * Copyright (C) 2015 Peng Wu <alexepico@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef FLEXIBLE_NGRAM_BDB_H +#define FLEXIBLE_NGRAM_BDB_H + +#ifdef HAVE_BERKELEY_DB +#include <errno.h> +#include <db.h> +#endif + +namespace pinyin{ + +/** + * FlexibleBigram: + * @MagicHeader: the struct type of the magic header. + * @ArrayHeader: the struct type of the array header. + * @ArrayItem: the struct type of the array item. + * + * The flexible bi-gram is mainly used for training purpose. + * + */ +template<typename MagicHeader, typename ArrayHeader, + typename ArrayItem> +class FlexibleBigram{ + /* Note: some flexible bi-gram file format check should be here. */ +private: + DB * m_db; + + phrase_token_t m_magic_header_index[2]; + + char m_magic_number[4]; + + void reset(){ + if ( m_db ){ + m_db->sync(m_db, 0); + m_db->close(m_db, 0); + m_db = NULL; + } + } + +public: + /** + * FlexibleBigram::FlexibleBigram: + * @magic_number: the 4 bytes magic number of the flexible bi-gram. + * + * The constructor of the FlexibleBigram. + * + */ + FlexibleBigram(const char * magic_number){ + m_db = NULL; + m_magic_header_index[0] = null_token; + m_magic_header_index[1] = null_token; + + memcpy(m_magic_number, magic_number, sizeof(m_magic_number)); + } + + /** + * FlexibleBigram::~FlexibleBigram: + * + * The destructor of the FlexibleBigram. + * + */ + ~FlexibleBigram(){ + reset(); + } + + /** + * FlexibleBigram::attach: + * @dbfile: the path name of the flexible bi-gram. + * @flags: the attach flags for the Berkeley DB. + * @returns: whether the attach operation is successful. + * + * Attach Berkeley DB on filesystem for training purpose. + * + */ + bool attach(const char * dbfile, guint32 flags){ + reset(); + u_int32_t db_flags = 0; + + if ( flags & ATTACH_READONLY ) + db_flags |= DB_RDONLY; + if ( flags & ATTACH_READWRITE ) + assert( !(flags & ATTACH_READONLY ) ); + + if ( !dbfile ) + return false; + int ret = db_create(&m_db, NULL, 0); + if ( ret != 0 ) + assert(false); + + ret = m_db->open(m_db, NULL, dbfile, NULL, DB_HASH, db_flags, 0644); + if ( ret != 0 && (flags & ATTACH_CREATE) ) { + db_flags |= DB_CREATE; + /* Create database file here, and write the signature. */ + ret = m_db->open(m_db, NULL, dbfile, NULL, DB_HASH, db_flags, 0644); + if ( ret != 0 ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = m_magic_header_index; + db_key.size = sizeof(m_magic_header_index); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.data = m_magic_number; + db_data.size = sizeof(m_magic_number); + db_data.flags = DB_DBT_PARTIAL; + db_data.doff = 0; + db_data.dlen = sizeof(m_magic_number); + + ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); + return ret == 0; + } + + /* check the signature. */ + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = m_magic_header_index; + db_key.size = sizeof(m_magic_header_index); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.flags = DB_DBT_PARTIAL; + db_data.doff = 0; + db_data.dlen = sizeof(m_magic_number); + ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); + if ( ret != 0 ) + return false; + if ( sizeof(m_magic_number) != db_data.size ) + return false; + if ( memcmp(db_data.data, m_magic_number, + sizeof(m_magic_number)) == 0 ) + return true; + return false; + } + + /** + * FlexibleBigram::load: + * @index: the previous token in the flexible bi-gram. + * @single_gram: the single gram of the previous token. + * @returns: whether the load operation is successful. + * + * Load the single gram of the previous token. + * + */ + bool load(phrase_token_t index, + FlexibleSingleGram<ArrayHeader, ArrayItem> * & single_gram){ + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + + single_gram = NULL; + + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + int ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); + if ( ret != 0) + return false; + + single_gram = new FlexibleSingleGram<ArrayHeader, ArrayItem> + (db_data.data, db_data.size); + + return true; + } + + /** + * FlexibleBigram::store: + * @index: the previous token in the flexible bi-gram. + * @single_gram: the single gram of the previous token. + * @returns: whether the store operation is successful. + * + * Store the single gram of the previous token. + * + */ + bool store(phrase_token_t index, + FlexibleSingleGram<ArrayHeader, ArrayItem> * single_gram){ + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.data = single_gram->m_chunk.begin(); + db_data.size = single_gram->m_chunk.size(); + + int ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); + return ret == 0; + } + + /** + * FlexibleBigram::remove: + * @index: the previous token in the flexible bi-gram. + * @returns: whether the remove operation is successful. + * + * Remove the single gram of the previous token. + * + */ + bool remove(phrase_token_t index){ + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + + int ret = m_db->del(m_db, NULL, &db_key, 0); + return ret == 0; + } + + /** + * FlexibleBigram::get_all_items: + * @items: the GArray to store all previous tokens. + * @returns: whether the get operation is successful. + * + * Get the array of all previous tokens for parameter estimation. + * + */ + bool get_all_items(GArray * items){ + g_array_set_size(items, 0); + + if ( !m_db ) + return false; + + DBC * cursorp; + DBT key, data; + int ret; + + /* Get a cursor */ + m_db->cursor(m_db, NULL, &cursorp, 0); + + if (NULL == cursorp) + return false; + + /* Initialize our DBTs. */ + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + + /* Iterate over the database, retrieving each record in turn. */ + while ((ret = cursorp->c_get(cursorp, &key, &data, DB_NEXT)) == 0 ){ + if (key.size != sizeof(phrase_token_t)){ + /* skip magic header. */ + continue; + } + phrase_token_t * token = (phrase_token_t *) key.data; + g_array_append_val(items, *token); + } + + if ( ret != DB_NOTFOUND ){ + fprintf(stderr, "training db error, exit!"); + + if (cursorp != NULL) + cursorp->c_close(cursorp); + + exit(EIO); + } + + /* Cursors must be closed */ + if (cursorp != NULL) + cursorp->c_close(cursorp); + return true; + } + + /** + * FlexibleBigram::get_magic_header: + * @header: the magic header. + * @returns: whether the get operation is successful. + * + * Get the magic header of the flexible bi-gram. + * + */ + bool get_magic_header(MagicHeader & header){ + /* clear retval */ + memset(&header, 0, sizeof(MagicHeader)); + + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = m_magic_header_index; + db_key.size = sizeof(m_magic_header_index); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.flags = DB_DBT_PARTIAL; + db_data.doff = sizeof(m_magic_number); + db_data.dlen = sizeof(MagicHeader); + + int ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); + if ( ret != 0 ) + return false; + + if ( sizeof(MagicHeader) != db_data.size ) + return false; + + memcpy(&header, db_data.data, sizeof(MagicHeader)); + return true; + } + + /** + * FlexibleBigram::set_magic_header: + * @header: the magic header. + * @returns: whether the set operation is successful. + * + * Set the magic header of the flexible bi-gram. + * + */ + bool set_magic_header(const MagicHeader & header){ + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = m_magic_header_index; + db_key.size = sizeof(m_magic_header_index); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.data = (void *) &header; + db_data.size = sizeof(MagicHeader); + db_data.flags = DB_DBT_PARTIAL; + db_data.doff = sizeof(m_magic_number); + db_data.dlen = sizeof(MagicHeader); + + int ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); + return ret == 0; + } + + /** + * FlexibleBigram::get_array_header: + * @index: the previous token in the flexible bi-gram. + * @header: the array header in the single gram of the previous token. + * @returns: whether the get operation is successful. + * + * Get the array header in the single gram of the previous token. + * + */ + bool get_array_header(phrase_token_t index, ArrayHeader & header){ + /* clear retval */ + memset(&header, 0, sizeof(ArrayHeader)); + + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.flags = DB_DBT_PARTIAL; + db_data.doff = 0; + db_data.dlen = sizeof(ArrayHeader); + int ret = m_db->get(m_db, NULL, &db_key, &db_data, 0); + if ( ret != 0 ) + return false; + + assert(db_data.size == sizeof(ArrayHeader)); + memcpy(&header, db_data.data, sizeof(ArrayHeader)); + return true; + } + + /** + * FlexibleBigram::set_array_header: + * @index: the previous token of the flexible bi-gram. + * @header: the array header in the single gram of the previous token. + * @returns: whether the set operation is successful. + * + * Set the array header in the single gram of the previous token. + * + */ + bool set_array_header(phrase_token_t index, const ArrayHeader & header){ + if ( !m_db ) + return false; + + DBT db_key; + memset(&db_key, 0, sizeof(DBT)); + db_key.data = &index; + db_key.size = sizeof(phrase_token_t); + DBT db_data; + memset(&db_data, 0, sizeof(DBT)); + db_data.data = (void *)&header; + db_data.size = sizeof(ArrayHeader); + db_data.flags = DB_DBT_PARTIAL; + db_data.doff = 0; + db_data.dlen = sizeof(ArrayHeader); + + int ret = m_db->put(m_db, NULL, &db_key, &db_data, 0); + return ret == 0; + } + +}; + +}; + +#endif |