/*
* This file is part of rasdaman community.
*
* Rasdaman community is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Rasdaman community is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with rasdaman community. If not, see .
*
* Copyright 2003, 2004, 2005, 2006, 2007, 2008, 2009 Peter Baumann /
rasdaman GmbH.
*
* For more information please see
* or contact Peter Baumann via .
/
/**
* INCLUDE: lincompstream.cc
*
* MODULE: compression
* CLASS: r_LinCompStream, r_LinDecompStream
*
* COMMENTS:
*
*/
#include
#include
#include
#include
#include "raslib/rminit.hh"
#include "raslib/rmdebug.hh"
#include "raslib/parseparams.hh"
#include "raslib/memblockvec.hh"
#include "compression/lincompstream.hh"
// for creation of linear (de)compression streams from identifiers
#include "compression/nocompstream.hh"
#include "compression/rlestream.hh"
#include "compression/zlibstream.hh"
#include "compression/arithstream.hh"
#include "compression/filestream.hh"
#include "compression/compresstime.hh"
const char r_Linear_Stream::bankNameSep = ':';
r_Linear_Stream::r_Linear_Stream( void )
{
params = NULL;
myTimer = NULL;
}
r_Linear_Stream::~r_Linear_Stream( void )
{
if (params != NULL)
delete params;
CBENCH_STATEMENT(if (myTimer != NULL) delete myTimer);
}
r_Linear_Stream::r_Lin_Stream_Format r_Linear_Stream::get_id( const char *name )
{
if (strcasecmp(name, "none") == 0)
return r_Lin_Stream_None;
else if (strcasecmp(name, "rle") == 0)
return r_Lin_Stream_RLE;
else if (strcasecmp(name, "zlib") == 0)
return r_Lin_Stream_ZLib;
else if (strcasecmp(name, "arith") == 0)
return r_Lin_Stream_Arith;
else if (strcasecmp(name, "file") == 0)
return r_Lin_Stream_File;
return r_Lin_Stream_NUMBER;
}
const char *r_Linear_Stream::get_format_string( r_Lin_Stream_Format fmt )
{
switch (fmt)
{
case r_Lin_Stream_None:
return "None";
case r_Lin_Stream_RLE:
return "RLE";
case r_Lin_Stream_ZLib:
return "ZLib";
case r_Lin_Stream_Arith:
return "Arithmetic";
case r_Lin_Stream_File:
return "File";
default:
break;
}
return "???";
}
r_Linear_Stream::r_Lin_Stream_Format *r_Linear_Stream::get_bank_ids( const char *names )
{
r_Lin_Stream_Format *fmts;
unsigned int number, i;
const char *d;
char buffer[32];
char *b;
d = names; number = 1;
while (*d != '\0')
{
if (*d == bankNameSep) number++;
d++;
}
fmts = new r_Lin_Stream_Format[number+1];
d = names; i = 0;
while (*d != '\0')
{
while (isspace((unsigned int)(*d))) d++;
b = buffer;
while ((!isspace((unsigned int)(*d))) && (*d != bankNameSep) && (*d != '\0')) *b++ = *d++;
*b = '\0';
if ((fmts[i++] = get_id(buffer)) == r_Lin_Stream_NUMBER)
{
RMInit::logOut << "r_Linear_Stream::get_bank_ids(): unknown format \""
<< buffer << '\"' << endl;
delete [] fmts;
return NULL;
}
while (isspace((unsigned int)(*d))) d++;
if (*d == bankNameSep) d++;
}
fmts[i] = r_Lin_Stream_NUMBER;
if (i != number)
{
RMInit::logOut << "r_Linear_Stream::get_bank_ids(): warning, inconsistent stream number "
<< i << ", should be " << number;
}
return fmts;
}
char *r_Linear_Stream::get_bank_string( const r_Lin_Stream_Format *ids )
{
unsigned int i, len;
char *result, *b;
len = 0;
for (i=0; ids[i] != r_Lin_Stream_NUMBER; i++)
{
len += strlen(get_format_string(ids[i])) + 1;
}
result = new char[len+1];
b = result;
for (i=0; ids[i] != r_Lin_Stream_NUMBER; i++)
{
const char *str = get_format_string(ids[i]);
strcpy(b, str);
b += strlen(str);
*b++ = bankNameSep;
}
// remove trailing name separator
if (b != result) b--;
*b = '\0';
return result;
}
int r_Linear_Stream::set_num_symbols( unsigned int syms )
{
// by default this is ignored by most streams (except for arithmetic coding)
return 0;
}
// this method should basically only be called when COMPBENCHMARK is defined
void r_Linear_Stream::instantiate_timer( const char *func, int level )
{
#ifdef COMPBENCHMARK
if (myTimer != NULL)
delete myTimer;
myTimer = new RMTimer(get_name(), func, level);
myTimer->start();
#endif
}
std::ostream &operator<<( std::ostream &s, r_Linear_Stream::r_Lin_Stream_Format fmt )
{
s << r_Linear_Stream::get_format_string(fmt);
return s;
}
/*
* Linear compression storage
*/
r_Lin_Comp_Store::r_Lin_Comp_Store( r_ULong bsize )
{
mblocks = NULL;
streamer = NULL;
cacheSize = bsize;
cacheOff = 0;
cache = NULL;
}
r_Lin_Comp_Store::r_Lin_Comp_Store( r_Lin_Comp_Stream *str, r_ULong csize )
{
mblocks = NULL;
streamer = str;
cacheSize = csize;
cacheOff = 0;
cache = NULL;
}
r_Lin_Comp_Store::~r_Lin_Comp_Store( void )
{
// although cache is used in both modes, it's merely a pointer to a block within
// mblocks in static mode and mustn't be freed directly in that case.
if (mblocks != NULL)
{
delete mblocks;
}
else if (cache != NULL)
{
delete [] ((char*)cache);
}
}
int r_Lin_Comp_Store::start( r_Bytes typeSize )
{
if (streamer != NULL)
{
RMDBGONCE( 3, RMDebug::module_compression, "r_Lin_Comp_Store", "start() stream " << streamer->get_name() );
streamer->begin(typeSize);
cache = new char[cacheSize];
}
else
{
RMDBGONCE( 3, RMDebug::module_compression, "r_Lin_Comp_Store", "start() static, bsize " << cacheSize );
mblocks = new r_Memory_Block_Vector(cacheSize);
cache = mblocks->add();
}
if (cache == NULL)
return -1;
return 0;
}
void *r_Lin_Comp_Store::frag_ptr( void )
{
return (void*)(((char*)cache) + cacheOff);
}
r_ULong r_Lin_Comp_Store::frag_size( void ) const
{
return cacheSize - cacheOff;
}
int r_Lin_Comp_Store::frag_stored( r_ULong size )
{
if (size != 0)
{
cacheOff += size;
if (cacheOff >= cacheSize)
{
if (streamer == NULL)
{
cache = mblocks->add();
}
else
{
if (streamer->put(cache, cacheOff) < 0)
return -1;
}
cacheOff = 0;
}
}
return 0;
}
int r_Lin_Comp_Store::put( const void *data, r_ULong size )
{
const char *dptr = (const char*)data;
r_ULong left = size;
// note: cache in static mode!
while (left != 0)
{
r_ULong rest;
rest = cacheSize - cacheOff;
#ifdef RMANDEBUG
if (rest == 0)
{
RMInit::logOut << "r_Lin_Comp_Store::put(): fatal error!" << endl;
throw r_Error (COMPRESSIONFAILED);
}
#endif
if (rest > left)
rest = left;
if (rest != 0)
{
memcpy(((char*)cache) + cacheOff, dptr, rest);
if (frag_stored(rest) < 0)
return -1;
left -= rest;
dptr += rest;
}
}
return 0;
}
int r_Lin_Comp_Store::put( unsigned char val )
{
((unsigned char*)cache)[cacheOff] = val;
return frag_stored(1);
}
int r_Lin_Comp_Store::flush( void )
{
if (streamer != NULL)
{
if (cacheOff != 0)
{
if (streamer->put(cache, cacheOff) < 0)
return -1;
cacheOff = 0;
}
}
return 0;
}
r_ULong r_Lin_Comp_Store::stop( void )
{
RMDBGONCE( 3, RMDebug::module_compression, "r_Lin_Comp_Store", "stop()" );
if (streamer == NULL)
{
return mblocks->get_size(cacheOff);
}
else
{
if (flush() < 0)
return 0;
return streamer->end();
}
}
void r_Lin_Comp_Store::copy_data( void *dest )
{
if (streamer == NULL)
{
mblocks->copy_data(dest, cacheOff);
}
else
{
streamer->copy_data(dest);
}
}
void r_Lin_Comp_Store::print_status( std::ostream &str ) const
{
if (streamer == NULL)
{
str << "[static; " << mblocks->get_number() << " blocks, " << mblocks->get_size(cacheOff) << " bytes]";
}
else
{
str << "[stream: " << streamer->get_name() << ']';
}
}
std::ostream &operator<<( std::ostream &str, const r_Lin_Comp_Store &store )
{
store.print_status(str);
return str;
}
/*
* Linear compression encoders...
*/
r_Lin_Comp_Stream::r_Lin_Comp_Stream( const r_Lin_Comp_Stream &src ) : r_Linear_Stream()
{
target = NULL; // anything else would have bad side-effects...
streamer = src.streamer;
blockSize = src.blockSize;
autoDeleteStreams = 0;
}
r_Lin_Comp_Stream::r_Lin_Comp_Stream( void ) : r_Linear_Stream()
{
target = NULL;
streamer = NULL;
blockSize = 4096;
autoDeleteStreams = 0;
}
r_Lin_Comp_Stream::~r_Lin_Comp_Stream( void )
{
if (target != NULL)
delete target;
if (autoDeleteStreams != 0)
free_streams();
}
void r_Lin_Comp_Stream::set_block_size( r_ULong bsize )
{
blockSize = bsize;
}
void r_Lin_Comp_Stream::set_stream( r_Lin_Comp_Stream *str )
{
streamer = str;
}
r_Lin_Comp_Stream *r_Lin_Comp_Stream::get_stream( void )
{
return streamer;
}
const r_Lin_Comp_Stream *r_Lin_Comp_Stream::get_stream( void ) const
{
return streamer;
}
void r_Lin_Comp_Stream::print_bank_name( std::ostream &str ) const
{
const r_Lin_Comp_Stream *s = streamer;
str << get_name();
while (s != NULL)
{
str << " : " << s->get_name();
s = s->get_stream();
}
}
void r_Lin_Comp_Stream::free_streams( void )
{
r_Lin_Comp_Stream *s = streamer;
while (s != NULL)
{
r_Lin_Comp_Stream *next = s->get_stream();
delete s;
s = next;
}
}
const char *r_Lin_Comp_Stream::get_stream_file( void ) const
{
const r_Lin_Comp_Stream *s = this;
while (s != NULL)
{
if (s->get_format() == r_Lin_Stream_File)
return ((r_File_Comp_Stream*)s)->get_file_name();
s = s->streamer;
}
return NULL;
}
int r_Lin_Comp_Stream::set_stream_file( const char *name )
{
r_Lin_Comp_Stream *s = this;
while (s != NULL)
{
if (s->get_format() == r_Lin_Stream_File)
{
((r_File_Comp_Stream*)s)->set_file_name(name);
return 1;
}
s = s->streamer;
}
return 0;
}
void r_Lin_Comp_Stream::set_storage_handler( const r_Storage_Man &newStore )
{
mystore = newStore;
}
void r_Lin_Comp_Stream::set_params( const char *str )
{
if (params != NULL)
params->process(str);
}
void r_Lin_Comp_Stream::init_target( void )
{
if (target != NULL)
delete target;
if (streamer == NULL)
target = new r_Lin_Comp_Store(blockSize);
else
target = new r_Lin_Comp_Store(streamer);
}
void r_Lin_Comp_Stream::exit_target( void )
{
if (target != NULL)
{
delete target;
target = NULL;
}
}
// static member function for creation
r_Lin_Comp_Stream* r_Lin_Comp_Stream::create( r_Lin_Stream_Format fmt, const char *pstr ) throw(r_Error)
{
switch (fmt)
{
case r_Lin_Stream_None:
return new r_No_Comp_Stream();
case r_Lin_Stream_RLE:
return new r_RLE_Comp_Stream();
case r_Lin_Stream_ZLib:
return new r_ZLib_Comp_Stream( pstr );
case r_Lin_Stream_Arith:
return new r_Arith_Comp_Stream();
case r_Lin_Stream_File:
return new r_File_Comp_Stream( pstr);
default:
RMInit::logOut << "Unknown linear compression format " << fmt << endl;
r_Error err(r_Error::r_Error_General);
throw(err);
}
return NULL;
}
// allocate a stream bank
r_Lin_Comp_Stream* r_Lin_Comp_Stream::create( r_Lin_Stream_Format *ids, const char *pstr ) throw(r_Error)
{
r_Lin_Comp_Stream *result = NULL;
unsigned int i;
for (i=0; ids[i] != r_Lin_Stream_NUMBER; i++) ;
try
{
for (; i!=0; i--)
{
r_Lin_Comp_Stream *str = create(ids[i-1], pstr);
str->set_stream(result);
result = str;
}
}
catch( r_Error &err )
{
if (result != NULL)
{
result->free_streams();
delete result;
}
throw(err);
}
// mark the root stream as a stream bank owning all child streams
if (result != NULL)
result->autoDeleteStreams = 1;
return result;
}
std::ostream &operator<<( std::ostream &str, const r_Lin_Comp_Stream &comp )
{
comp.print_bank_name(str);
return str;
}
/*
* Linear decompression storage
*/
r_Lin_Decomp_Store::r_Lin_Decomp_Store( void )
{
RMDBGONCE( 3, RMDebug::module_compression, "r_Lin_Decomp_Store", "r_Lin_Decomp_Store()" );
streamer = NULL;
cache = NULL;
}
r_Lin_Decomp_Store::r_Lin_Decomp_Store( r_Lin_Decomp_Stream *str, r_ULong csize )
{
RMDBGONCE( 3, RMDebug::module_compression, "r_Lin_Decomp_Store", "r_Lin_Decomp_Store(" << str->get_name() << ')' );
streamer = str;
cacheSize = csize;
cache = new char[cacheSize];
}
r_Lin_Decomp_Store::~r_Lin_Decomp_Store( void )
{
if (cache != NULL)
delete [] (char*)cache;
}
int r_Lin_Decomp_Store::start( r_Bytes typeSize, const void *data, r_ULong size )
{
if (streamer == NULL)
{
RMDBGONCE( 4, RMDebug::module_compression, "r_Lin_Decomp_Store", "start() static" );
dataBase = data;
dataSize = size;
dataOffset = 0;
}
else
{
RMDBGONCE( 4, RMDebug::module_compression, "r_Lin_Decomp_Store", "start() stream " << streamer->get_name() );
cacheLevel = 0;
cacheOff = 0;
return streamer->begin(typeSize, data, size);
}
return 0;
}
int r_Lin_Decomp_Store::ensure_data( void )
{
// in static mode, we don't have to do anything.
if ((streamer != NULL) && (cacheOff >= cacheLevel))
{
// returns 0 for OK or the _negative_ number of unread bytes
int status = streamer->get(cache, cacheSize);
if (status < 0)
{
cacheLevel = (r_ULong)((r_Long)cacheSize + (r_Long)status);
if (cacheLevel == 0)
return -1;
}
else
{
// total refill of cache possible
cacheLevel = cacheSize;
}
cacheOff = 0;
//cout << "CACHE LEVEL " << cacheLevel << endl;
return 1;
}
return 0;
}
int r_Lin_Decomp_Store::get( void *data, r_ULong size )
{
r_ULong rest;
if (streamer == NULL)
{
rest = dataSize - dataOffset;
if (rest > size)
rest = size;
memcpy(data, ((const char*)dataBase) + dataOffset, rest);
dataOffset += rest;
if (rest < size)
return -((int)(size - rest));
}
else
{
r_ULong left = size;
char *dptr = (char*)data;
while (left != 0)
{
if (ensure_data() < 0)
return -((int)left);
rest = cacheLevel - cacheOff;
if (rest > left)
rest = left;
memcpy(dptr, ((char*)cache) + cacheOff, rest);
dptr += rest;
cacheOff += rest;
left -= rest;
}
}
return 0;
}
int r_Lin_Decomp_Store::get( unsigned char &val )
{
if (streamer == NULL)
{
if (dataOffset >= dataSize)
return -1;
val = ((unsigned char*)dataBase)[dataOffset++];
}
else
{
if (ensure_data() < 0)
return -1;
val = ((unsigned char*)cache)[cacheOff++];
}
return 0;
}
int r_Lin_Decomp_Store::stop( void )
{
RMDBGONCE( 4, RMDebug::module_compression, "r_Lin_Decomp_Store", "stop()" );
if (streamer != NULL)
{
return streamer->end();
}
return 0;
}
const void *r_Lin_Decomp_Store::frag_ptr( void ) const
{
if (streamer == NULL)
return (const void*)(((char*)dataBase)+dataOffset);
return (const void*)(((char*)cache)+cacheOff);
}
r_ULong r_Lin_Decomp_Store::frag_size( void ) const
{
if (streamer == NULL)
return dataSize - dataOffset;
return cacheLevel - cacheOff;
}
int r_Lin_Decomp_Store::frag_read( r_ULong size )
{
if (streamer == NULL)
dataOffset += size;
else
cacheOff += size;
return 0;
}
void r_Lin_Decomp_Store::print_status( std::ostream &str ) const
{
if (streamer == NULL)
{
str << "[static: current " << dataOffset << ", total " << dataSize << ']';
}
else
{
str << "[stream: " << streamer->get_name() << ']';
}
}
std::ostream &operator<<( std::ostream &str, const r_Lin_Decomp_Store &store )
{
store.print_status(str);
return str;
}
/*
* linear compression decoders
*/
r_Lin_Decomp_Stream::r_Lin_Decomp_Stream( void )
{
params = NULL;
source = NULL;
streamer = NULL;
autoDeleteStreams = 0;
}
r_Lin_Decomp_Stream::~r_Lin_Decomp_Stream( void )
{
if (source != NULL)
delete source;
if (autoDeleteStreams != 0)
free_streams();
}
void r_Lin_Decomp_Stream::set_params( const char *str )
{
if (params != NULL)
params->process(str);
}
void r_Lin_Decomp_Stream::set_stream( r_Lin_Decomp_Stream *str )
{
streamer = str;
}
r_Lin_Decomp_Stream *r_Lin_Decomp_Stream::get_stream( void )
{
return streamer;
}
const r_Lin_Decomp_Stream *r_Lin_Decomp_Stream::get_stream( void ) const
{
return streamer;
}
void r_Lin_Decomp_Stream::print_bank_name( std::ostream &str ) const
{
const r_Lin_Decomp_Stream *s = streamer;
str << get_name();
while (s != NULL)
{
str << " : " << s->get_name();
s = s->get_stream();
}
}
void r_Lin_Decomp_Stream::free_streams( void )
{
r_Lin_Decomp_Stream *s = streamer;
while (s != NULL)
{
r_Lin_Decomp_Stream *next = s->get_stream();
delete s;
s = next;
}
}
const char *r_Lin_Decomp_Stream::get_stream_file( void ) const
{
const r_Lin_Decomp_Stream *s = this;
while (s != NULL)
{
if (s->get_format() == r_Lin_Stream_File)
return ((r_File_Decomp_Stream*)s)->get_file_name();
s = s->streamer;
}
return NULL;
}
int r_Lin_Decomp_Stream::set_stream_file( const char *name )
{
r_Lin_Decomp_Stream *s = this;
while (s != NULL)
{
if (s->get_format() == r_Lin_Stream_File)
{
((r_File_Decomp_Stream*)s)->set_file_name(name);
return 1;
}
s = s->streamer;
}
return 0;
}
void r_Lin_Decomp_Stream::init_source( void )
{
if (source != NULL)
delete source;
if (streamer == NULL)
source = new r_Lin_Decomp_Store();
else
source = new r_Lin_Decomp_Store(streamer);
}
void r_Lin_Decomp_Stream::exit_source( void )
{
delete source;
source = NULL;
}
// static member function for creation
r_Lin_Decomp_Stream* r_Lin_Decomp_Stream::create( r_Lin_Stream_Format fmt, const char *pstr ) throw(r_Error)
{
switch (fmt)
{
case r_Lin_Stream_None:
return new r_No_Decomp_Stream();
case r_Lin_Stream_RLE:
return new r_RLE_Decomp_Stream();
case r_Lin_Stream_ZLib:
return new r_ZLib_Decomp_Stream();
case r_Lin_Stream_Arith:
return new r_Arith_Decomp_Stream();
case r_Lin_Stream_File:
return new r_File_Decomp_Stream( pstr );
default:
RMInit::logOut << "Unknown linear compression format " << fmt << endl;
r_Error err(r_Error::r_Error_General);
throw(err);
}
return NULL;
}
// allocate a stream bank
r_Lin_Decomp_Stream* r_Lin_Decomp_Stream::create( r_Lin_Stream_Format *ids, const char *pstr ) throw(r_Error)
{
r_Lin_Decomp_Stream *result = NULL;
unsigned int i;
for (i=0; ids[i] != r_Lin_Stream_NUMBER; i++) ;
try
{
for (; i!=0; i--)
{
r_Lin_Decomp_Stream *str = create(ids[i-1], pstr);
str->set_stream(result);
result = str;
}
}
catch( r_Error &err )
{
if (result != NULL)
{
result->free_streams();
delete result;
}
throw(err);
}
// mark the root stream as a stream bank owning all the child streams
if (result != NULL)
result->autoDeleteStreams = 1;
return result;
}
std::ostream &operator<<( std::ostream &str, const r_Lin_Decomp_Stream &decomp )
{
decomp.print_bank_name(str);
return str;
}
/*
* The container object for matching compression / decompression streams
*/
r_Lin_Codec_Stream::r_Lin_Codec_Stream( void ) : comp(NULL), decomp(NULL)
{
}
r_Lin_Codec_Stream::r_Lin_Codec_Stream( r_Linear_Stream::r_Lin_Stream_Format fmt, const char *pstr )
{
comp = r_Lin_Comp_Stream::create(fmt, pstr);
decomp = r_Lin_Decomp_Stream::create(fmt, pstr);
}
r_Lin_Codec_Stream::r_Lin_Codec_Stream( r_Linear_Stream::r_Lin_Stream_Format *ids, const char *pstr )
{
comp = r_Lin_Comp_Stream::create(ids, pstr);
decomp = r_Lin_Decomp_Stream::create(ids, pstr);
}
r_Lin_Codec_Stream::r_Lin_Codec_Stream( const r_Lin_Codec_Stream &lc )
{
comp = lc.comp->clone();
decomp = lc.decomp->clone();
}
r_Lin_Codec_Stream::~r_Lin_Codec_Stream( void )
{
if (comp != NULL)
delete comp;
if (decomp != NULL)
delete decomp;
}