From 537280f183d3a35e8dee8a20238c71e7440cb28c Mon Sep 17 00:00:00 2001 From: Yonit Halperin Date: Sun, 20 Jun 2010 17:18:56 +0300 Subject: Lossy compression of RGBA images (on WAN connection) The RGB channels are compressed using JPEG. The alpha channel is compressed using LZ. --- common/canvas_base.c | 92 +++++++++++++++++++++++++++++++- common/lz.c | 12 +++++ common/lz_common.h | 9 ++-- server/red_worker.c | 144 +++++++++++++++++++++++++++++++++++++++++---------- spice.proto | 14 +++++ 5 files changed, 238 insertions(+), 33 deletions(-) diff --git a/common/canvas_base.c b/common/canvas_base.c index 0148270f..2b00f09e 100644 --- a/common/canvas_base.c +++ b/common/canvas_base.c @@ -65,6 +65,10 @@ #define ROUND(_x) ((int)floor((_x) + 0.5)) +#define IS_IMAGE_LOSSY(descriptor) \ + (((descriptor)->type == SPICE_IMAGE_TYPE_JPEG) || \ + ((descriptor)->type == SPICE_IMAGE_TYPE_JPEG_ALPHA)) + #ifdef WIN32 typedef struct __declspec (align(1)) LZImage { #else @@ -607,6 +611,85 @@ static pixman_image_t *canvas_get_jpeg(CanvasBase *canvas, SpiceJPEGImage *image return surface; } +static pixman_image_t *canvas_get_jpeg_alpha(CanvasBase *canvas, + SpiceJPEGAlphaImage *image, int invers) +{ + pixman_image_t *surface = NULL; + int stride; + int width; + int height; + uint8_t *dest; + int alpha_top_down = FALSE; + LzData *lz_data = &canvas->lz_data; + LzImageType lz_alpha_type; + uint8_t *comp_alpha_buf = NULL; + uint8_t *decomp_alpha_buf = NULL; + int alpha_size; + int lz_alpha_width, lz_alpha_height, n_comp_pixels, lz_alpha_top_down; + + canvas->jpeg->ops->begin_decode(canvas->jpeg, image->jpeg_alpha.data, + image->jpeg_alpha.jpeg_size, + &width, &height); + ASSERT((uint32_t)width == image->descriptor.width); + ASSERT((uint32_t)height == image->descriptor.height); + + if (image->jpeg_alpha.flags & SPICE_JPEG_ALPHA_FLAGS_TOP_DOWN) { + alpha_top_down = TRUE; + } + +#ifdef WIN32 + lz_data->decode_data.dc = canvas->dc; +#endif + surface = alloc_lz_image_surface(&lz_data->decode_data, PIXMAN_a8r8g8b8, + width, height, width*height, alpha_top_down); + + if (surface == NULL) { + CANVAS_ERROR("create surface failed"); + } + + dest = (uint8_t *)pixman_image_get_data(surface); + stride = pixman_image_get_stride(surface); + + canvas->jpeg->ops->decode(canvas->jpeg, dest, stride, SPICE_BITMAP_FMT_32BIT); + + comp_alpha_buf = image->jpeg_alpha.data + image->jpeg_alpha.jpeg_size; + alpha_size = image->jpeg_alpha.data_size - image->jpeg_alpha.jpeg_size; + + lz_decode_begin(lz_data->lz, comp_alpha_buf, alpha_size, &lz_alpha_type, + &lz_alpha_width, &lz_alpha_height, &n_comp_pixels, + &lz_alpha_top_down, NULL); + ASSERT(lz_alpha_type == LZ_IMAGE_TYPE_XXXA); + ASSERT(lz_alpha_top_down == alpha_top_down); + ASSERT(lz_alpha_width == width); + ASSERT(lz_alpha_height == height); + ASSERT(n_comp_pixels == width * height); + + if (!alpha_top_down) { + decomp_alpha_buf = dest + stride * (height - 1); + } else { + decomp_alpha_buf = dest; + } + lz_decode(lz_data->lz, LZ_IMAGE_TYPE_XXXA, decomp_alpha_buf); + + if (invers) { + uint8_t *end = dest + height * stride; + for (; dest != end; dest += stride) { + uint32_t *pix; + uint32_t *end_pix; + + pix = (uint32_t *)dest; + end_pix = pix + width; + for (; pix < end_pix; pix++) { + *pix ^= 0x00ffffff; + } + } + } +#ifdef DUMP_JPEG + dump_jpeg(image->jpeg_alpha.data, image->jpeg_alpha.jpeg_size); +#endif + return surface; +} + static pixman_image_t *canvas_bitmap_to_surface(CanvasBase *canvas, SpiceBitmap* bitmap, SpicePalette *palette, int want_original) { @@ -1088,6 +1171,11 @@ static pixman_image_t *canvas_get_image_internal(CanvasBase *canvas, SPICE_ADDRE surface = canvas_get_jpeg(canvas, image, 0); break; } + case SPICE_IMAGE_TYPE_JPEG_ALPHA: { + SpiceJPEGAlphaImage *image = (SpiceJPEGAlphaImage *)descriptor; + surface = canvas_get_jpeg_alpha(canvas, image, 0); + break; + } #if defined(SW_CANVAS_CACHE) case SPICE_IMAGE_TYPE_GLZ_RGB: { LZImage *image = (LZImage *)descriptor; @@ -1138,7 +1226,7 @@ static pixman_image_t *canvas_get_image_internal(CanvasBase *canvas, SPICE_ADDRE #endif descriptor->type != SPICE_IMAGE_TYPE_FROM_CACHE ) { #ifdef SW_CANVAS_CACHE - if (descriptor->type != SPICE_IMAGE_TYPE_JPEG) { + if (!IS_IMAGE_LOSSY(descriptor)) { canvas->bits_cache->ops->put(canvas->bits_cache, descriptor->id, surface); } else { canvas->bits_cache->ops->put_lossy(canvas->bits_cache, descriptor->id, surface); @@ -1151,7 +1239,7 @@ static pixman_image_t *canvas_get_image_internal(CanvasBase *canvas, SPICE_ADDRE #endif #ifdef SW_CANVAS_CACHE } else if (descriptor->flags & SPICE_IMAGE_FLAGS_CACHE_REPLACE_ME) { - if (descriptor->type == SPICE_IMAGE_TYPE_JPEG) { + if (IS_IMAGE_LOSSY(descriptor)) { CANVAS_ERROR("invalid cache replace request: the image is lossy"); } canvas->bits_cache->ops->replace_lossy(canvas->bits_cache, descriptor->id, surface); diff --git a/common/lz.c b/common/lz.c index 73a797c7..e5635849 100644 --- a/common/lz.c +++ b/common/lz.c @@ -565,6 +565,9 @@ int lz_encode(LzContext *lz, LzImageType type, int width, int height, int top_do lz_rgb32_compress(encoder); lz_rgb_alpha_compress(encoder); break; + case LZ_IMAGE_TYPE_XXXA: + lz_rgb_alpha_compress(encoder); + break; case LZ_IMAGE_TYPE_INVALID: default: encoder->usr->error(encoder->usr, "bad image type\n"); @@ -661,6 +664,7 @@ void lz_decode(LzContext *lz, LzImageType to_type, uint8_t *buf) case LZ_IMAGE_TYPE_RGB24: case LZ_IMAGE_TYPE_RGB32: case LZ_IMAGE_TYPE_RGBA: + case LZ_IMAGE_TYPE_XXXA: case LZ_IMAGE_TYPE_INVALID: default: encoder->usr->error(encoder->usr, "bad image type\n"); @@ -705,6 +709,14 @@ void lz_decode(LzContext *lz, LzImageType to_type, uint8_t *buf) encoder->usr->error(encoder->usr, "unsupported output format\n"); } break; + case LZ_IMAGE_TYPE_XXXA: + if (encoder->type == to_type) { + alpha_size = lz_rgb_alpha_decompress(encoder, (rgb32_pixel_t *)buf, size); + out_size = alpha_size; + } else { + encoder->usr->error(encoder->usr, "unsupported output format\n"); + } + break; case LZ_IMAGE_TYPE_PLT1_LE: case LZ_IMAGE_TYPE_PLT1_BE: case LZ_IMAGE_TYPE_PLT4_LE: diff --git a/common/lz_common.h b/common/lz_common.h index d5019fd3..34276afb 100644 --- a/common/lz_common.h +++ b/common/lz_common.h @@ -39,17 +39,18 @@ typedef enum { LZ_IMAGE_TYPE_RGB16, LZ_IMAGE_TYPE_RGB24, LZ_IMAGE_TYPE_RGB32, - LZ_IMAGE_TYPE_RGBA + LZ_IMAGE_TYPE_RGBA, + LZ_IMAGE_TYPE_XXXA } LzImageType; #define LZ_IMAGE_TYPE_MASK 0x0f #define LZ_IMAGE_TYPE_LOG 4 // number of bits required for coding the image type /* access to the arrays is based on the image types */ -static const int IS_IMAGE_TYPE_PLT[] = {0, 1, 1, 1, 1, 1, 0, 0, 0, 0}; -static const int IS_IMAGE_TYPE_RGB[] = {0, 0, 0, 0, 0, 0, 1, 1, 1, 1}; +static const int IS_IMAGE_TYPE_PLT[] = {0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0}; +static const int IS_IMAGE_TYPE_RGB[] = {0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1}; static const int PLT_PIXELS_PER_BYTE[] = {0, 8, 8, 2, 2, 1}; -static const int RGB_BYTES_PER_PIXEL[] = {0, 1, 1, 1, 1, 1, 2, 3, 4, 4}; +static const int RGB_BYTES_PER_PIXEL[] = {0, 1, 1, 1, 1, 1, 2, 3, 4, 4, 4}; #define LZ_MAGIC (*(uint32_t *)"LZ ") diff --git a/server/red_worker.c b/server/red_worker.c index ff78483c..4ee0c616 100644 --- a/server/red_worker.c +++ b/server/red_worker.c @@ -177,6 +177,7 @@ static const char *glz_stat_name = "glz"; static const char *quic_stat_name = "quic"; static const char *jpeg_stat_name = "jpeg"; static const char *zlib_stat_name = "zlib_glz"; +static const char *jpeg_alpha_stat_name = "jpeg_alpha"; static inline void stat_compress_init(stat_info_t *info, const char *name) { @@ -474,6 +475,7 @@ typedef struct __attribute__ ((__packed__)) RedImage { SpiceSurface surface; SpiceJPEGData jpeg; SpiceZlibGlzRGBData zlib_glz; + SpiceJPEGAlphaData jpeg_alpha; }; } RedImage; @@ -700,6 +702,7 @@ struct DisplayChannel { stat_info_t quic_stat; stat_info_t jpeg_stat; stat_info_t zlib_glz_stat; + stat_info_t jpeg_alpha_stat; #endif }; @@ -1090,24 +1093,34 @@ static void print_compress_stats(DisplayChannel *display_channel) stat_byte_to_mega(display_channel->jpeg_stat.comp_size), stat_cpu_time_to_sec(display_channel->jpeg_stat.total) ); + red_printf("JPEG-RGBA\t%8d\t%13.2f\t%12.2f\t%12.2f", + display_channel->jpeg_alpha_stat.count, + stat_byte_to_mega(display_channel->jpeg_alpha_stat.orig_size), + stat_byte_to_mega(display_channel->jpeg_alpha_stat.comp_size), + stat_cpu_time_to_sec(display_channel->jpeg_alpha_stat.total) + ); red_printf("-------------------------------------------------------------------"); red_printf("Total \t%8d\t%13.2f\t%12.2f\t%12.2f", display_channel->lz_stat.count + display_channel->glz_stat.count + display_channel->quic_stat.count + - display_channel->jpeg_stat.count, + display_channel->jpeg_stat.count + + display_channel->jpeg_alpha_stat.count, stat_byte_to_mega(display_channel->lz_stat.orig_size + display_channel->glz_stat.orig_size + display_channel->quic_stat.orig_size + - display_channel->jpeg_stat.orig_size), + display_channel->jpeg_stat.orig_size + + display_channel->jpeg_alpha_stat.orig_size), stat_byte_to_mega(display_channel->lz_stat.comp_size + glz_enc_size + display_channel->quic_stat.comp_size + - display_channel->jpeg_stat.comp_size), + display_channel->jpeg_stat.comp_size + + display_channel->jpeg_alpha_stat.comp_size), stat_cpu_time_to_sec(display_channel->lz_stat.total + display_channel->glz_stat.total + display_channel->zlib_glz_stat.total + display_channel->quic_stat.total + - display_channel->jpeg_stat.total) + display_channel->jpeg_stat.total + + display_channel->jpeg_alpha_stat.total) ); } @@ -6326,7 +6339,7 @@ static inline int red_glz_compress_image(DisplayChannel *display_channel, stat_compress_add(&display_channel->glz_stat, start_time, src->stride * src->y, glz_size); - if (!display_channel->enable_zlib_glz_wrap || (glz_size < MIN_GLZ_SIZE_FOR_ZLIB)) { + if (!display_channel->enable_zlib_glz_wrap || (glz_size < MIN_GLZ_SIZE_FOR_ZLIB)) { goto glz; } #ifdef COMPRESS_STAT @@ -6471,22 +6484,34 @@ static int red_jpeg_compress_image(DisplayChannel *display_channel, RedImage *de { RedWorker *worker = display_channel->base.worker; JpegData *jpeg_data = &worker->jpeg_data; + LzData *lz_data = &worker->lz_data; JpegEncoderContext *jpeg = worker->jpeg; + LzContext *lz = worker->lz; JpegEncoderImageType jpeg_in_type; - int size; + int jpeg_size = 0; + int has_alpha = FALSE; + int alpha_lz_size = 0; + int comp_head_filled; + int comp_head_left; + uint8_t *lz_out_start_byte; + #ifdef COMPRESS_STAT stat_time_t start_time = stat_now(); #endif switch (src->format) { - case SPICE_BITMAP_FMT_32BIT: - jpeg_in_type = JPEG_IMAGE_TYPE_BGRX32; - break; case SPICE_BITMAP_FMT_16BIT: jpeg_in_type = JPEG_IMAGE_TYPE_RGB16; break; case SPICE_BITMAP_FMT_24BIT: jpeg_in_type = JPEG_IMAGE_TYPE_BGR24; break; + case SPICE_BITMAP_FMT_32BIT: + jpeg_in_type = JPEG_IMAGE_TYPE_BGRX32; + break; + case SPICE_BITMAP_FMT_RGBA: + jpeg_in_type = JPEG_IMAGE_TYPE_BGRX32; + has_alpha = TRUE; + break; default: return FALSE; } @@ -6525,6 +6550,7 @@ static int red_jpeg_compress_image(DisplayChannel *display_channel, RedImage *de } if ((src->flags & QXL_BITMAP_UNSTABLE)) { + ASSERT(!has_alpha); jpeg_data->data.u.unstable_lines_data.next = data; jpeg_data->data.u.unstable_lines_data.src_stride = stride; jpeg_data->data.u.unstable_lines_data.dest_stride = src->stride; @@ -6540,14 +6566,16 @@ static int red_jpeg_compress_image(DisplayChannel *display_channel, RedImage *de sizeof(jpeg_data->data.u.unstable_lines_data.input_bufs[0]->buf) / jpeg_data->data.u.unstable_lines_data.dest_stride; jpeg_data->usr.more_lines = jpeg_usr_more_lines_unstable; - size = jpeg_encode(jpeg, display_channel->jpeg_quality, jpeg_in_type, src->x, src->y, - NULL, 0, src->stride, (uint8_t*)jpeg_data->data.bufs_head->buf, - sizeof(jpeg_data->data.bufs_head->buf)); + jpeg_size = jpeg_encode(jpeg, display_channel->jpeg_quality, jpeg_in_type, + src->x, src->y, NULL, 0, src->stride, + (uint8_t*)jpeg_data->data.bufs_head->buf, + sizeof(jpeg_data->data.bufs_head->buf)); } else { jpeg_data->usr.more_lines = jpeg_usr_no_more_lines; - size = jpeg_encode(jpeg, display_channel->jpeg_quality, jpeg_in_type, src->x, src->y, - data, src->y, stride, (uint8_t*)jpeg_data->data.bufs_head->buf, - sizeof(jpeg_data->data.bufs_head->buf)); + jpeg_size = jpeg_encode(jpeg, display_channel->jpeg_quality, jpeg_in_type, + src->x, src->y, data, src->y, stride, + (uint8_t*)jpeg_data->data.bufs_head->buf, + sizeof(jpeg_data->data.bufs_head->buf)); } } else { int stride; @@ -6584,24 +6612,83 @@ static int red_jpeg_compress_image(DisplayChannel *display_channel, RedImage *de jpeg_data->usr.more_lines = jpeg_usr_more_lines_reverse; stride = -src->stride; } - size = jpeg_encode(jpeg, display_channel->jpeg_quality, jpeg_in_type, src->x, src->y, NULL, - 0, stride, (uint8_t*)jpeg_data->data.bufs_head->buf, - sizeof(jpeg_data->data.bufs_head->buf)); + jpeg_size = jpeg_encode(jpeg, display_channel->jpeg_quality, jpeg_in_type, src->x, src->y, NULL, + 0, stride, (uint8_t*)jpeg_data->data.bufs_head->buf, + sizeof(jpeg_data->data.bufs_head->buf)); } // the compressed buffer is bigger than the original data - if (size > (src->y * src->stride)) { + if (jpeg_size > (src->y * src->stride)) { + longjmp(jpeg_data->data.jmp_env, 1); + } + + if (!has_alpha) { + dest->descriptor.type = SPICE_IMAGE_TYPE_JPEG; + dest->jpeg.data_size = jpeg_size; + + o_comp_data->comp_buf = jpeg_data->data.bufs_head; + o_comp_data->comp_buf_size = jpeg_size; + o_comp_data->is_lossy = TRUE; + + stat_compress_add(&display_channel->jpeg_stat, start_time, src->stride * src->y, + o_comp_data->comp_buf_size); + return TRUE; + } + + lz_data->data.bufs_head = jpeg_data->data.bufs_tail; + lz_data->data.bufs_tail = lz_data->data.bufs_head; + + comp_head_filled = jpeg_size % sizeof(lz_data->data.bufs_head->buf); + comp_head_left = sizeof(lz_data->data.bufs_head->buf) - comp_head_filled; + lz_out_start_byte = ((uint8_t *)lz_data->data.bufs_head->buf) + comp_head_filled; + + lz_data->data.display_channel = display_channel; + + if ((src->flags & QXL_BITMAP_DIRECT)) { + lz_data->usr.more_lines = lz_usr_no_more_lines; + alpha_lz_size = lz_encode(lz, LZ_IMAGE_TYPE_XXXA, src->x, src->y, + (src->flags & QXL_BITMAP_TOP_DOWN), + (uint8_t*)get_virt(&worker->mem_slots, src->data, + src->stride * src->y, group_id), + src->y, src->stride, + lz_out_start_byte, + comp_head_left); + } else { + lz_data->data.u.lines_data.enc_get_virt = cb_get_virt; + lz_data->data.u.lines_data.enc_get_virt_opaque = &worker->mem_slots; + lz_data->data.u.lines_data.enc_validate_virt = cb_validate_virt; + lz_data->data.u.lines_data.enc_validate_virt_opaque = &worker->mem_slots; + lz_data->data.u.lines_data.stride = src->stride; + lz_data->data.u.lines_data.next = src->data; + lz_data->data.u.lines_data.group_id = group_id; + lz_data->usr.more_lines = lz_usr_more_lines; + + alpha_lz_size = lz_encode(lz, LZ_IMAGE_TYPE_XXXA, src->x, src->y, + (src->flags & QXL_BITMAP_TOP_DOWN), + NULL, 0, src->stride, + lz_out_start_byte, + comp_head_left); + } + + + // the compressed buffer is bigger than the original data + if ((jpeg_size + alpha_lz_size) > (src->y * src->stride)) { longjmp(jpeg_data->data.jmp_env, 1); } - dest->descriptor.type = SPICE_IMAGE_TYPE_JPEG; - dest->jpeg.data_size = size; + dest->descriptor.type = SPICE_IMAGE_TYPE_JPEG_ALPHA; + dest->jpeg_alpha.flags = 0; + if (src->flags & QXL_BITMAP_TOP_DOWN) { + dest->jpeg_alpha.flags |= SPICE_JPEG_ALPHA_FLAGS_TOP_DOWN; + } + + dest->jpeg_alpha.jpeg_size = jpeg_size; + dest->jpeg_alpha.data_size = jpeg_size + alpha_lz_size; o_comp_data->comp_buf = jpeg_data->data.bufs_head; - o_comp_data->comp_buf_size = size; + o_comp_data->comp_buf_size = jpeg_size + alpha_lz_size; o_comp_data->is_lossy = TRUE; - - stat_compress_add(&display_channel->jpeg_stat, start_time, src->stride * src->y, + stat_compress_add(&display_channel->jpeg_alpha_stat, start_time, src->stride * src->y, o_comp_data->comp_buf_size); return TRUE; } @@ -6811,7 +6898,8 @@ static inline int red_compress_image(DisplayChannel *display_channel, if (can_lossy && display_channel->enable_jpeg && ((image_compression == SPICE_IMAGE_COMPRESS_AUTO_LZ) || (image_compression == SPICE_IMAGE_COMPRESS_AUTO_GLZ))) { - if (src->format != SPICE_BITMAP_FMT_RGBA) { + // if we use lz for alpha, the stride can't be extra + if (src->format != SPICE_BITMAP_FMT_RGBA || !_stride_is_extra(src)) { return red_jpeg_compress_image(display_channel, dest, src, o_comp_data, drawable->group_id); } @@ -9073,8 +9161,8 @@ static void red_send_image(DisplayChannel *display_channel, ImageItem *item) &bitmap, worker->mem_slots.internal_groupslot_id); if (grad_level == BITMAP_GRADUAL_HIGH) { - lossy_comp = display_channel->enable_jpeg && item->can_lossy && - (item->image_format != SPICE_BITMAP_FMT_RGBA); + // if we use lz for alpha, the stride can't be extra + lossy_comp = display_channel->enable_jpeg && item->can_lossy; } else { lz_comp = TRUE; } @@ -10593,6 +10681,7 @@ static void handle_new_display_channel(RedWorker *worker, RedsStreamContext *pee stat_compress_init(&display_channel->quic_stat, quic_stat_name); stat_compress_init(&display_channel->jpeg_stat, jpeg_stat_name); stat_compress_init(&display_channel->zlib_glz_stat, zlib_stat_name); + stat_compress_init(&display_channel->jpeg_alpha_stat, jpeg_alpha_stat_name); } static void red_disconnect_cursor(RedChannel *channel) @@ -11144,6 +11233,7 @@ static void handle_dev_input(EventListener *listener, uint32_t events) stat_reset(&worker->display_channel->glz_stat); stat_reset(&worker->display_channel->jpeg_stat); stat_reset(&worker->display_channel->zlib_glz_stat); + stat_reset(&worker->display_channel->jpeg_alpha_stat); } #endif break; diff --git a/spice.proto b/spice.proto index 84d3fa98..dbd3dde1 100644 --- a/spice.proto +++ b/spice.proto @@ -294,6 +294,7 @@ enum8 image_type { JPEG, FROM_CACHE_LOSSLESS, ZLIB_GLZ_RGB, + JPEG_ALPHA, }; flags8 image_flags { @@ -321,6 +322,10 @@ flags8 bitmap_flags { TOP_DOWN, }; +flags8 jpeg_alpha_flags { + TOP_DOWN, +}; + enum8 image_scale_mode { INTERPOLATE, NEAREST, @@ -477,6 +482,13 @@ struct ZlibGlzRGBData { uint8 data[data_size] @end @nomarshal; } @ctype(SpiceZlibGlzRGBData); +struct JPEGAlphaData { + jpeg_alpha_flags flags; + uint32 jpeg_size; + uint32 data_size; + uint8 data[data_size] @end @nomarshal; +} @ctype(SpiceJPEGAlphaData); + struct Surface { uint32 surface_id; }; @@ -500,6 +512,8 @@ struct Image { LZPLTData lzplt_data @ctype(SpiceLZPLTData); case ZLIB_GLZ_RGB: ZlibGlzRGBData zlib_glz_data @ctype(SpiceZlibGlzRGBData); + case JPEG_ALPHA: + JPEGAlphaData jpeg_alpha_data @ctype(SpiceJPEGAlphaData); case SURFACE: Surface surface_data; } u @end; -- cgit