server: add async io support

The new _ASYNC io's in qxl_dev listed at the end get six new api
functions, and an additional callback function "async_complete". When
the async version of a specific io is used, completion is notified by
calling async_complete, and no READY message is written or expected by
the dispatcher.

update_area has been changed to push QXLRects to the worker thread, where
the conversion to SpiceRect takes place.

A cookie has been added to each async call to QXLWorker, and is passed back via
async_complete.

Added api:

QXLWorker:
    update_area_async
    add_memslot_async
    destroy_surfaces_async
    destroy_primary_surface_async
    create_primary_surface_async
    destroy_surface_wait_async

QXLInterface:
    async_complete

8 files changed, 358 insertions, 91 deletions

diff --git a/server/red_dispatcher.c b/server/red_dispatcher.c
index aa1990e0..7f3efe87 100644
--- a/server/red_dispatcher.c
+++ b/server/red_dispatcher.c
@@ -43,7 +43,6 @@ static int num_active_workers = 0;
 
 //volatile
 
-typedef struct RedDispatcher RedDispatcher;
 struct RedDispatcher {
     QXLWorker base;
     QXLInstance *qxl;
@@ -55,6 +54,9 @@ struct RedDispatcher {
     int y_res;
     int use_hardware_cursor;
     RedDispatcher *next;
+    RedWorkerMessage async_message;
+    pthread_mutex_t  async_lock;
+    QXLDevSurfaceCreate *surface_create;
 };
 
 typedef struct RedWorkeState {
@@ -214,30 +216,49 @@ static void red_dispatcher_update_area(RedDispatcher *dispatcher, uint32_t surfa
                                    uint32_t num_dirty_rects, uint32_t clear_dirty_region)
 {
     RedWorkerMessage message = RED_WORKER_MESSAGE_UPDATE;
-    SpiceRect *dirty_rects = spice_new0(SpiceRect, num_dirty_rects);
-    SpiceRect *area = spice_new0(SpiceRect, 1);
-    int i;
-
-    red_get_rect_ptr(area, qxl_area);
 
     write_message(dispatcher->channel, &message);
     send_data(dispatcher->channel, &surface_id, sizeof(uint32_t));
-    send_data(dispatcher->channel, &area, sizeof(SpiceRect *));
-    send_data(dispatcher->channel, &dirty_rects, sizeof(SpiceRect *));
+    send_data(dispatcher->channel, &qxl_area, sizeof(QXLRect *));
+    send_data(dispatcher->channel, &qxl_dirty_rects, sizeof(QXLRect *));
     send_data(dispatcher->channel, &num_dirty_rects, sizeof(uint32_t));
     send_data(dispatcher->channel, &clear_dirty_region, sizeof(uint32_t));
     read_message(dispatcher->channel, &message);
     ASSERT(message == RED_WORKER_MESSAGE_READY);
+}
 
-    for (i = 0; i < num_dirty_rects; i++) {
-        qxl_dirty_rects[i].top    = dirty_rects[i].top;
-        qxl_dirty_rects[i].left   = dirty_rects[i].left;
-        qxl_dirty_rects[i].bottom = dirty_rects[i].bottom;
-        qxl_dirty_rects[i].right  = dirty_rects[i].right;
+static RedWorkerMessage red_dispatcher_async_start(RedDispatcher *dispatcher,
+                                                   RedWorkerMessage message)
+{
+    pthread_mutex_lock(&dispatcher->async_lock);
+    if (dispatcher->async_message != RED_WORKER_MESSAGE_NOP) {
+        red_printf("error: async clash. second async ignored");
+        pthread_mutex_unlock(&dispatcher->async_lock);
+        return RED_WORKER_MESSAGE_NOP;
     }
+    dispatcher->async_message = message;
+    pthread_mutex_unlock(&dispatcher->async_lock);
+    return message;
+}
+
+static void red_dispatcher_update_area_async(RedDispatcher *dispatcher,
+                                         uint32_t surface_id,
+                                         QXLRect *qxl_area,
+                                         uint32_t clear_dirty_region,
+                                         uint64_t cookie)
+{
+    RedWorkerMessage message = red_dispatcher_async_start(dispatcher,
+                                                          RED_WORKER_MESSAGE_UPDATE_ASYNC);
 
-    free(dirty_rects);
-    free(area);
+    if (message == RED_WORKER_MESSAGE_NOP) {
+        return;
+    }
+
+    write_message(dispatcher->channel, &message);
+    send_data(dispatcher->channel, &cookie, sizeof(cookie));
+    send_data(dispatcher->channel, &surface_id, sizeof(uint32_t));
+    send_data(dispatcher->channel, qxl_area, sizeof(QXLRect));
+    send_data(dispatcher->channel, &clear_dirty_region, sizeof(uint32_t));
 }
 
 static void qxl_worker_update_area(QXLWorker *qxl_worker, uint32_t surface_id,
@@ -263,6 +284,19 @@ static void qxl_worker_add_memslot(QXLWorker *qxl_worker, QXLDevMemSlot *mem_slo
     red_dispatcher_add_memslot((RedDispatcher*)qxl_worker, mem_slot);
 }
 
+static void red_dispatcher_add_memslot_async(RedDispatcher *dispatcher, QXLDevMemSlot *mem_slot, uint64_t cookie)
+{
+    RedWorkerMessage message = red_dispatcher_async_start(dispatcher,
+                                                          RED_WORKER_MESSAGE_ADD_MEMSLOT_ASYNC);
+
+    if (message == RED_WORKER_MESSAGE_NOP) {
+        return;
+    }
+    write_message(dispatcher->channel, &message);
+    send_data(dispatcher->channel, &cookie, sizeof(cookie));
+    send_data(dispatcher->channel, mem_slot, sizeof(QXLDevMemSlot));
+}
+
 static void red_dispatcher_del_memslot(RedDispatcher *dispatcher, uint32_t slot_group_id, uint32_t slot_id)
 {
     RedWorkerMessage message = RED_WORKER_MESSAGE_DEL_MEMSLOT;
@@ -291,15 +325,20 @@ static void qxl_worker_destroy_surfaces(QXLWorker *qxl_worker)
     red_dispatcher_destroy_surfaces((RedDispatcher*)qxl_worker);
 }
 
-static void red_dispatcher_destroy_primary(RedDispatcher *dispatcher, uint32_t surface_id)
+static void red_dispatcher_destroy_surfaces_async(RedDispatcher *dispatcher, uint64_t cookie)
 {
-    RedWorkerMessage message = RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE;
+    RedWorkerMessage message = red_dispatcher_async_start(dispatcher,
+                                                      RED_WORKER_MESSAGE_DESTROY_SURFACES_ASYNC);
 
+    if (message == RED_WORKER_MESSAGE_NOP) {
+        return;
+    }
     write_message(dispatcher->channel, &message);
-    send_data(dispatcher->channel, &surface_id, sizeof(uint32_t));
-    read_message(dispatcher->channel, &message);
-    ASSERT(message == RED_WORKER_MESSAGE_READY);
+    send_data(dispatcher->channel, &cookie, sizeof(cookie));
+}
 
+static void red_dispatcher_destroy_primary_surface_complete(RedDispatcher *dispatcher)
+{
     dispatcher->x_res = 0;
     dispatcher->y_res = 0;
     dispatcher->use_hardware_cursor = FALSE;
@@ -308,34 +347,86 @@ static void red_dispatcher_destroy_primary(RedDispatcher *dispatcher, uint32_t s
     update_client_mouse_allowed();
 }
 
-static void qxl_worker_destroy_primary(QXLWorker *qxl_worker, uint32_t surface_id)
+static void
+red_dispatcher_destroy_primary_surface(RedDispatcher *dispatcher,
+                                       uint32_t surface_id, int async, uint64_t cookie)
 {
-    red_dispatcher_destroy_primary((RedDispatcher*)qxl_worker, surface_id);
+    RedWorkerMessage message;
+
+    if (async) {
+        message = red_dispatcher_async_start(dispatcher,
+                                             RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE_ASYNC);
+        if (message == RED_WORKER_MESSAGE_NOP) {
+            return;
+        }
+    } else {
+        message = RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE;
+    }
+
+    write_message(dispatcher->channel, &message);
+    if (async) {
+        send_data(dispatcher->channel, &cookie, sizeof(cookie));
+    }
+    send_data(dispatcher->channel, &surface_id, sizeof(uint32_t));
+    if (!async) {
+        read_message(dispatcher->channel, &message);
+        ASSERT(message == RED_WORKER_MESSAGE_READY);
+        red_dispatcher_destroy_primary_surface_complete(dispatcher);
+    }
 }
 
-static void red_dispatcher_create_primary(RedDispatcher *dispatcher, uint32_t surface_id,
-                                      QXLDevSurfaceCreate *surface)
+static void qxl_worker_destroy_primary_surface(QXLWorker *qxl_worker, uint32_t surface_id)
+{
+    red_dispatcher_destroy_primary_surface((RedDispatcher*)qxl_worker, surface_id, 0, 0);
+}
+
+static void red_dispatcher_create_primary_surface_complete(RedDispatcher *dispatcher)
 {
-    RedWorkerMessage message = RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE;
+    QXLDevSurfaceCreate *surface = dispatcher->surface_create;
 
     dispatcher->x_res = surface->width;
     dispatcher->y_res = surface->height;
     dispatcher->use_hardware_cursor = surface->mouse_mode;
     dispatcher->primary_active = TRUE;
 
+    update_client_mouse_allowed();
+    dispatcher->surface_create = NULL;
+}
+
+static void
+red_dispatcher_create_primary_surface(RedDispatcher *dispatcher, uint32_t surface_id,
+                                      QXLDevSurfaceCreate *surface, int async, uint64_t cookie)
+{
+    RedWorkerMessage message;
+
+    if (async) {
+        message = red_dispatcher_async_start(dispatcher,
+                                             RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE_ASYNC);
+        if (message == RED_WORKER_MESSAGE_NOP) {
+            return;
+        }
+    } else {
+        message = RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE;
+    }
+    dispatcher->surface_create = surface;
+
     write_message(dispatcher->channel, &message);
+    if (async) {
+        send_data(dispatcher->channel, &cookie, sizeof(cookie));
+    }
     send_data(dispatcher->channel, &surface_id, sizeof(uint32_t));
     send_data(dispatcher->channel, surface, sizeof(QXLDevSurfaceCreate));
-    read_message(dispatcher->channel, &message);
-    ASSERT(message == RED_WORKER_MESSAGE_READY);
-
-    update_client_mouse_allowed();
+    if (!async) {
+        read_message(dispatcher->channel, &message);
+        ASSERT(message == RED_WORKER_MESSAGE_READY);
+        red_dispatcher_create_primary_surface_complete(dispatcher);
+    }
 }
 
-static void qxl_worker_create_primary(QXLWorker *qxl_worker, uint32_t surface_id,
+static void qxl_worker_create_primary_surface(QXLWorker *qxl_worker, uint32_t surface_id,
                                       QXLDevSurfaceCreate *surface)
 {
-    red_dispatcher_create_primary((RedDispatcher*)qxl_worker, surface_id, surface);
+    red_dispatcher_create_primary_surface((RedDispatcher*)qxl_worker, surface_id, surface, 0, 0);
 }
 
 static void red_dispatcher_reset_image_cache(RedDispatcher *dispatcher)
@@ -366,19 +457,36 @@ static void qxl_worker_reset_cursor(QXLWorker *qxl_worker)
     red_dispatcher_reset_cursor((RedDispatcher*)qxl_worker);
 }
 
-static void red_dispatcher_destroy_surface_wait(RedDispatcher *dispatcher, uint32_t surface_id)
+static void red_dispatcher_destroy_surface_wait(RedDispatcher *dispatcher, uint32_t surface_id,
+                                                int async, uint64_t cookie)
 {
-    RedWorkerMessage message = RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT;
+    RedWorkerMessage message;
+
+    if (async ) {
+        message = red_dispatcher_async_start(dispatcher,
+                                             RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT_ASYNC);
+        if (message == RED_WORKER_MESSAGE_NOP) {
+            return;
+        }
+    } else {
+        message = RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT;
+    }
 
     write_message(dispatcher->channel, &message);
+    if (async) {
+        send_data(dispatcher->channel, &cookie, sizeof(cookie));
+    }
     send_data(dispatcher->channel, &surface_id, sizeof(uint32_t));
+    if (async) {
+        return;
+    }
     read_message(dispatcher->channel, &message);
     ASSERT(message == RED_WORKER_MESSAGE_READY);
 }
 
 static void qxl_worker_destroy_surface_wait(QXLWorker *qxl_worker, uint32_t surface_id)
 {
-    red_dispatcher_destroy_surface_wait((RedDispatcher*)qxl_worker, surface_id);
+    red_dispatcher_destroy_surface_wait((RedDispatcher*)qxl_worker, surface_id, 0, 0);
 }
 
 static void red_dispatcher_reset_memslots(RedDispatcher *dispatcher)
@@ -607,14 +715,14 @@ void spice_qxl_destroy_surfaces(QXLInstance *instance)
 SPICE_GNUC_VISIBLE
 void spice_qxl_destroy_primary_surface(QXLInstance *instance, uint32_t surface_id)
 {
-    red_dispatcher_destroy_primary(instance->st->dispatcher, surface_id);
+    red_dispatcher_destroy_primary_surface(instance->st->dispatcher, surface_id, 0, 0);
 }
 
 SPICE_GNUC_VISIBLE
 void spice_qxl_create_primary_surface(QXLInstance *instance, uint32_t surface_id,
                                 QXLDevSurfaceCreate *surface)
 {
-    red_dispatcher_create_primary(instance->st->dispatcher, surface_id, surface);
+    red_dispatcher_create_primary_surface(instance->st->dispatcher, surface_id, surface, 0, 0);
 }
 
 SPICE_GNUC_VISIBLE
@@ -632,7 +740,7 @@ void spice_qxl_reset_cursor(QXLInstance *instance)
 SPICE_GNUC_VISIBLE
 void spice_qxl_destroy_surface_wait(QXLInstance *instance, uint32_t surface_id)
 {
-    red_dispatcher_destroy_surface_wait(instance->st->dispatcher, surface_id);
+    red_dispatcher_destroy_surface_wait(instance->st->dispatcher, surface_id, 0, 0);
 }
 
 SPICE_GNUC_VISIBLE
@@ -641,6 +749,71 @@ void spice_qxl_loadvm_commands(QXLInstance *instance, struct QXLCommandExt *ext,
     red_dispatcher_loadvm_commands(instance->st->dispatcher, ext, count);
 }
 
+SPICE_GNUC_VISIBLE
+void spice_qxl_update_area_async(QXLInstance *instance, uint32_t surface_id, QXLRect *qxl_area,
+                                 uint32_t clear_dirty_region, uint64_t cookie)
+{
+    red_dispatcher_update_area_async(instance->st->dispatcher, surface_id, qxl_area,
+                                     clear_dirty_region, cookie);
+}
+
+SPICE_GNUC_VISIBLE
+void spice_qxl_add_memslot_async(QXLInstance *instance, QXLDevMemSlot *slot, uint64_t cookie)
+{
+    red_dispatcher_add_memslot_async(instance->st->dispatcher, slot, cookie);
+}
+
+SPICE_GNUC_VISIBLE
+void spice_qxl_destroy_surfaces_async(QXLInstance *instance, uint64_t cookie)
+{
+    red_dispatcher_destroy_surfaces_async(instance->st->dispatcher, cookie);
+}
+
+SPICE_GNUC_VISIBLE
+void spice_qxl_destroy_primary_surface_async(QXLInstance *instance, uint32_t surface_id, uint64_t cookie)
+{
+    red_dispatcher_destroy_primary_surface(instance->st->dispatcher, surface_id, 1, cookie);
+}
+
+SPICE_GNUC_VISIBLE
+void spice_qxl_create_primary_surface_async(QXLInstance *instance, uint32_t surface_id,
+                                QXLDevSurfaceCreate *surface, uint64_t cookie)
+{
+    red_dispatcher_create_primary_surface(instance->st->dispatcher, surface_id, surface, 1, cookie);
+}
+
+SPICE_GNUC_VISIBLE
+void spice_qxl_destroy_surface_async(QXLInstance *instance, uint32_t surface_id, uint64_t cookie)
+{
+    red_dispatcher_destroy_surface_wait(instance->st->dispatcher, surface_id, 1, cookie);
+}
+
+void red_dispatcher_async_complete(struct RedDispatcher *dispatcher, uint64_t cookie)
+{
+    pthread_mutex_lock(&dispatcher->async_lock);
+    switch (dispatcher->async_message) {
+    case RED_WORKER_MESSAGE_UPDATE_ASYNC:
+        break;
+    case RED_WORKER_MESSAGE_ADD_MEMSLOT_ASYNC:
+        break;
+    case RED_WORKER_MESSAGE_DESTROY_SURFACES_ASYNC:
+        break;
+    case RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE_ASYNC:
+        red_dispatcher_create_primary_surface_complete(dispatcher);
+        break;
+    case RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE_ASYNC:
+        red_dispatcher_destroy_primary_surface_complete(dispatcher);
+        break;
+    case RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT_ASYNC:
+        break;
+    default:
+        red_printf("unexpected message");
+    }
+    dispatcher->async_message = RED_WORKER_MESSAGE_NOP;
+    pthread_mutex_unlock(&dispatcher->async_lock);
+    dispatcher->qxl->st->qif->async_complete(dispatcher->qxl, cookie);
+}
+
 RedDispatcher *red_dispatcher_init(QXLInstance *qxl)
 {
     RedDispatcher *dispatcher;
@@ -673,6 +846,8 @@ RedDispatcher *red_dispatcher_init(QXLInstance *qxl)
     init_data.num_renderers = num_renderers;
     memcpy(init_data.renderers, renderers, sizeof(init_data.renderers));
 
+    dispatcher->async_message = RED_WORKER_MESSAGE_NOP;
+    pthread_mutex_init(&dispatcher->async_lock, NULL);
     init_data.image_compression = image_compression;
     init_data.jpeg_state = jpeg_state;
     init_data.zlib_glz_state = zlib_glz_state;
@@ -689,8 +864,8 @@ RedDispatcher *red_dispatcher_init(QXLInstance *qxl)
     dispatcher->base.del_memslot = qxl_worker_del_memslot;
     dispatcher->base.reset_memslots = qxl_worker_reset_memslots;
     dispatcher->base.destroy_surfaces = qxl_worker_destroy_surfaces;
-    dispatcher->base.create_primary_surface = qxl_worker_create_primary;
-    dispatcher->base.destroy_primary_surface = qxl_worker_destroy_primary;
+    dispatcher->base.create_primary_surface = qxl_worker_create_primary_surface;
+    dispatcher->base.destroy_primary_surface = qxl_worker_destroy_primary_surface;
 
     dispatcher->base.reset_image_cache = qxl_worker_reset_image_cache;
     dispatcher->base.reset_cursor = qxl_worker_reset_cursor;
@@ -705,6 +880,7 @@ RedDispatcher *red_dispatcher_init(QXLInstance *qxl)
     init_data.num_memslots_groups = init_info.num_memslots_groups;
     init_data.internal_groupslot_id = init_info.internal_groupslot_id;
     init_data.n_surfaces = init_info.n_surfaces;
+    init_data.dispatcher = dispatcher;
 
     num_active_workers = 1;
 
@@ -745,4 +921,3 @@ RedDispatcher *red_dispatcher_init(QXLInstance *qxl)
     dispatchers = dispatcher;
     return dispatcher;
 }
-
diff --git a/server/red_dispatcher.h b/server/red_dispatcher.h
index 3f3c1aef..fa347f19 100644
--- a/server/red_dispatcher.h
+++ b/server/red_dispatcher.h
@@ -18,7 +18,6 @@
 #ifndef _H_RED_DISPATCHER
 #define _H_RED_DISPATCHER
 
-
 struct RedDispatcher *red_dispatcher_init(QXLInstance *qxl);
 
 void red_dispatcher_set_mm_time(uint32_t);
@@ -29,5 +28,7 @@ int red_dispatcher_count(void);
 int red_dispatcher_add_renderer(const char *name);
 uint32_t red_dispatcher_qxl_ram_size(void);
 int red_dispatcher_qxl_count(void);
+void red_dispatcher_async_complete(struct RedDispatcher*, uint64_t);
+
 #endif
 
diff --git a/server/red_parse_qxl.c b/server/red_parse_qxl.c
index 258a541d..7737c047 100644
--- a/server/red_parse_qxl.c
+++ b/server/red_parse_qxl.c
@@ -148,7 +148,7 @@ static void red_get_point16_ptr(SpicePoint16 *red, QXLPoint16 *qxl)
     red->y = qxl->y;
 }
 
-void red_get_rect_ptr(SpiceRect *red, QXLRect *qxl)
+void red_get_rect_ptr(SpiceRect *red, const QXLRect *qxl)
 {
     red->top    = qxl->top;
     red->left   = qxl->left;
diff --git a/server/red_parse_qxl.h b/server/red_parse_qxl.h
index 5de03256..a713dcfb 100644
--- a/server/red_parse_qxl.h
+++ b/server/red_parse_qxl.h
@@ -110,7 +110,7 @@ typedef struct RedCursorCmd {
     uint8_t *device_data;
 } RedCursorCmd;
 
-void red_get_rect_ptr(SpiceRect *red, QXLRect *qxl);
+void red_get_rect_ptr(SpiceRect *red, const QXLRect *qxl);
 
 void red_get_drawable(RedMemSlotInfo *slots, int group_id,
                       RedDrawable *red, QXLPHYSICAL addr, uint32_t flags);
diff --git a/server/red_worker.c b/server/red_worker.c
index 2349ac51..f29a4207 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -61,6 +61,7 @@
 #include "generated_marshallers.h"
 #include "zlib_encoder.h"
 #include "red_channel.h"
+#include "red_dispatcher.h"
 
 //#define COMPRESS_STAT
 //#define DUMP_BITMAP
@@ -805,6 +806,7 @@ typedef struct RedWorker {
     DisplayChannel *display_channel;
     CursorChannel *cursor_channel;
     QXLInstance *qxl;
+    RedDispatcher *dispatcher;
     int id;
     int channel;
     int running;
@@ -9424,28 +9426,53 @@ static void red_wait_pipe_item_sent(RedChannel *channel, PipeItem *item)
     red_unref_channel(channel);
 }
 
+static inline void handle_dev_update_async(RedWorker *worker)
+{
+    QXLRect qxl_rect;
+    SpiceRect rect;
+    uint32_t surface_id;
+    uint32_t clear_dirty_region;
+
+    receive_data(worker->channel, &surface_id, sizeof(uint32_t));
+    receive_data(worker->channel, &qxl_rect, sizeof(QXLRect));
+    receive_data(worker->channel, &clear_dirty_region, sizeof(uint32_t));
+
+    red_get_rect_ptr(&rect, &qxl_rect);
+    flush_display_commands(worker);
+
+    ASSERT(worker->running);
+
+    validate_surface(worker, surface_id);
+    red_update_area(worker, &rect, surface_id);
+}
+
 static inline void handle_dev_update(RedWorker *worker)
 {
-    RedWorkerMessage message;
-    const SpiceRect *rect;
+    const QXLRect *qxl_rect;
+    SpiceRect *rect = spice_new0(SpiceRect, 1);
+    QXLRect *qxl_dirty_rects;
     SpiceRect *dirty_rects;
     RedSurface *surface;
     uint32_t num_dirty_rects;
     uint32_t surface_id;
     uint32_t clear_dirty_region;
+    int i;
 
     receive_data(worker->channel, &surface_id, sizeof(uint32_t));
-    receive_data(worker->channel, &rect, sizeof(SpiceRect *));
-    receive_data(worker->channel, &dirty_rects, sizeof(SpiceRect *));
+    receive_data(worker->channel, &qxl_rect, sizeof(QXLRect *));
+    receive_data(worker->channel, &qxl_dirty_rects, sizeof(QXLRect *));
     receive_data(worker->channel, &num_dirty_rects, sizeof(uint32_t));
     receive_data(worker->channel, &clear_dirty_region, sizeof(uint32_t));
 
+    dirty_rects = spice_new0(SpiceRect, num_dirty_rects);
+    red_get_rect_ptr(rect, qxl_rect);
     flush_display_commands(worker);
 
     ASSERT(worker->running);
 
     validate_surface(worker, surface_id);
     red_update_area(worker, rect, surface_id);
+    free(rect);
 
     surface = &worker->surfaces[surface_id];
     region_ret_rects(&surface->draw_dirty_region, dirty_rects, num_dirty_rects);
@@ -9453,15 +9480,17 @@ static inline void handle_dev_update(RedWorker *worker)
     if (clear_dirty_region) {
         region_clear(&surface->draw_dirty_region);
     }
-
-    message = RED_WORKER_MESSAGE_READY;
-    write_message(worker->channel, &message);
+    for (i = 0; i < num_dirty_rects; i++) {
+        qxl_dirty_rects[i].top    = dirty_rects[i].top;
+        qxl_dirty_rects[i].left   = dirty_rects[i].left;
+        qxl_dirty_rects[i].bottom = dirty_rects[i].bottom;
+        qxl_dirty_rects[i].right  = dirty_rects[i].right;
+    }
+    free(dirty_rects);
 }
 
-
 static inline void handle_dev_add_memslot(RedWorker *worker)
 {
-    RedWorkerMessage message;
     QXLDevMemSlot dev_slot;
 
     receive_data(worker->channel, &dev_slot, sizeof(QXLDevMemSlot));
@@ -9469,9 +9498,6 @@ static inline void handle_dev_add_memslot(RedWorker *worker)
     red_memslot_info_add_slot(&worker->mem_slots, dev_slot.slot_group_id, dev_slot.slot_id,
                               dev_slot.addr_delta, dev_slot.virt_start, dev_slot.virt_end,
                               dev_slot.generation);
-
-    message = RED_WORKER_MESSAGE_READY;
-    write_message(worker->channel, &message);
 }
 
 static inline void handle_dev_del_memslot(RedWorker *worker)
@@ -9507,7 +9533,6 @@ static inline void destroy_surface_wait(RedWorker *worker, int surface_id)
 
 static inline void handle_dev_destroy_surface_wait(RedWorker *worker)
 {
-    RedWorkerMessage message;
     uint32_t surface_id;
 
     receive_data(worker->channel, &surface_id, sizeof(uint32_t));
@@ -9519,9 +9544,6 @@ static inline void handle_dev_destroy_surface_wait(RedWorker *worker)
     if (worker->surfaces[0].context.canvas) {
         destroy_surface_wait(worker, 0);
     }
-
-    message = RED_WORKER_MESSAGE_READY;
-    write_message(worker->channel, &message);
 }
 
 static inline void red_cursor_reset(RedWorker *worker)
@@ -9549,7 +9571,6 @@ static inline void red_cursor_reset(RedWorker *worker)
 static inline void handle_dev_destroy_surfaces(RedWorker *worker)
 {
     int i;
-    RedWorkerMessage message;
 
     flush_all_qxl_commands(worker);
     //to handle better
@@ -9572,14 +9593,10 @@ static inline void handle_dev_destroy_surfaces(RedWorker *worker)
     red_display_clear_glz_drawables(worker->display_channel);
 
     red_cursor_reset(worker);
-
-    message = RED_WORKER_MESSAGE_READY;
-    write_message(worker->channel, &message);
 }
 
 static inline void handle_dev_create_primary_surface(RedWorker *worker)
 {
-    RedWorkerMessage message;
     uint32_t surface_id;
     QXLDevSurfaceCreate surface;
     uint8_t *line_0;
@@ -9609,14 +9626,10 @@ static inline void handle_dev_create_primary_surface(RedWorker *worker)
     if (worker->cursor_channel) {
         red_channel_pipe_add_type(&worker->cursor_channel->common.base, PIPE_ITEM_TYPE_CURSOR_INIT);
     }
-
-    message = RED_WORKER_MESSAGE_READY;
-    write_message(worker->channel, &message);
 }
 
 static inline void handle_dev_destroy_primary_surface(RedWorker *worker)
 {
-    RedWorkerMessage message;
     uint32_t surface_id;
 
     receive_data(worker->channel, &surface_id, sizeof(uint32_t));
@@ -9632,22 +9645,78 @@ static inline void handle_dev_destroy_primary_surface(RedWorker *worker)
     ASSERT(!worker->surfaces[surface_id].context.canvas);
 
     red_cursor_reset(worker);
+}
 
-    message = RED_WORKER_MESSAGE_READY;
-    write_message(worker->channel, &message);
+static void handle_dev_stop(RedWorker *worker)
+{
+    int x;
+
+    ASSERT(worker->running);
+    worker->running = FALSE;
+    red_display_clear_glz_drawables(worker->display_channel);
+    for (x = 0; x < NUM_SURFACES; ++x) {
+        if (worker->surfaces[x].context.canvas) {
+            red_current_flush(worker, x);
+        }
+    }
+    red_wait_outgoing_item((RedChannel *)worker->display_channel);
+    red_wait_outgoing_item((RedChannel *)worker->cursor_channel);
+}
+
+static void handle_dev_start(RedWorker *worker)
+{
+    RedChannel *cursor_red_channel = &worker->cursor_channel->common.base;
+    RedChannel *display_red_channel = &worker->display_channel->common.base;
+
+    ASSERT(!worker->running);
+    if (worker->cursor_channel) {
+        cursor_red_channel->migrate = FALSE;
+    }
+    if (worker->display_channel) {
+        display_red_channel->migrate = FALSE;
+    }
+    worker->running = TRUE;
 }
 
 static void handle_dev_input(EventListener *listener, uint32_t events)
 {
     RedWorker *worker = SPICE_CONTAINEROF(listener, RedWorker, dev_listener);
     RedWorkerMessage message;
-    RedChannel *cursor_red_channel = &worker->cursor_channel->common.base;
     RedChannel *display_red_channel = &worker->display_channel->common.base;
     int ring_is_empty;
+    int call_async_complete = 0;
+    int write_ready = 0;
+    uint64_t cookie;
 
     read_message(worker->channel, &message);
 
+    /* for async messages we do the common work in the handler, and
+     * send a ready or call async_complete from here, hence the added switch. */
+    switch (message) {
+    case RED_WORKER_MESSAGE_UPDATE_ASYNC:
+    case RED_WORKER_MESSAGE_ADD_MEMSLOT_ASYNC:
+    case RED_WORKER_MESSAGE_DESTROY_SURFACES_ASYNC:
+    case RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE_ASYNC:
+    case RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE_ASYNC:
+    case RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT_ASYNC:
+        call_async_complete = 1;
+        receive_data(worker->channel, &cookie, sizeof(cookie));
+        break;
+    case RED_WORKER_MESSAGE_UPDATE:
+    case RED_WORKER_MESSAGE_ADD_MEMSLOT:
+    case RED_WORKER_MESSAGE_DESTROY_SURFACES:
+    case RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE:
+    case RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE:
+    case RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT:
+        write_ready = 1;
+    default:
+        break;
+    }
+
     switch (message) {
+    case RED_WORKER_MESSAGE_UPDATE_ASYNC:
+        handle_dev_update_async(worker);
+        break;
     case RED_WORKER_MESSAGE_UPDATE:
         handle_dev_update(worker);
         break;
@@ -9679,15 +9748,19 @@ static void handle_dev_input(EventListener *listener, uint32_t events)
         message = RED_WORKER_MESSAGE_READY;
         write_message(worker->channel, &message);
         break;
+    case RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT_ASYNC:
     case RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT:
         handle_dev_destroy_surface_wait(worker);
         break;
+    case RED_WORKER_MESSAGE_DESTROY_SURFACES_ASYNC:
     case RED_WORKER_MESSAGE_DESTROY_SURFACES:
         handle_dev_destroy_surfaces(worker);
         break;
+    case RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE_ASYNC:
     case RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE:
         handle_dev_create_primary_surface(worker);
         break;
+    case RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE_ASYNC:
     case RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE:
         handle_dev_destroy_primary_surface(worker);
         break;
@@ -9706,33 +9779,15 @@ static void handle_dev_input(EventListener *listener, uint32_t events)
         red_disconnect_display((RedChannel *)worker->display_channel);
         break;
     case RED_WORKER_MESSAGE_STOP: {
-        int x;
-
         red_printf("stop");
-        ASSERT(worker->running);
-        worker->running = FALSE;
-        red_display_clear_glz_drawables(worker->display_channel);
-        for (x = 0; x < NUM_SURFACES; ++x) {
-            if (worker->surfaces[x].context.canvas) {
-                red_current_flush(worker, x);
-            }
-        }
-        red_wait_outgoing_item((RedChannel *)worker->display_channel);
-        red_wait_outgoing_item((RedChannel *)worker->cursor_channel);
+        handle_dev_stop(worker);
         message = RED_WORKER_MESSAGE_READY;
         write_message(worker->channel, &message);
         break;
     }
     case RED_WORKER_MESSAGE_START:
         red_printf("start");
-        ASSERT(!worker->running);
-        if (worker->cursor_channel) {
-            cursor_red_channel->migrate = FALSE;
-        }
-        if (worker->display_channel) {
-            display_red_channel->migrate = FALSE;
-        }
-        worker->running = TRUE;
+        handle_dev_start(worker);
         break;
     case RED_WORKER_MESSAGE_DISPLAY_MIGRATE:
         red_printf("migrate");
@@ -9814,6 +9869,7 @@ static void handle_dev_input(EventListener *listener, uint32_t events)
         receive_data(worker->channel, &worker->mouse_mode, sizeof(uint32_t));
         red_printf("mouse mode %u", worker->mouse_mode);
         break;
+    case RED_WORKER_MESSAGE_ADD_MEMSLOT_ASYNC:
     case RED_WORKER_MESSAGE_ADD_MEMSLOT:
         handle_dev_add_memslot(worker);
         break;
@@ -9859,6 +9915,13 @@ static void handle_dev_input(EventListener *listener, uint32_t events)
     default:
         red_error("message error");
     }
+    if (call_async_complete) {
+        red_dispatcher_async_complete(worker->dispatcher, cookie);
+    }
+    if (write_ready) {
+        message = RED_WORKER_MESSAGE_READY;
+        write_message(worker->channel, &message);
+    }
 }
 
 static void handle_dev_free(EventListener *ctx)
@@ -9875,6 +9938,7 @@ static void red_init(RedWorker *worker, WorkerInitData *init_data)
     ASSERT(sizeof(CursorItem) <= QXL_CURSUR_DEVICE_DATA_SIZE);
 
     memset(worker, 0, sizeof(RedWorker));
+    worker->dispatcher = init_data->dispatcher;
     worker->qxl = init_data->qxl;
     worker->id = init_data->id;
     worker->channel = init_data->channel;
diff --git a/server/red_worker.h b/server/red_worker.h
index b4e2ed2e..604437bf 100644
--- a/server/red_worker.h
+++ b/server/red_worker.h
@@ -70,6 +70,13 @@ enum {
     RED_WORKER_MESSAGE_RESET_IMAGE_CACHE,
     RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT,
     RED_WORKER_MESSAGE_LOADVM_COMMANDS,
+    /* async commands */
+    RED_WORKER_MESSAGE_UPDATE_ASYNC,
+    RED_WORKER_MESSAGE_ADD_MEMSLOT_ASYNC,
+    RED_WORKER_MESSAGE_DESTROY_SURFACES_ASYNC,
+    RED_WORKER_MESSAGE_CREATE_PRIMARY_SURFACE_ASYNC,
+    RED_WORKER_MESSAGE_DESTROY_PRIMARY_SURFACE_ASYNC,
+    RED_WORKER_MESSAGE_DESTROY_SURFACE_WAIT_ASYNC,
 };
 
 typedef uint32_t RedWorkerMessage;
@@ -83,6 +90,8 @@ enum {
     RED_RENDERER_OGL_PIXMAP,
 };
 
+typedef struct RedDispatcher RedDispatcher;
+
 typedef struct WorkerInitData {
     struct QXLInstance *qxl;
     int id;
@@ -100,6 +109,7 @@ typedef struct WorkerInitData {
     uint8_t memslot_id_bits;
     uint8_t internal_groupslot_id;
     uint32_t n_surfaces;
+    RedDispatcher *dispatcher;
 } WorkerInitData;
 
 void *red_worker_main(void *arg);
diff --git a/server/spice-server.syms b/server/spice-server.syms
index 4e120ee9..28e6025b 100644
--- a/server/spice-server.syms
+++ b/server/spice-server.syms
@@ -73,6 +73,12 @@ global:
     spice_qxl_reset_cursor;
     spice_qxl_destroy_surface_wait;
     spice_qxl_loadvm_commands;
+    spice_qxl_update_area_async;
+    spice_qxl_add_memslot_async;
+    spice_qxl_destroy_surfaces_async;
+    spice_qxl_destroy_primary_surface_async;
+    spice_qxl_create_primary_surface_async;
+    spice_qxl_destroy_surface_async;
 } SPICE_SERVER_0.8.1;
 
 SPICE_SERVER_0.10.0 {
diff --git a/server/spice.h b/server/spice.h
index 0b82bb78..c7f0a6ef 100644
--- a/server/spice.h
+++ b/server/spice.h
@@ -20,6 +20,7 @@
 
 #include <stdint.h>
 #include <sys/socket.h>
+#include <spice/qxl_dev.h>
 
 #define SPICE_SERVER_VERSION 0x000900 /* release 0.9.0 */
 
@@ -143,6 +144,15 @@ void spice_qxl_reset_image_cache(QXLInstance *instance);
 void spice_qxl_reset_cursor(QXLInstance *instance);
 void spice_qxl_destroy_surface_wait(QXLInstance *instance, uint32_t surface_id);
 void spice_qxl_loadvm_commands(QXLInstance *instance, struct QXLCommandExt *ext, uint32_t count);
+/* async versions of commands. when complete spice calls async_complete */
+void spice_qxl_update_area_async(QXLInstance *instance, uint32_t surface_id, QXLRect *qxl_area,
+                                 uint32_t clear_dirty_region, uint64_t cookie);
+void spice_qxl_add_memslot_async(QXLInstance *instance, QXLDevMemSlot *slot, uint64_t cookie);
+void spice_qxl_destroy_surfaces_async(QXLInstance *instance, uint64_t cookie);
+void spice_qxl_destroy_primary_surface_async(QXLInstance *instance, uint32_t surface_id, uint64_t cookie);
+void spice_qxl_create_primary_surface_async(QXLInstance *instance, uint32_t surface_id,
+                                QXLDevSurfaceCreate *surface, uint64_t cookie);
+void spice_qxl_destroy_surface_async(QXLInstance *instance, uint32_t surface_id, uint64_t cookie);
 
 typedef struct QXLDrawArea {
     uint8_t *buf;
@@ -212,6 +222,7 @@ struct QXLInterface {
     int (*req_cursor_notification)(QXLInstance *qin);
     void (*notify_update)(QXLInstance *qin, uint32_t update_id);
     int (*flush_resources)(QXLInstance *qin);
+    void (*async_complete)(QXLInstance *qin, uint64_t cookie);
 };
 
 struct QXLInstance {