server: handling semi-seamless migration in the target side

(1) not sending anything to a migrated client till we recieve SPICE_MSGC_MIGRATE_END
(2) start a new client migration (handle client_migrate_info) only after SPICE_MSGC_MIGRATE_END
    from the previous migration was received for this client
(3) use the correct ticket

Note: we assume the same channles are linked before and ater migration. i.e.,
      SPICE_MSGC_MAIN_ATTACH_CHANNELS is not sent from the clients.

6 files changed, 248 insertions, 46 deletions

diff --git a/server/main_channel.c b/server/main_channel.c
index ffc593d2..b2439b29 100644
--- a/server/main_channel.c
+++ b/server/main_channel.c
@@ -129,6 +129,8 @@ struct MainChannelClient {
 #endif
     int mig_wait_connect;
     int mig_connect_ok;
+    int mig_wait_prev_complete;
+    int init_sent;
 };
 
 enum NetTestStage {
@@ -138,6 +140,9 @@ enum NetTestStage {
     NET_TEST_STAGE_RATE,
 };
 
+static void main_channel_release_pipe_item(RedChannelClient *rcc,
+                                           PipeItem *base, int item_pushed);
+
 int main_channel_is_connected(MainChannel *main_chan)
 {
     return red_channel_is_connected(&main_chan->base);
@@ -289,6 +294,11 @@ static PipeItem *main_multi_media_time_item_new(
 
 static void main_channel_push_channels(MainChannelClient *mcc)
 {
+    if (red_client_during_migrate_at_target(mcc->base.client)) {
+        red_printf("warning: ignoring unexpected SPICE_MSGC_MAIN_ATTACH_CHANNELS"
+                   "during migration");
+        return;
+    }
     red_channel_client_pipe_add_type(&mcc->base, SPICE_MSG_MAIN_CHANNELS_LIST);
 }
 
@@ -451,7 +461,7 @@ static uint64_t main_channel_handle_migrate_data(RedChannelClient *base,
     return TRUE;
 }
 
-void main_channel_push_init(MainChannelClient *mcc, int connection_id,
+void main_channel_push_init(MainChannelClient *mcc,
     int display_channels_hint, int current_mouse_mode,
     int is_client_mouse_allowed, int multi_media_time,
     int ram_hint)
@@ -459,7 +469,7 @@ void main_channel_push_init(MainChannelClient *mcc, int connection_id,
     PipeItem *item;
 
     item = main_init_item_new(mcc,
-             connection_id, display_channels_hint, current_mouse_mode,
+             mcc->connection_id, display_channels_hint, current_mouse_mode,
              is_client_mouse_allowed, multi_media_time, ram_hint);
     red_channel_client_pipe_add_push(&mcc->base, item);
 }
@@ -612,6 +622,13 @@ static void main_channel_send_item(RedChannelClient *rcc, PipeItem *base)
     MainChannelClient *mcc = SPICE_CONTAINEROF(rcc, MainChannelClient, base);
     SpiceMarshaller *m = red_channel_client_get_marshaller(rcc);
 
+    if (!mcc->init_sent && base->type != SPICE_MSG_MAIN_INIT) {
+        red_printf("Init msg for client %p was not sent yet "
+                   "(client is probably during migration). Ignoring msg type %d",
+                   rcc->client, base->type);
+        main_channel_release_pipe_item(rcc, base, FALSE);
+        return;
+    }
     red_channel_client_init_send_data(rcc, base->type, base);
     switch (base->type) {
         case SPICE_MSG_MAIN_CHANNELS_LIST:
@@ -646,6 +663,7 @@ static void main_channel_send_item(RedChannelClient *rcc, PipeItem *base)
                 mcc->ping_id);
             break;
         case SPICE_MSG_MAIN_INIT:
+            mcc->init_sent = TRUE;
             main_channel_marshall_init(m,
                 SPICE_CONTAINEROF(base, InitPipeItem, base));
             break;
@@ -710,6 +728,25 @@ void main_channel_client_handle_migrate_connected(MainChannelClient *mcc, int su
     }
 }
 
+void main_channel_client_handle_migrate_end(MainChannelClient *mcc)
+{
+    if (!red_client_during_migrate_at_target(mcc->base.client)) {
+        red_printf("unexpected SPICE_MSGC_MIGRATE_END");
+        return;
+    }
+    if (!red_channel_client_test_remote_cap(&mcc->base,
+                                            SPICE_MAIN_CAP_SEMI_SEAMLESS_MIGRATE)) {
+        red_printf("unexpected SPICE_MSGC_MIGRATE_END, "
+                   "client does not support semi-seamless migration");
+            return;
+    }
+    red_client_migrate_complete(mcc->base.client);
+    if (mcc->mig_wait_prev_complete) {
+        red_channel_client_pipe_add_type(&mcc->base, SPICE_MSG_MAIN_MIGRATE_BEGIN);
+        mcc->mig_wait_connect = TRUE;
+        mcc->mig_wait_prev_complete = FALSE;
+    }
+}
 static int main_channel_handle_parsed(RedChannelClient *rcc, uint32_t size, uint16_t type, void *message)
 {
     MainChannel *main_chan = SPICE_CONTAINEROF(rcc->channel, MainChannel, base);
@@ -797,6 +834,9 @@ static int main_channel_handle_parsed(RedChannelClient *rcc, uint32_t size, uint
                 }
     case SPICE_MSGC_DISCONNECTING:
         break;
+    case SPICE_MSGC_MAIN_MIGRATE_END:
+        main_channel_client_handle_migrate_end(mcc);
+        break;
     default:
         red_printf("unexpected type %d", type);
     }
@@ -989,8 +1029,13 @@ int main_channel_migrate_connect(MainChannel *main_channel, RedsMigSpice *mig_ta
         MainChannelClient * mcc = SPICE_CONTAINEROF(client_link, MainChannelClient, base.channel_link);
         if (red_channel_client_test_remote_cap(&mcc->base,
                                                SPICE_MAIN_CAP_SEMI_SEAMLESS_MIGRATE)) {
-            red_channel_client_pipe_add_type(&mcc->base, SPICE_MSG_MAIN_MIGRATE_BEGIN);
-            mcc->mig_wait_connect = TRUE;
+            if (red_client_during_migrate_at_target(mcc->base.client)) {
+                red_printf("client %p: wait till previous migration completes", mcc->base.client);
+                mcc->mig_wait_prev_complete = TRUE;
+            } else {
+                red_channel_client_pipe_add_type(&mcc->base, SPICE_MSG_MAIN_MIGRATE_BEGIN);
+                mcc->mig_wait_connect = TRUE;
+            }
             mcc->mig_connect_ok = FALSE;
             main_channel->num_clients_mig_wait++;
         }
@@ -1011,6 +1056,7 @@ void main_channel_migrate_cancel_wait(MainChannel *main_chan)
             mcc->mig_wait_connect = FALSE;
             mcc->mig_connect_ok = FALSE;
         }
+        mcc->mig_wait_prev_complete = FALSE;
     }
     main_chan->num_clients_mig_wait = 0;
 }
diff --git a/server/main_channel.h b/server/main_channel.h
index d97857db..c5d407ec 100644
--- a/server/main_channel.h
+++ b/server/main_channel.h
@@ -78,7 +78,7 @@ void main_channel_push_agent_data(MainChannel *main_chan, uint8_t* data, size_t
 void main_channel_client_start_net_test(MainChannelClient *mcc);
 // TODO: huge. Consider making a reds_* interface for these functions
 // and calling from main.
-void main_channel_push_init(MainChannelClient *mcc, int connection_id, int display_channels_hint,
+void main_channel_push_init(MainChannelClient *mcc, int display_channels_hint,
     int current_mouse_mode, int is_client_mouse_allowed, int multi_media_time,
     int ram_hint);
 void main_channel_push_notify(MainChannel *main_chan, uint8_t *mess, const int mess_len);
diff --git a/server/red_channel.c b/server/red_channel.c
index 51415cbf..2ce0094c 100644
--- a/server/red_channel.c
+++ b/server/red_channel.c
@@ -1215,7 +1215,7 @@ void red_channel_client_pipe_remove_and_release(RedChannelClient *rcc,
  * pretty tied together.
  */
 
-RedClient *red_client_new()
+RedClient *red_client_new(int migrated)
 {
     RedClient *client;
 
@@ -1223,6 +1223,7 @@ RedClient *red_client_new()
     ring_init(&client->channels);
     pthread_mutex_init(&client->lock, NULL);
     client->thread_id = pthread_self();
+    client->migrated = migrated;
 
     return client;
 }
@@ -1286,6 +1287,18 @@ void red_client_set_main(RedClient *client, MainChannelClient *mcc) {
     client->mcc = mcc;
 }
 
+void red_client_migrate_complete(RedClient *client)
+{
+    ASSERT(client->migrated);
+    client->migrated = FALSE;
+    reds_on_client_migrate_complete(client);
+}
+
+int red_client_during_migrate_at_target(RedClient *client)
+{
+    return client->migrated;
+}
+
 /*
  * Functions to push the same item to multiple pipes.
  */
diff --git a/server/red_channel.h b/server/red_channel.h
index e30401c9..cce69658 100644
--- a/server/red_channel.h
+++ b/server/red_channel.h
@@ -450,13 +450,15 @@ struct RedClient {
     pthread_t thread_id;
 
     int disconnecting;
-
+    int migrated;
 };
 
-RedClient *red_client_new(void);
+RedClient *red_client_new(int migrated);
 MainChannelClient *red_client_get_main(RedClient *client);
 // main should be set once before all the other channels are created
 void red_client_set_main(RedClient *client, MainChannelClient *mcc);
+void red_client_migrate_complete(RedClient *client);
+int red_client_during_migrate_at_target(RedClient *client);
 
 void red_client_migrate(RedClient *client);
 // disconnects all the client's channels (should be called from the client's thread)
diff --git a/server/reds.c b/server/reds.c
index 43fdc2a3..045e2752 100644
--- a/server/reds.c
+++ b/server/reds.c
@@ -191,6 +191,18 @@ typedef struct RedsStatValue {
 
 #endif
 
+typedef struct RedsMigPendingLink {
+    RingItem ring_link; // list of links that belongs to the same client
+    SpiceLinkMess *link_msg;
+    RedsStream *stream;
+} RedsMigPendingLink;
+
+typedef struct RedsMigTargetClient {
+    RingItem link;
+    RedClient *client;
+    Ring pending_links;
+} RedsMigTargetClient;
+
 typedef struct RedsState {
     int listen_socket;
     int secure_listen_socket;
@@ -206,7 +218,8 @@ typedef struct RedsState {
     int mig_wait_disconnect;
     int mig_inprogress;
     int expect_migrate;
-    int mig_target;
+    Ring mig_target_clients;
+    int num_mig_target_clients;
     RedsMigSpice *mig_spice;
     int num_of_channels;
     Ring channels;
@@ -219,7 +232,6 @@ typedef struct RedsState {
     SpiceTimer *vdi_port_write_timer;
     int vdi_port_write_timer_started;
 
-    TicketAuthentication taTicket;
     SSL_CTX *ctx;
 
 #ifdef RED_STATISTICS
@@ -295,6 +307,8 @@ struct ChannelSecurityOptions {
 };
 
 static void migrate_timeout(void *opaque);
+static RedsMigTargetClient* reds_mig_target_client_find(RedClient *client);
+static void reds_mig_target_client_free(RedsMigTargetClient *mig_client);
 
 static ChannelSecurityOptions *channels_security = NULL;
 static int default_channel_security =
@@ -593,6 +607,8 @@ static int reds_main_channel_connected(void)
 
 void reds_client_disconnect(RedClient *client)
 {
+    RedsMigTargetClient *mig_client;
+
     if (!client || client->disconnecting) {
         return;
     }
@@ -607,6 +623,10 @@ void reds_client_disconnect(RedClient *client)
     // TODO: we need to handle agent properly for all clients!!!! (e.g., cut and paste, how?)
     // We shouldn't initialize the agent when there are still clients connected
 
+    mig_client = reds_mig_target_client_find(client);
+    if (mig_client) {
+        reds_mig_target_client_free(mig_client);
+    }
     ring_remove(&client->link);
     reds->num_clients--;
     red_client_destroy(client);
@@ -693,14 +713,14 @@ static void reds_update_mouse_mode(void)
 
 static void reds_agent_remove(void)
 {
-    if (!reds->mig_target) {
-        reds_reset_vdp();
-    }
+    // TODO: agent is broken with multiple clients. also need to figure out what to do when
+    // part of the clients are during target migration.
+    reds_reset_vdp();
 
     vdagent = NULL;
     reds_update_mouse_mode();
 
-    if (reds_main_channel_connected() && !reds->mig_target) {
+    if (reds_main_channel_connected()) {
         main_channel_push_agent_disconnected(reds->main_channel);
     }
 }
@@ -740,7 +760,7 @@ static int write_to_vdi_port(void)
     int total = 0;
     int n;
 
-    if (!vdagent || reds->mig_target) {
+    if (!vdagent) {
         return 0;
     }
 
@@ -844,8 +864,8 @@ static int read_from_vdi_port(void)
     }
     inside_call = 1;
 
-    if (reds->mig_target || !vdagent) {
-        // discard data only if we are migrating or vdagent has not been
+    if (!vdagent) {
+        // discard data only if we are migrating (?) or vdagent has not been
         // initialized.
         inside_call = 0;
         return 0;
@@ -931,7 +951,7 @@ void reds_handle_agent_mouse_event(const VDAgentMouseState *mouse_state)
     if (!inputs_inited()) {
         return;
     }
-    if (reds->mig_target || !(ring_item = ring_get_head(&reds->agent_state.internal_bufs))) {
+    if (!(ring_item = ring_get_head(&reds->agent_state.internal_bufs))) {
         reds->pending_mouse_event = TRUE;
         vdi_port_write_timer_start();
         return;
@@ -1346,7 +1366,6 @@ void reds_on_main_receive_migrate_data(MainMigrateData *data, uint8_t *end)
     reds_main_channel_restore_vdi_wqueue(data, pos, end);
     ASSERT(state->num_client_tokens + state->num_tokens == REDS_AGENT_WINDOW_SIZE);
 
-    reds->mig_target = FALSE;
     while (write_to_vdi_port() || read_from_vdi_port());
 }
 
@@ -1486,6 +1505,75 @@ int reds_expects_link_id(uint32_t connection_id)
     return 1;
 }
 
+static void reds_mig_target_client_add(RedClient *client)
+{
+    RedsMigTargetClient *mig_client;
+
+    ASSERT(reds);
+    red_printf("");
+    mig_client = spice_malloc0(sizeof(RedsMigTargetClient));
+    mig_client->client = client;
+    ring_init(&mig_client->pending_links);
+    ring_add(&reds->mig_target_clients, &mig_client->link);
+    reds->num_mig_target_clients++;
+
+}
+
+static RedsMigTargetClient* reds_mig_target_client_find(RedClient *client)
+{
+    RingItem *item;
+
+    RING_FOREACH(item, &reds->mig_target_clients) {
+        RedsMigTargetClient *mig_client;
+
+        mig_client = SPICE_CONTAINEROF(item, RedsMigTargetClient, link);
+        if (mig_client->client == client) {
+            return mig_client;
+        }
+    }
+    return NULL;
+}
+
+static void reds_mig_target_client_add_pending_link(RedsMigTargetClient *client,
+                                                    SpiceLinkMess *link_msg,
+                                                    RedsStream *stream)
+{
+    RedsMigPendingLink *mig_link;
+
+    ASSERT(reds);
+    ASSERT(client);
+    mig_link = spice_malloc0(sizeof(RedsMigPendingLink));
+    mig_link->link_msg = link_msg;
+    mig_link->stream = stream;
+
+    ring_add(&client->pending_links, &mig_link->ring_link);
+}
+
+static void reds_mig_target_client_free(RedsMigTargetClient *mig_client)
+{
+    RingItem *now, *next;
+
+    ring_remove(&mig_client->link);
+    reds->num_mig_target_clients--;
+
+    RING_FOREACH_SAFE(now, next, &mig_client->pending_links) {
+        RedsMigPendingLink *mig_link = SPICE_CONTAINEROF(now, RedsMigPendingLink, ring_link);
+        ring_remove(now);
+        free(mig_link);
+    }
+    free(mig_client);
+}
+
+static void reds_mig_target_client_disconnect_all()
+{
+    RingItem *now, *next;
+
+    RING_FOREACH_SAFE(now, next, &reds->mig_target_clients) {
+        RedsMigTargetClient *mig_client = SPICE_CONTAINEROF(now, RedsMigTargetClient, link);
+        reds_client_disconnect(mig_client->client);
+    }
+}
+
 // TODO: now that main is a separate channel this should
 // actually be joined with reds_handle_other_links, become reds_handle_link
 static void reds_handle_main_link(RedLinkInfo *link)
@@ -1496,6 +1584,7 @@ static void reds_handle_main_link(RedLinkInfo *link)
     uint32_t *caps;
     uint32_t connection_id;
     MainChannelClient *mcc;
+    int mig_target = FALSE;
 
     red_printf("");
     ASSERT(reds->main_channel);
@@ -1509,18 +1598,13 @@ static void reds_handle_main_link(RedLinkInfo *link)
         reds_send_link_result(link, SPICE_LINK_ERR_OK);
         while((connection_id = rand()) == 0);
         reds->agent_state.num_tokens = 0;
-        memcpy(&(reds->taTicket), &taTicket, sizeof(reds->taTicket));
-        reds->mig_target = FALSE;
+        mig_target = FALSE;
     } else {
-        // migration - check if this is one of the expected connection_id's
-        if (!reds_expects_link_id(link_mess->connection_id)) {
-            reds_send_link_result(link, SPICE_LINK_ERR_BAD_CONNECTION_ID);
-            reds_link_free(link);
-            return;
-        }
+        // TODO: make sure link_mess->connection_id is the same
+        // connection id the migration src had (use vmstate to store the connection id)
         reds_send_link_result(link, SPICE_LINK_ERR_OK);
         connection_id = link_mess->connection_id;
-        reds->mig_target = TRUE;
+        mig_target = TRUE;
     }
 
     reds->mig_inprogress = FALSE;
@@ -1534,11 +1618,11 @@ static void reds_handle_main_link(RedLinkInfo *link)
     link->link_mess = NULL;
     reds_link_free(link);
     caps = (uint32_t *)((uint8_t *)link_mess + link_mess->caps_offset);
-    client = red_client_new();
+    client = red_client_new(mig_target);
     ring_add(&reds->clients, &client->link);
     reds->num_clients++;
     mcc = main_channel_link(reds->main_channel, client,
-                            stream, connection_id, reds->mig_target,
+                            stream, connection_id, mig_target,
                             link_mess->num_common_caps,
                             link_mess->num_common_caps ? caps : NULL, link_mess->num_channel_caps,
                             link_mess->num_channel_caps ? caps + link_mess->num_common_caps : NULL);
@@ -1551,9 +1635,10 @@ static void reds_handle_main_link(RedLinkInfo *link)
         reds->agent_state.plug_generation++;
     }
 
-    if (!reds->mig_target) {
-        reds->agent_state.num_client_tokens = REDS_AGENT_WINDOW_SIZE;
-        main_channel_push_init(mcc, connection_id, red_dispatcher_count(),
+    reds->agent_state.num_client_tokens = REDS_AGENT_WINDOW_SIZE;
+
+    if (!mig_target) {
+        main_channel_push_init(mcc, red_dispatcher_count(),
             reds->mouse_mode, reds->is_client_mouse_allowed,
             reds_get_mm_time() - MM_TIME_DELTA,
             red_dispatcher_qxl_ram_size());
@@ -1561,6 +1646,8 @@ static void reds_handle_main_link(RedLinkInfo *link)
         main_channel_client_start_net_test(mcc);
         /* Now that we have a client, forward any pending agent data */
         while (read_from_vdi_port());
+    } else {
+        reds_mig_target_client_add(client);
     }
 }
 
@@ -1614,7 +1701,8 @@ static void reds_channel_do_link(RedChannel *channel, RedClient *client,
     }
 
     caps = (uint32_t *)((uint8_t *)link_msg + link_msg->caps_offset);
-    channel->client_cbs.connect(channel, client, stream, reds->mig_target,
+    channel->client_cbs.connect(channel, client, stream,
+                                red_client_during_migrate_at_target(client),
                                 link_msg->num_common_caps,
                                 link_msg->num_common_caps ? caps : NULL,
                                 link_msg->num_channel_caps,
@@ -1622,11 +1710,55 @@ static void reds_channel_do_link(RedChannel *channel, RedClient *client,
                                 caps + link_msg->num_common_caps : NULL);
 }
 
+void reds_on_client_migrate_complete(RedClient *client)
+{
+    RedsMigTargetClient *mig_client;
+    MainChannelClient *mcc;
+    RingItem *item;
+
+    red_printf("%p", client);
+    mcc = red_client_get_main(client);
+    mig_client = reds_mig_target_client_find(client);
+    if (!mig_client) {
+        red_printf("Error: mig target client was not found");
+        return;
+    }
+
+    // TODO: not doing net test. consider doing it on client_migrate_info
+    main_channel_push_init(mcc, red_dispatcher_count(),
+                           reds->mouse_mode, reds->is_client_mouse_allowed,
+                           reds_get_mm_time() - MM_TIME_DELTA,
+                           red_dispatcher_qxl_ram_size());
+
+    RING_FOREACH(item, &mig_client->pending_links) {
+        RedsMigPendingLink *mig_link;
+        RedChannel *channel;
+
+        mig_link = SPICE_CONTAINEROF(item, RedsMigPendingLink, ring_link);
+        channel = reds_find_channel(mig_link->link_msg->channel_type,
+                                    mig_link->link_msg->channel_id);
+        if (!channel) {
+            red_printf("warning: client %p channel (%d, %d) (type, id) wasn't found",
+                       client,
+                       mig_link->link_msg->channel_type,
+                       mig_link->link_msg->channel_id);
+            continue;
+        }
+        reds_channel_do_link(channel, client, mig_link->link_msg, mig_link->stream);
+    }
+
+    reds_mig_target_client_free(mig_client);
+
+    /* Now that we have a client, forward any pending agent data */
+    while (read_from_vdi_port());
+}
+
 static void reds_handle_other_links(RedLinkInfo *link)
 {
     RedChannel *channel;
     RedClient *client = NULL;
     SpiceLinkMess *link_mess;
+    RedsMigTargetClient *mig_client;
 
     link_mess = link->link_mess;
     if (reds->main_channel) {
@@ -1654,8 +1786,16 @@ static void reds_handle_other_links(RedLinkInfo *link)
     reds_send_link_result(link, SPICE_LINK_ERR_OK);
     reds_show_new_channel(link, link_mess->connection_id);
     reds_stream_remove_watch(link->stream);
-    reds_channel_do_link(channel, client, link_mess, link->stream);
-    free(link_mess);
+
+    mig_client = reds_mig_target_client_find(client);
+    if (red_client_during_migrate_at_target(client)) {
+        ASSERT(mig_client);
+        reds_mig_target_client_add_pending_link(mig_client, link_mess, link->stream);
+    } else {
+        ASSERT(!mig_client);
+        reds_channel_do_link(channel, client, link_mess, link->stream);
+        free(link_mess);
+    }
     link->stream = NULL;
     link->link_mess = NULL;
     reds_link_free(link);
@@ -1683,10 +1823,9 @@ static void reds_handle_ticket(void *opaque)
                         (unsigned char *)password, link->tiTicketing.rsa, RSA_PKCS1_OAEP_PADDING);
 
     if (ticketing_enabled) {
-        int expired = !link->link_mess->connection_id && taTicket.expiration_time < ltime;
-        char *actual_sever_pass = link->link_mess->connection_id ? reds->taTicket.password :
-                                                                   taTicket.password;
-        if (strlen(actual_sever_pass) == 0) {
+        int expired =  taTicket.expiration_time < ltime;
+
+        if (strlen(taTicket.password) == 0) {
             reds_send_link_result(link, SPICE_LINK_ERR_PERMISSION_DENIED);
             red_printf("Ticketing is enabled, but no password is set. "
                        "please set a ticket first");
@@ -1694,7 +1833,7 @@ static void reds_handle_ticket(void *opaque)
             return;
         }
 
-        if (expired || strncmp(password, actual_sever_pass, SPICE_MAX_PASSWORD_LENGTH) != 0) {
+        if (expired || strncmp(password, taTicket.password, SPICE_MAX_PASSWORD_LENGTH) != 0) {
             reds_send_link_result(link, SPICE_LINK_ERR_PERMISSION_DENIED);
             reds_link_free(link);
             return;
@@ -3049,7 +3188,10 @@ static void migrate_timeout(void *opaque)
     red_printf("");
     ASSERT(reds->mig_wait_connect || reds->mig_wait_disconnect);
     if (reds->mig_wait_connect) {
+        /* we will fall back to the switch host scheme when migration completes */
         main_channel_migrate_cancel_wait(reds->main_channel);
+        /* in case part of the client haven't yet completed the previous migration, disconnect them */
+        reds_mig_target_client_disconnect_all();
         reds_mig_cleanup();
     } else {
         reds_mig_disconnect();
@@ -3108,9 +3250,7 @@ static void attach_to_red_agent(SpiceCharDeviceInstance *sin)
     state->read_filter.discard_all = FALSE;
     reds->agent_state.plug_generation++;
 
-    if (!reds->mig_target) {
-        main_channel_push_agent_connected(reds->main_channel);
-    }
+    main_channel_push_agent_connected(reds->main_channel);
 }
 
 SPICE_GNUC_VISIBLE void spice_server_char_device_wakeup(SpiceCharDeviceInstance* sin)
@@ -3413,6 +3553,7 @@ static int do_spice_init(SpiceCoreInterface *core_interface)
     reds->num_clients = 0;
     main_dispatcher_init(core);
     ring_init(&reds->channels);
+    ring_init(&reds->mig_target_clients);
 
     if (!(reds->mig_timer = core->timer_add(migrate_timeout, NULL))) {
         red_error("migration timer create failed");
@@ -3847,7 +3988,7 @@ SPICE_GNUC_VISIBLE int spice_server_migrate_connect(SpiceServer *s, const char*
 
     reds->expect_migrate = TRUE;
 
-
+    /* main channel will take care of clients that are still during migration (at target)*/
     if (main_channel_migrate_connect(reds->main_channel, reds->mig_spice)) {
         reds_mig_started();
     } else {
diff --git a/server/reds.h b/server/reds.h
index 9feb9ab3..450825de 100644
--- a/server/reds.h
+++ b/server/reds.h
@@ -147,6 +147,6 @@ void reds_on_main_migrate_connected(void); //should be called when all the clien
                                            // are connected to the target
 void reds_on_main_receive_migrate_data(MainMigrateData *data, uint8_t *end);
 void reds_on_main_mouse_mode_request(void *message, size_t size);
-
+void reds_on_client_migrate_complete(RedClient *client);
 #endif