glusterd/mux: Optimize brick disconnect handler code

Removed unnecessary iteration during brick disconnect
handler when multiplex is enabled.

Change-Id: I62dd3337b7e7da085da5d76aaae206e0b0edff9f
fixes: bz#1650115
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>

4 files changed, 85 insertions, 140 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 1d81d99df1..ef1df3c378 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -6061,43 +6061,6 @@ out:
 
 static int gd_stale_rpc_disconnect_log;
 
-static int
-glusterd_mark_bricks_stopped_by_proc(glusterd_brick_proc_t *brick_proc)
-{
-    glusterd_brickinfo_t *brickinfo = NULL;
-    glusterd_brickinfo_t *brickinfo_tmp = NULL;
-    glusterd_volinfo_t *volinfo = NULL;
-    int ret = -1;
-
-    cds_list_for_each_entry(brickinfo, &brick_proc->bricks, brick_list)
-    {
-        ret = glusterd_get_volinfo_from_brick(brickinfo->path, &volinfo);
-        if (ret) {
-            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
-                   "Failed to get volinfo"
-                   " from brick(%s)",
-                   brickinfo->path);
-            goto out;
-        }
-        cds_list_for_each_entry(brickinfo_tmp, &volinfo->bricks, brick_list)
-        {
-            if (strcmp(brickinfo->path, brickinfo_tmp->path) == 0) {
-                glusterd_set_brick_status(brickinfo_tmp, GF_BRICK_STOPPED);
-                brickinfo_tmp->start_triggered = _gf_false;
-                /* When bricks are stopped, ports also need to
-                 * be cleaned up
-                 */
-                pmap_registry_remove(THIS, brickinfo_tmp->port,
-                                     brickinfo_tmp->path,
-                                     GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
-            }
-        }
-    }
-    return 0;
-out:
-    return ret;
-}
-
 int
 __glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
                             rpc_clnt_event_t event, void *data)
@@ -6108,7 +6071,6 @@ __glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
     glusterd_brickinfo_t *brickinfo = NULL;
     glusterd_volinfo_t *volinfo = NULL;
     xlator_t *this = NULL;
-    int brick_proc_found = 0;
     int32_t pid = -1;
     glusterd_brickinfo_t *brickinfo_tmp = NULL;
     glusterd_brick_proc_t *brick_proc = NULL;
@@ -6236,32 +6198,21 @@ __glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
             if (brickpath)
                 GF_FREE(brickpath);
 
-            if (is_brick_mx_enabled()) {
-                cds_list_for_each_entry(brick_proc, &conf->brick_procs,
-                                        brick_proc_list)
+            if (is_brick_mx_enabled() && glusterd_is_brick_started(brickinfo)) {
+                brick_proc = brickinfo->brick_proc;
+                if (!brick_proc)
+                    break;
+                cds_list_for_each_entry(brickinfo_tmp, &brick_proc->bricks,
+                                        mux_bricks)
                 {
-                    cds_list_for_each_entry(brickinfo_tmp, &brick_proc->bricks,
-                                            brick_list)
-                    {
-                        if (strcmp(brickinfo_tmp->path, brickinfo->path) == 0) {
-                            ret = glusterd_mark_bricks_stopped_by_proc(
-                                brick_proc);
-                            if (ret) {
-                                gf_msg(THIS->name, GF_LOG_ERROR, 0,
-                                       GD_MSG_BRICK_STOP_FAIL,
-                                       "Unable to stop "
-                                       "bricks of process"
-                                       " to which brick(%s)"
-                                       " belongs",
-                                       brickinfo->path);
-                                goto out;
-                            }
-                            brick_proc_found = 1;
-                            break;
-                        }
-                    }
-                    if (brick_proc_found == 1)
-                        break;
+                    glusterd_set_brick_status(brickinfo_tmp, GF_BRICK_STOPPED);
+                    brickinfo_tmp->start_triggered = _gf_false;
+                    /* When bricks are stopped, ports also need to
+                     * be cleaned up
+                     */
+                    pmap_registry_remove(
+                        THIS, brickinfo_tmp->port, brickinfo_tmp->path,
+                        GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
                 }
             } else {
                 glusterd_set_brick_status(brickinfo, GF_BRICK_STOPPED);
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index d9e0d6110a..ea3c4e9340 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1085,6 +1085,7 @@ glusterd_brickinfo_new(glusterd_brickinfo_t **brickinfo)
         goto out;
 
     CDS_INIT_LIST_HEAD(&new_brickinfo->brick_list);
+    CDS_INIT_LIST_HEAD(&new_brickinfo->mux_bricks);
     pthread_mutex_init(&new_brickinfo->restart_mutex, NULL);
     *brickinfo = new_brickinfo;
 
@@ -2049,6 +2050,7 @@ glusterd_volume_start_glusterfs(glusterd_volinfo_t *volinfo,
     rpc_clnt_connection_t *conn = NULL;
     int pid = -1;
     int32_t len = 0;
+    glusterd_brick_proc_t *brick_proc = NULL;
 
     GF_ASSERT(volinfo);
     GF_ASSERT(brickinfo);
@@ -2267,15 +2269,21 @@ retry:
         goto out;
     }
 
-    ret = glusterd_brick_process_add_brick(brickinfo);
+    ret = glusterd_brickprocess_new(&brick_proc);
     if (ret) {
-        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
-               "Adding brick %s:%s "
-               "to brick process failed.",
-               brickinfo->hostname, brickinfo->path);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPROC_NEW_FAILED,
+               "Failed to create "
+               "new brick process instance");
         goto out;
     }
 
+    brick_proc->port = brickinfo->port;
+    cds_list_add_tail(&brick_proc->brick_proc_list, &priv->brick_procs);
+    brickinfo->brick_proc = brick_proc;
+    cds_list_add_tail(&brickinfo->mux_bricks, &brick_proc->bricks);
+    brickinfo->brick_proc = brick_proc;
+    brick_proc->brick_count++;
+
 connect:
     ret = glusterd_brick_connect(volinfo, brickinfo, socketpath);
     if (ret) {
@@ -2410,9 +2418,6 @@ glusterd_brick_process_remove_brick(glusterd_brickinfo_t *brickinfo,
     xlator_t *this = NULL;
     glusterd_conf_t *priv = NULL;
     glusterd_brick_proc_t *brick_proc = NULL;
-    glusterd_brickinfo_t *brickinfoiter = NULL;
-    glusterd_brick_proc_t *brick_proc_tmp = NULL;
-    glusterd_brickinfo_t *tmp = NULL;
 
     this = THIS;
     GF_VALIDATE_OR_GOTO("glusterd", this, out);
@@ -2421,52 +2426,45 @@ glusterd_brick_process_remove_brick(glusterd_brickinfo_t *brickinfo,
     GF_VALIDATE_OR_GOTO(this->name, priv, out);
     GF_VALIDATE_OR_GOTO(this->name, brickinfo, out);
 
-    cds_list_for_each_entry_safe(brick_proc, brick_proc_tmp, &priv->brick_procs,
-                                 brick_proc_list)
-    {
-        if (brickinfo->port != brick_proc->port) {
-            continue;
+    brick_proc = brickinfo->brick_proc;
+    if (!brick_proc) {
+        if (brickinfo->status != GF_BRICK_STARTED) {
+            /* this function will be called from gluster_pmap_signout and
+             * glusterd_volume_stop_glusterfs. So it is possible to have
+             * brick_proc set as null.
+             */
+            ret = 0;
         }
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO(this->name, (brick_proc->brick_count > 0), out);
-
-        cds_list_for_each_entry_safe(brickinfoiter, tmp, &brick_proc->bricks,
-                                     brick_list)
-        {
-            if (strcmp(brickinfoiter->path, brickinfo->path) == 0) {
-                cds_list_del_init(&brickinfoiter->brick_list);
+    GF_VALIDATE_OR_GOTO(this->name, (brick_proc->brick_count > 0), out);
 
-                GF_FREE(brickinfoiter->logfile);
-                GF_FREE(brickinfoiter);
-                brick_proc->brick_count--;
-                break;
-            }
-        }
+    cds_list_del_init(&brickinfo->mux_bricks);
+    brick_proc->brick_count--;
 
-        /* If all bricks have been removed, delete the brick process */
-        if (brick_proc->brick_count == 0) {
-            if (last_brick != NULL)
-                *last_brick = 1;
-            ret = glusterd_brickprocess_delete(brick_proc);
-            if (ret)
-                goto out;
-        }
-        break;
+    /* If all bricks have been removed, delete the brick process */
+    if (brick_proc->brick_count == 0) {
+        if (last_brick != NULL)
+            *last_brick = 1;
+        ret = glusterd_brickprocess_delete(brick_proc);
+        if (ret)
+            goto out;
     }
-
+    brickinfo->brick_proc = NULL;
     ret = 0;
 out:
     return ret;
 }
 
 int
-glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo)
+glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo,
+                                 glusterd_brickinfo_t *parent_brickinfo)
 {
     int ret = -1;
     xlator_t *this = NULL;
     glusterd_conf_t *priv = NULL;
     glusterd_brick_proc_t *brick_proc = NULL;
-    glusterd_brickinfo_t *brickinfo_dup = NULL;
 
     this = THIS;
     GF_VALIDATE_OR_GOTO("glusterd", this, out);
@@ -2475,36 +2473,28 @@ glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo)
     GF_VALIDATE_OR_GOTO(this->name, priv, out);
     GF_VALIDATE_OR_GOTO(this->name, brickinfo, out);
 
-    ret = glusterd_brickinfo_new(&brickinfo_dup);
-    if (ret) {
-        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_NEW_INFO_FAIL,
-               "Failed to create new brickinfo");
-        goto out;
-    }
-
-    ret = glusterd_brickinfo_dup(brickinfo, brickinfo_dup);
-    if (ret) {
-        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
-               "Failed to dup brickinfo");
-        goto out;
-    }
-
-    ret = glusterd_brick_proc_for_port(brickinfo->port, &brick_proc);
-    if (ret) {
-        ret = glusterd_brickprocess_new(&brick_proc);
+    if (!parent_brickinfo) {
+        ret = glusterd_brick_proc_for_port(brickinfo->port, &brick_proc);
         if (ret) {
-            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPROC_NEW_FAILED,
-                   "Failed to create "
-                   "new brick process instance");
-            goto out;
-        }
+            ret = glusterd_brickprocess_new(&brick_proc);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPROC_NEW_FAILED,
+                       "Failed to create "
+                       "new brick process instance");
+                goto out;
+            }
 
-        brick_proc->port = brickinfo->port;
+            brick_proc->port = brickinfo->port;
 
-        cds_list_add_tail(&brick_proc->brick_proc_list, &priv->brick_procs);
+            cds_list_add_tail(&brick_proc->brick_proc_list, &priv->brick_procs);
+        }
+    } else {
+        ret = 0;
+        brick_proc = parent_brickinfo->brick_proc;
     }
 
-    cds_list_add_tail(&brickinfo_dup->brick_list, &brick_proc->bricks);
+    cds_list_add_tail(&brickinfo->mux_bricks, &brick_proc->bricks);
+    brickinfo->brick_proc = brick_proc;
     brick_proc->brick_count++;
 out:
     return ret;
@@ -2637,6 +2627,7 @@ glusterd_volume_stop_glusterfs(glusterd_volinfo_t *volinfo,
 
     brickinfo->status = GF_BRICK_STOPPED;
     brickinfo->start_triggered = _gf_false;
+    brickinfo->brick_proc = NULL;
     if (del_brick)
         glusterd_delete_brick(volinfo, brickinfo);
 out:
@@ -5701,7 +5692,7 @@ attach_brick(xlator_t *this, glusterd_brickinfo_t *brickinfo,
                     goto out;
                 }
                 brickinfo->port = other_brick->port;
-                ret = glusterd_brick_process_add_brick(brickinfo);
+                ret = glusterd_brick_process_add_brick(brickinfo, other_brick);
                 if (ret) {
                     gf_msg(this->name, GF_LOG_ERROR, 0,
                            GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
@@ -6241,7 +6232,7 @@ glusterd_brick_start(glusterd_volinfo_t *volinfo,
 
             (void)glusterd_brick_connect(volinfo, brickinfo, socketpath);
 
-            ret = glusterd_brick_process_add_brick(brickinfo);
+            ret = glusterd_brick_process_add_brick(brickinfo, NULL);
             if (ret) {
                 gf_msg(this->name, GF_LOG_ERROR, 0,
                        GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index a00dc4564a..36f828f109 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -181,7 +181,8 @@ int32_t
 glusterd_resolve_brick(glusterd_brickinfo_t *brickinfo);
 
 int
-glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo);
+glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo,
+                                 glusterd_brickinfo_t *parent_brickinfo);
 
 int
 glusterd_brick_process_remove_brick(glusterd_brickinfo_t *brickinfo,
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 3b681ef954..b0a7d9a448 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -224,6 +224,17 @@ typedef enum gf_brick_status {
     GF_BRICK_STARTING
 } gf_brick_status_t;
 
+typedef struct glusterd_brickinfo glusterd_brickinfo_t;
+
+struct glusterd_brick_proc {
+    int port;
+    uint32_t brick_count;
+    struct cds_list_head brick_proc_list;
+    struct cds_list_head bricks;
+};
+
+typedef struct glusterd_brick_proc glusterd_brick_proc_t;
+
 struct glusterd_brickinfo {
     char hostname[NAME_MAX];
     char path[VALID_GLUSTERD_PATHMAX];
@@ -262,19 +273,10 @@ struct glusterd_brickinfo {
     gf_boolean_t port_registered;
     gf_boolean_t start_triggered;
     pthread_mutex_t restart_mutex;
+    glusterd_brick_proc_t *brick_proc; /* Information regarding mux bricks */
+    struct cds_list_head mux_bricks; /* List to store the bricks in brick_proc*/
 };
 
-typedef struct glusterd_brickinfo glusterd_brickinfo_t;
-
-struct glusterd_brick_proc {
-    int port;
-    uint32_t brick_count;
-    struct cds_list_head brick_proc_list;
-    struct cds_list_head bricks;
-};
-
-typedef struct glusterd_brick_proc glusterd_brick_proc_t;
-
 struct glusterd_gfproxyd_info {
     short port;
     char *logfile;