glusterd/snapshot: Execute lvm snapshots in parallel

Back-end LVM Snapshot is executed parallely as synop task
This helps is gaining performance when there are more bricks in a
node.

This patch also removes unwanted logs printed in snapshot cleanup

Change-Id: I3174cb4547ebb670eca37a98eb9d75ecb0672a90
BUG: 1061685
Signed-off-by: Vijaikumar M <vmallika@redhat.com>
Reviewed-on: http://review.gluster.org/7461
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Krishnan Parthasarathi <kparthas@redhat.com>

5 files changed, 248 insertions, 73 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index e6f6a0333a..95908cf62f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -69,7 +69,8 @@ typedef enum gf_gld_mem_types_ {
         gf_gld_mt_int                           = gf_common_mt_end + 53,
         gf_gld_mt_snap_t                        = gf_common_mt_end + 54,
         gf_gld_mt_missed_snapinfo_t             = gf_common_mt_end + 55,
-        gf_gld_mt_end                           = gf_common_mt_end + 56,
+        gf_gld_mt_snap_create_args_t            = gf_common_mt_end + 56,
+        gf_gld_mt_end                           = gf_common_mt_end + 57,
 } gf_gld_mem_types_t;
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index e52809f3e8..65b0fb12cd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -63,6 +63,18 @@
 #include "lvm-defaults.h"
 
 char snap_mount_folder[PATH_MAX];
+struct snap_create_args_ {
+        xlator_t             *this;
+        dict_t               *dict;
+        dict_t               *rsp_dict;
+        glusterd_volinfo_t   *snap_vol;
+        glusterd_brickinfo_t *brickinfo;
+        struct syncargs      *args;
+        int32_t               volcount;
+        int32_t               brickcount;
+        int32_t               brickorder;
+};
+typedef struct snap_create_args_ snap_create_args_t;
 
 /* This function is called to get the device path of the snap lvm. Usually
    if /dev/mapper/<group-name>-<lvm-name> is the device for the lvm,
@@ -1921,13 +1933,14 @@ out:
 int32_t
 glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
 {
-        char                 *mnt_pt        = NULL;
-        struct mntent        *entry         = NULL;
-        int32_t               brick_count   = -1;
-        int32_t               ret           = -1;
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        xlator_t             *this          = NULL;
-        FILE                 *mtab          = NULL;
+        char                 *mnt_pt            = NULL;
+        struct mntent        *entry             = NULL;
+        struct mntent         save_entry        = {0,};
+        int32_t               brick_count       = -1;
+        int32_t               ret               = -1;
+        glusterd_brickinfo_t *brickinfo         = NULL;
+        xlator_t             *this              = NULL;
+        char                  buff [PATH_MAX]   = "";
 
         this = THIS;
         GF_ASSERT (this);
@@ -1988,7 +2001,8 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
                         goto out;
                 }
 
-                entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+                entry = glusterd_get_mnt_entry_info (mnt_pt, buff,
+                                                    sizeof (buff), &save_entry);
                 if (!entry) {
                         gf_log (this->name, GF_LOG_WARNING, "getting the mount"
                                 " entry for the brick %s:%s of the snap %s "
@@ -2001,8 +2015,6 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
                 ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo,
                                                        mnt_pt,
                                                        entry->mnt_fsname);
-                if (mtab)
-                        endmntent (mtab);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "failed to "
                                 "remove the snapshot %s (%s)",
@@ -2052,8 +2064,8 @@ glusterd_snap_volume_remove (dict_t *rsp_dict,
                                "brick for volume %s", snap_vol->volname);
                         save_ret = ret;
 
-                        /* Continue to cleaning up the snap in case of error
-                           if force flag is enabled */
+                        /* Don't clean up the snap on error when
+                           force flag is disabled */
                         if (!force)
                                 goto out;
                 }
@@ -2156,16 +2168,14 @@ glusterd_snap_remove (dict_t *rsp_dict,
         list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
                 ret = glusterd_snap_volume_remove (rsp_dict, snap_vol,
                                                    remove_lvm, force);
-                if (ret) {
+                if (ret && !force) {
+                        /* Don't clean up the snap on error when
+                           force flag is disabled */
                         gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
                                "volinfo %s for snap %s", snap_vol->volname,
                                snap->snapname);
                         save_ret = ret;
-
-                        /* Continue to cleaning up the snap in case of error
-                           if force flag is enabled */
-                        if (!force)
-                                goto out;
+                        goto out;
                 }
         }
 
@@ -4619,9 +4629,11 @@ glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 
         ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "removing the snap %s failed",
+                /* Ignore failure as this is a cleanup of half cooked
+                   snapshot */
+                gf_log (this->name, GF_LOG_DEBUG, "removing the snap %s failed",
                         name);
-                goto out;
+                ret = 0;
         }
 
         name = NULL;
@@ -4676,6 +4688,175 @@ out:
         return ret;
 }
 
+int
+glusterd_take_brick_snapshot_task (void *opaque)
+{
+        int ret                             = 0;
+        snap_create_args_t  *snap_args      = NULL;
+        char                 key[PATH_MAX]  = "";
+
+        GF_ASSERT (opaque);
+
+        snap_args = (snap_create_args_t*) opaque;
+        THIS = snap_args->this;
+
+        ret = glusterd_take_brick_snapshot (snap_args->dict,
+                                            snap_args->snap_vol,
+                                            snap_args->brickinfo,
+                                            snap_args->volcount,
+                                            snap_args->brickorder);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to "
+                      "take backend snapshot for brick "
+                      "%s:%s volume(%s)", snap_args->brickinfo->hostname,
+                      snap_args->brickinfo->path, snap_args->snap_vol->volname);
+        }
+
+        snprintf (key, sizeof (key), "snap-vol%d.brick%d.status",
+                  snap_args->volcount, snap_args->brickorder);
+        if (dict_set_int32 (snap_args->rsp_dict, key, (ret)?0:1)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to "
+                        "add %s to dict", key);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_take_brick_snapshot_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        snap_create_args_t  *snap_args = NULL;
+        struct syncargs     *args      = NULL;
+
+        GF_ASSERT (opaque);
+
+        snap_args = (snap_create_args_t*) opaque;
+        args  = snap_args->args;
+
+        if (ret)
+                args->op_ret = ret;
+
+        GF_FREE (opaque);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+glusterd_schedule_brick_snapshot (dict_t *dict, dict_t *rsp_dict,
+                                  glusterd_snap_t *snap)
+{
+        int                     ret             = -1;
+        int32_t                 volcount        = 0;
+        int32_t                 brickcount      = 0;
+        int32_t                 brickorder      = 0;
+        int32_t                 taskcount       = 0;
+        char                    key[PATH_MAX]   = "";
+        xlator_t               *this            = NULL;
+        glusterd_volinfo_t     *snap_vol        = NULL;
+        glusterd_brickinfo_t   *brickinfo       = NULL;
+        struct syncargs         args            = {0};
+        snap_create_args_t     *snap_args       = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+        GF_ASSERT(snap);
+
+        synctask_barrier_init ((&args));
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                volcount++;
+                brickcount = 0;
+                brickorder = 0;
+                list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                        snprintf (key, sizeof(key) - 1,
+                                  "snap-vol%d.brick%d.order", volcount,
+                                  brickcount);
+                        ret = dict_set_int32 (rsp_dict, key, brickorder);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                goto out;
+                        }
+
+                        if ((uuid_compare (brickinfo->uuid, MY_UUID)) ||
+                            (brickinfo->snap_status == -1)) {
+                                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                                        brickcount++;
+                                        snprintf (key, sizeof (key),
+                                                  "snap-vol%d.brick%d.status",
+                                                  volcount, brickorder);
+                                        ret = dict_set_int32 (rsp_dict, key, 0);
+                                        if (ret) {
+                                                gf_log (this->name,
+                                                        GF_LOG_ERROR,
+                                                        "failed to add %s to "
+                                                        "dict", key);
+                                                goto out;
+                                        }
+                                }
+                                brickorder++;
+                                continue;
+                        }
+
+                        snap_args = GF_CALLOC (1, sizeof (*snap_args),
+                                               gf_gld_mt_snap_create_args_t);
+                        if (!snap_args) {
+                                ret = -1;
+                                goto out;
+                        }
+
+
+                        snap_args->this = this;
+                        snap_args->dict = dict;
+                        snap_args->rsp_dict = rsp_dict;
+                        snap_args->snap_vol = snap_vol;
+                        snap_args->brickinfo = brickinfo;
+                        snap_args->volcount = volcount;
+                        snap_args->brickcount = brickcount;
+                        snap_args->brickorder = brickorder;
+                        snap_args->args = &args;
+
+                        ret = synctask_new (this->ctx->env,
+                                            glusterd_take_brick_snapshot_task,
+                                            glusterd_take_brick_snapshot_cbk,
+                                            NULL, snap_args);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "spawn task for snapshot create");
+                                GF_FREE (snap_args);
+                                goto out;
+                        }
+                        taskcount++;
+                        brickcount++;
+                        brickorder++;
+                }
+
+                snprintf (key, sizeof (key), "snap-vol%d_brickcount", volcount);
+                ret = dict_set_int64 (rsp_dict, key, brickcount);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "add %s to dict", key);
+                        goto out;
+                }
+        }
+        synctask_barrier_wait ((&args), taskcount);
+        taskcount = 0;
+
+        if (args.op_ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create snapshot");
+
+        ret = args.op_ret;
+out:
+        if (ret && taskcount)
+                synctask_barrier_wait ((&args), taskcount);
+
+        return ret;
+}
+
 int32_t
 glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
                                  dict_t *rsp_dict)
@@ -4683,7 +4864,6 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
         int                     ret                     = -1;
         int64_t                 i                       = 0;
         int64_t                 volcount                = 0;
-        int32_t                 brickcount              = 0;
         char                    *snapname               = NULL;
         char                    *volname                = NULL;
         char                    *tmp_name               = NULL;
@@ -4692,7 +4872,6 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
         glusterd_snap_t         *snap                   = NULL;
         glusterd_volinfo_t      *origin_vol             = NULL;
         glusterd_volinfo_t      *snap_vol               = NULL;
-        glusterd_brickinfo_t    *brickinfo              = NULL;
         glusterd_conf_t         *priv                   = NULL;
 
         this = THIS;
@@ -4756,9 +4935,6 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
                         goto out;
                 }
 
-                /* TODO: Create a stub where the bricks are
-                   added parallely by worker threads so that
-                   the snap creating happens parallely.  */
                 snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
                                                  rsp_dict, i);
                 if (!snap_vol) {
@@ -4768,32 +4944,20 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
                         goto out;
                 }
         }
+        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
 
-        volcount=0;
-        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                volcount++;
-                brickcount = 0;
-                list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                        if ((uuid_compare (brickinfo->uuid, MY_UUID)) ||
-                            (brickinfo->snap_status == -1)) {
-                                brickcount++;
-                                continue;
-                        }
-                        ret = glusterd_take_brick_snapshot (dict,
-                                                            snap_vol,
-                                                            brickinfo,
-                                                            volcount,
-                                                            brickcount);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "Failed to "
-                                        "take backend snapshot for brick "
-                                        "%s:%s volume(%s)", brickinfo->hostname,
-                                        brickinfo->path, snap_vol->volname);
-                                        goto out;
-                        }
-                        brickcount++;
-                }
+        ret = glusterd_schedule_brick_snapshot (dict, rsp_dict, snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to take backend "
+                        "snapshot %s", snap->snapname);
+                goto out;
+        }
 
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
                 snap_vol->status = GLUSTERD_STATUS_STARTED;
                 ret = glusterd_store_volinfo (snap_vol,
                                              GLUSTERD_VOLINFO_VER_AC_INCREMENT);
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 5b1d80e84a..ca6d945482 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -2897,9 +2897,10 @@ out:
 int32_t
 glusterd_mount_brick_paths (char *brick_mount_path, char *device_path)
 {
-        FILE                    *mtab                = NULL;
         int32_t                  ret                 = -1;
         runner_t                 runner              = {0, };
+        char                     buff [PATH_MAX]     = {0, };
+        struct mntent            save_entry          = {0, };
         struct mntent           *entry               = NULL;
         xlator_t                *this                = NULL;
         glusterd_conf_t         *priv                = NULL;
@@ -2913,7 +2914,8 @@ glusterd_mount_brick_paths (char *brick_mount_path, char *device_path)
         GF_ASSERT (priv);
 
         /* Check if the brick_mount_path is already mounted */
-        entry = glusterd_get_mnt_entry_info (brick_mount_path, mtab);
+        entry = glusterd_get_mnt_entry_info (brick_mount_path, buff,
+                                             sizeof (buff), &save_entry);
         if (entry) {
                 gf_log (this->name, GF_LOG_INFO,
                         "brick_mount_path (%s) already mounted.",
@@ -2949,8 +2951,6 @@ glusterd_mount_brick_paths (char *brick_mount_path, char *device_path)
         }
 
 out:
-        if (mtab)
-                endmntent (mtab);
         gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index dd4f84265d..1134d29dc5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -6981,15 +6981,21 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count)
 }
 
 struct mntent *
-glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab)
+glusterd_get_mnt_entry_info (char *mnt_pt, char *buff, int buflen,
+                             struct mntent *entry_ptr)
 {
-        struct mntent  *entry                = NULL;
+        struct mntent  *entry   = NULL;
+        FILE           *mtab    = NULL;
+
+        GF_ASSERT (mnt_pt);
+        GF_ASSERT (buff);
+        GF_ASSERT (entry_ptr);
 
         mtab = setmntent (_PATH_MOUNTED, "r");
         if (!mtab)
                 goto out;
 
-        entry = getmntent (mtab);
+        entry = getmntent_r (mtab, entry_ptr, buff, buflen);
 
         while (1) {
                 if (!entry)
@@ -6998,10 +7004,13 @@ glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab)
                 if (!strcmp (entry->mnt_dir, mnt_pt) &&
                     strcmp (entry->mnt_type, "rootfs"))
                         break;
-                entry = getmntent (mtab);
+                entry = getmntent_r (mtab, entry_ptr, buff, buflen);
         }
 
 out:
+        if (NULL != mtab) {
+                endmntent (mtab);
+        }
         return entry;
 }
 
@@ -7011,9 +7020,10 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
 {
         int             ret                  = -1;
         char            key[1024]            = {0};
+        char            buff [PATH_MAX]      = {0};
         char            base_key[1024]       = {0};
+        struct mntent   save_entry           = {0};
         char           *mnt_pt               = NULL;
-        FILE           *mtab                 = NULL;
         struct mntent  *entry                = NULL;
 
         snprintf (base_key, sizeof (base_key), "brick%d", count);
@@ -7022,7 +7032,8 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
         if (ret)
                 goto out;
 
-        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff),
+                                             &save_entry);
         if (!entry) {
                 ret = -1;
                 goto out;
@@ -7052,8 +7063,6 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
 
  out:
         GF_FREE (mnt_pt);
-        if (mtab)
-                endmntent (mtab);
 
         return ret;
 }
@@ -7064,8 +7073,9 @@ glusterd_get_brick_mount_device (char *brick_path)
         int             ret                  = -1;
         char           *mnt_pt               = NULL;
         char           *device               = NULL;
-        FILE           *mtab                 = NULL;
+        char            buff [PATH_MAX]      = "";
         struct mntent  *entry                = NULL;
+        struct mntent   save_entry           = {0,};
         xlator_t       *this                 = NULL;
 
         this = THIS;
@@ -7079,7 +7089,8 @@ glusterd_get_brick_mount_device (char *brick_path)
                 goto out;
         }
 
-        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff),
+                                             &save_entry);
         if (NULL == entry) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to get mnt entry "
                         "for %s mount path", mnt_pt);
@@ -7090,10 +7101,6 @@ glusterd_get_brick_mount_device (char *brick_path)
         device = gf_strdup (entry->mnt_fsname);
 
 out:
-        if (NULL != mtab) {
-                endmntent (mtab);
-        }
-
         return device;
 }
 
@@ -10362,7 +10369,8 @@ glusterd_merge_brick_status (dict_t *dst, dict_t *src)
                 ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
                 if (ret) {
                         gf_log (this->name, GF_LOG_TRACE,
-                                "No bricks for this volume in this dict");
+                                "No bricks for this volume in this dict (%s)",
+                                snapbrckcnt);
                         continue;
                 }
 
@@ -10374,7 +10382,8 @@ glusterd_merge_brick_status (dict_t *dst, dict_t *src)
                         ret = dict_get_int64 (src, snapbrckord, &brick_order);
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get brick order");
+                                        "Failed to get brick order (%s)",
+                                        snapbrckord);
                                 goto out;
                         }
 
@@ -10384,14 +10393,14 @@ glusterd_merge_brick_status (dict_t *dst, dict_t *src)
                         ret = dict_get_int32 (src, key, &brick_online);
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "get the brick status");
+                                        "get the brick status (%s)", key);
                                 goto out;
                         }
 
                         ret = dict_set_int32 (dst, key, brick_online);
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set the brick status");
+                                        "set the brick status (%s)", key);
                                 goto out;
                         }
                         brick_online = 0;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 14f6ce4d08..122969ec25 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -696,7 +696,8 @@ char*
 glusterd_get_brick_mount_device (char *brick_path);
 
 struct mntent *
-glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab);
+glusterd_get_mnt_entry_info (char *mnt_pt, char *buff, int buflen,
+                             struct mntent *entry_ptr);
 
 int
 glusterd_get_brick_root (char *path, char **mount_point);