gfproxy: Introduce new server-side daemon called GFProxy

Summmary:
Adds a new server-side daemon called gfproxyd & a new FUSE client
called gfproxy-client

Updates: #242
BUG: 1428063
Change-Id: I83210098d3a381922bc64fed1110ae1b76e6519f
Tested-by: Shreyas Siravara <sshreyas@fb.com>
Reviewed-by: Kevin Vigor <kvigor@fb.com>
Signed-off-by: Shreyas Siravara <sshreyas@fb.com>
Signed-off-by: Poornima G <pgurusid@redhat.com>

7 files changed, 360 insertions, 21 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index c7e419c569..8c39732526 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -264,6 +264,50 @@ build_volfile_path (char *volume_id, char *path,
 
         }
 
+        volid_ptr = strstr (volume_id, "gfproxy-client/");
+        if (volid_ptr) {
+                volid_ptr = strchr (volid_ptr, '/');
+                if (!volid_ptr) {
+                        ret = -1;
+                        goto out;
+                }
+                volid_ptr++;
+
+                ret = glusterd_volinfo_find (volid_ptr, &volinfo);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Couldn't find volinfo");
+                        goto out;
+                }
+
+                glusterd_get_gfproxy_client_volfile (volinfo, path, path_len);
+
+                ret = 0;
+                goto out;
+        }
+
+        volid_ptr = strstr (volume_id, "gfproxy/");
+        if (volid_ptr) {
+                volid_ptr = strchr (volid_ptr, '/');
+                if (!volid_ptr) {
+                        ret = -1;
+                        goto out;
+                }
+                volid_ptr++;
+
+                ret = glusterd_volinfo_find (volid_ptr, &volinfo);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Couldn't find volinfo");
+                        goto out;
+                }
+
+                glusterd_get_gfproxyd_volfile (volinfo, path, path_len);
+
+                ret = 0;
+                goto out;
+        }
+
         volid_ptr = strstr (volume_id, "/snaps/");
         if (volid_ptr) {
                 ret = get_snap_volname_and_volinfo (volid_ptr, &volname,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index fd70516a28..623a3841f8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -12516,6 +12516,45 @@ out:
 }
 
 void
+glusterd_get_gfproxy_client_volfile (glusterd_volinfo_t *volinfo,
+                                        char *path, int path_len)
+{
+        char                    workdir[PATH_MAX]      = {0, };
+        glusterd_conf_t        *priv                    = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+
+        switch (volinfo->transport_type) {
+        case GF_TRANSPORT_TCP:
+                snprintf (path, path_len,
+                                "%s/trusted-%s.tcp-gfproxy-fuse.vol",
+                                workdir, volinfo->volname);
+                break;
+
+        case GF_TRANSPORT_RDMA:
+                snprintf (path, path_len,
+                                "%s/trusted-%s.rdma-gfproxy-fuse.vol",
+                                workdir, volinfo->volname);
+                break;
+        default:
+                break;
+        }
+}
+
+void
+glusterd_get_gfproxyd_volfile (glusterd_volinfo_t *volinfo,
+                                char *path, int path_len)
+{
+        char                    workdir[PATH_MAX]      = {0, };
+        glusterd_conf_t        *priv                    = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+
+        snprintf (path, path_len, "%s/%s.gfproxyd.vol", workdir,
+                  volinfo->volname);
+}
+
+void
 glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo,
                                 char *path, int path_len)
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 65ea7565d3..7e94c94ffc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -738,6 +738,14 @@ void
 glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo,
                                 char *path, int path_len);
 
+void
+glusterd_get_gfproxy_client_volfile (glusterd_volinfo_t *volinfo,
+                                     char *path, int path_len);
+
+void
+glusterd_get_gfproxyd_volfile (glusterd_volinfo_t *volinfo,
+                               char *path, int path_len);
+
 int32_t
 glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo,
                         glusterd_brickinfo_t *dup_brickinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index f3d4d08bf0..93b6e95d5f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -58,6 +58,20 @@ extern struct volopt_map_entry glusterd_volopt_map[];
         }                                                               \
 } while (0 /* CONSTCOND */)
 
+/**
+ * Needed for GFProxy
+ */
+#define GF_PROXY_DAEMON_PORT 40000
+#define GF_PROXY_DAEMON_PORT_STR "40000"
+
+static int
+volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                            dict_t *set_dict, void *param);
+
+static int
+build_client_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *mod_dict);
+
 /*********************************************
  *
  * xlator generation / graph manipulation API
@@ -1467,6 +1481,75 @@ server_spec_extended_option_handler (volgen_graph_t *graph,
 static void get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo);
 
 static int
+gfproxy_server_graph_builder (volgen_graph_t *graph,
+                                glusterd_volinfo_t *volinfo,
+                                dict_t *set_dict, void *param)
+{
+        xlator_t        *xl             = NULL;
+        /*char            *value          = NULL;*/
+        char            transt[16]      = {0, };
+        char            key[1024]       = {0, };
+        /*char            port_str[7]     = {0, };*/
+        int             ret             = 0;
+        char            *username       = NULL;
+        char            *password       = NULL;
+        /*int             rclusters       = 0;*/
+
+        /* We are a trusted client */
+        ret = dict_set_uint32 (set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+        if (ret != 0)
+                goto out;
+
+        ret = dict_set_str (set_dict, "gfproxy-server", "on");
+        if (ret != 0)
+                goto out;
+
+        /* Build the client section of the graph first */
+        build_client_graph (graph, volinfo, set_dict);
+
+        /* Clear this setting so that future users of set_dict do not end up
+         * thinking they are a gfproxy server */
+        dict_del (set_dict, "gfproxy-server");
+        dict_del (set_dict, "trusted-client");
+
+        /* Then add the server to it */
+        get_vol_transport_type (volinfo, transt);
+        xl = volgen_graph_add (graph, "protocol/server", volinfo->volname);
+        if (!xl)
+                goto out;
+
+        ret = xlator_set_option (xl, "listen-port", GF_PROXY_DAEMON_PORT_STR);
+        if (ret != 0)
+                goto out;
+
+        ret = xlator_set_option (xl, "transport-type", transt);
+        if (ret != 0)
+                goto out;
+
+        /* Set username and password */
+        username = glusterd_auth_get_username (volinfo);
+        password = glusterd_auth_get_password (volinfo);
+        if (username) {
+                snprintf (key, sizeof (key), "auth.login.%s-server.allow",
+                                volinfo->volname);
+                ret = xlator_set_option (xl, key, username);
+                if (ret)
+                        return -1;
+        }
+
+        if (password) {
+                snprintf (key, sizeof (key), "auth.login.%s.password",
+                                username);
+                ret = xlator_set_option (xl, key, password);
+                if (ret != 0)
+                        goto out;
+        }
+
+out:
+        return ret;
+}
+
+static int
 brick_graph_add_posix (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
@@ -2719,6 +2802,42 @@ out:
 }
 
 static int
+gfproxy_server_perfxl_option_handler (volgen_graph_t *graph,
+                                        struct volopt_map_entry *vme,
+                                        void *param)
+{
+
+        GF_ASSERT (param);
+
+        /* write-behind is the *not* allowed for gfproxy-servers */
+        if (strstr (vme->key, "write-behind")) {
+                return 0;
+        }
+
+        perfxl_option_handler (graph, vme, param);
+
+        return 0;
+}
+
+static int
+gfproxy_client_perfxl_option_handler (volgen_graph_t *graph,
+                                        struct volopt_map_entry *vme,
+                                        void *param)
+{
+
+        GF_ASSERT (param);
+
+        /* write-behind is the only allowed "perf" for gfproxy-clients */
+        if (!strstr (vme->key, "write-behind"))
+                return 0;
+
+        perfxl_option_handler (graph, vme, param);
+
+        return 0;
+}
+
+
+static int
 nfsperfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
                        void *param)
 {
@@ -2946,8 +3065,10 @@ _free_xlator_opt_key (char *key)
 }
 
 static xlator_t *
-volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                           char *hostname, char *subvol, char *xl_id,
+volgen_graph_build_client (volgen_graph_t *graph,
+                           glusterd_volinfo_t *volinfo,
+                           char *hostname, char *port,
+                           char *subvol, char *xl_id,
                            char *transt, dict_t *set_dict)
 {
         xlator_t                *xl                 = NULL;
@@ -2978,6 +3099,12 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         goto err;
         }
 
+        if (port) {
+                ret = xlator_set_option (xl, "remote-port", port);
+                if (ret)
+                        goto err;
+        }
+
         ret = xlator_set_option (xl, "remote-subvolume", subvol);
         if (ret)
                 goto err;
@@ -3001,7 +3128,8 @@ volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         ret = dict_get_uint32 (set_dict, "trusted-client",
                                &client_type);
 
-        if (!ret && client_type == GF_CLIENT_TRUSTED) {
+        if (!ret && (client_type == GF_CLIENT_TRUSTED
+              || client_type == GF_CLIENT_TRUSTED_PROXY)) {
                 str = NULL;
                 str = glusterd_auth_get_username (volinfo);
                 if (str) {
@@ -3085,7 +3213,9 @@ volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         i = 0;
         cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
                 xl = volgen_graph_build_client (graph, volinfo,
-                                                brick->hostname, brick->path,
+                                                brick->hostname,
+                                                NULL,
+                                                brick->path,
                                                 brick->brick_id,
                                                 transt, set_dict);
                 if (!xl) {
@@ -3317,8 +3447,9 @@ volgen_graph_build_snapview_client (volgen_graph_t *graph,
 
         get_transport_type (volinfo, set_dict, transt, _gf_false);
 
-        prot_clnt = volgen_graph_build_client (graph, volinfo, NULL, subvol,
-                                               xl_id, transt, set_dict);
+        prot_clnt = volgen_graph_build_client (graph, volinfo,
+                                                NULL, NULL, subvol,
+                                                xl_id, transt, set_dict);
         if (!prot_clnt) {
                 ret = -1;
                 goto out;
@@ -3933,14 +4064,35 @@ static int client_graph_set_perf_options(volgen_graph_t *graph,
         int             ret                      = 0;
 
         /*
+         * Logic to make sure gfproxy-client gets custom performance translators
+         */
+        ret = dict_get_str_boolean (set_dict, "gfproxy-client", 0);
+        if (ret == 1) {
+                return volgen_graph_set_options_generic (
+                    graph, set_dict, volinfo,
+                    &gfproxy_client_perfxl_option_handler);
+        }
+
+        /*
+         * Logic to make sure gfproxy-server gets custom performance translators
+         */
+        ret = dict_get_str_boolean (set_dict, "gfproxy-server", 0);
+        if (ret == 1) {
+                return volgen_graph_set_options_generic (
+                        graph, set_dict, volinfo,
+                        &gfproxy_server_perfxl_option_handler);
+        }
+
+        /*
          * Logic to make sure NFS doesn't have performance translators by
          * default for a volume
          */
-        volname = volinfo->volname;
         ret = client_graph_set_rda_options (graph, volinfo, set_dict);
         if (ret < 0)
                 return ret;
 
+        volname = volinfo->volname;
+
         tmp_data = dict_get (set_dict, "nfs-volume-file");
         if (!tmp_data)
                 return volgen_graph_set_options_generic(graph, set_dict,
@@ -4172,25 +4324,48 @@ client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
         gf_boolean_t     ob            = _gf_false;
         int              uss_enabled   = -1;
         xlator_t        *this          = THIS;
+        char            *subvol        = NULL;
+        size_t          subvol_namelen = 0;
 
         GF_ASSERT (this);
         GF_ASSERT (conf);
 
-        volname = volinfo->volname;
-        ret = volgen_graph_build_clients (graph, volinfo, set_dict,
-                                          param);
-        if (ret)
+        ret = dict_get_str_boolean (set_dict, "gfproxy-client", 0);
+        if (ret == -1)
                 goto out;
 
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                ret = volume_volgen_graph_build_clusters_tier
-                                        (graph, volinfo, _gf_false);
-        else
-                ret = volume_volgen_graph_build_clusters
-                                        (graph, volinfo, _gf_false);
+        volname = volinfo->volname;
+        if (ret == 0) {
+                ret = volgen_graph_build_clients (graph, volinfo, set_dict,
+                                                  param);
+                if (ret)
+                        goto out;
 
-        if (ret == -1)
-                goto out;
+                if (volinfo->type == GF_CLUSTER_TYPE_TIER)
+                        ret = volume_volgen_graph_build_clusters_tier
+                                                (graph, volinfo, _gf_false);
+                else
+                        ret = volume_volgen_graph_build_clusters
+                                                (graph, volinfo, _gf_false);
+
+                if (ret == -1)
+                        goto out;
+        } else {
+                ret = dict_get_str (set_dict,
+                                        "config.gfproxyd-remote-host", &tmp);
+                if (ret == -1)
+                        goto out;
+
+                subvol_namelen = strlen (volinfo->volname) +
+                                strlen ("-server") + 1;
+                subvol = alloca (subvol_namelen);
+                snprintf (subvol, subvol_namelen,
+                                "%s-server", volinfo->volname);
+
+                volgen_graph_build_client (graph, volinfo, tmp,
+                                           GF_PROXY_DAEMON_PORT_STR, subvol,
+                                           "gfproxy", "tcp", set_dict);
+        }
 
         ret = dict_get_str_boolean (set_dict, "features.shard", _gf_false);
         if (ret == -1)
@@ -5325,6 +5500,21 @@ get_brick_filepath (char *filename, glusterd_volinfo_t *volinfo,
                           brickinfo->hostname, brick);
 }
 
+static void
+get_gfproxyd_filepath (char *filename, glusterd_volinfo_t *volinfo)
+{
+        char  path[PATH_MAX]   = {0, };
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        snprintf (filename, PATH_MAX,
+                        "%s/%s.gfproxyd.vol", path,
+                        volinfo->volname);
+}
+
 gf_boolean_t
 glusterd_is_valid_volfpath (char *volname, char *brick)
 {
@@ -5370,6 +5560,32 @@ out:
 }
 
 static int
+glusterd_generate_gfproxyd_volfile (glusterd_volinfo_t *volinfo)
+{
+        volgen_graph_t graph = {0, };
+        char    filename[PATH_MAX] = {0, };
+        int     ret = -1;
+
+        GF_ASSERT (volinfo);
+
+        get_gfproxyd_filepath (filename, volinfo);
+
+        struct glusterd_gfproxyd_info info = {
+                .port = GF_PROXY_DAEMON_PORT,
+        };
+
+        ret = build_graph_generic (&graph, volinfo,
+                                   NULL, &info,
+                                   &gfproxy_server_graph_builder);
+        if (ret == 0)
+                ret = volgen_write_volfile (&graph, filename);
+
+        volgen_graph_free (&graph);
+
+        return ret;
+}
+
+static int
 glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
                                  glusterd_brickinfo_t *brickinfo,
                                  dict_t *mod_dict, void *data)
@@ -5661,7 +5877,8 @@ glusterd_generate_client_per_brick_volfile (glusterd_volinfo_t *volinfo)
 
         cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
                 xl = volgen_graph_build_client (&graph, volinfo,
-                                                brick->hostname, brick->path,
+                                                brick->hostname,
+                                                NULL, brick->path,
                                                 brick->brick_id,
                                                 "tcp", dict);
                 if (!xl) {
@@ -5792,6 +6009,11 @@ generate_client_volfiles (glusterd_volinfo_t *volinfo,
                         ret = glusterd_get_trusted_client_filepath (filepath,
                                                                     volinfo,
                                                                     type);
+                } else if (client_type == GF_CLIENT_TRUSTED_PROXY) {
+                        glusterd_get_gfproxy_client_volfile (volinfo,
+                                                             filepath,
+                                                             PATH_MAX);
+                        ret = dict_set_str (dict, "gfproxy-client", "on");
                 } else {
                         ret = glusterd_get_client_filepath (filepath,
                                                             volinfo,
@@ -6037,6 +6259,7 @@ build_bitd_volume_graph (volgen_graph_t *graph,
 
                 xl = volgen_graph_build_client (&cgraph, volinfo,
                                                 brickinfo->hostname,
+                                                NULL,
                                                 brickinfo->path,
                                                 brickinfo->brick_id,
                                                 transt, set_dict);
@@ -6198,6 +6421,7 @@ build_scrub_volume_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
 
                 xl = volgen_graph_build_client (&cgraph, volinfo,
                                                 brickinfo->hostname,
+                                                NULL,
                                                 brickinfo->path,
                                                 brickinfo->brick_id,
                                                 transt, set_dict);
@@ -6329,12 +6553,25 @@ glusterd_create_volfiles (glusterd_volinfo_t *volinfo)
                 goto out;
         }
 
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED_PROXY);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate gfproxy client volfiles");
+                goto out;
+        }
+
         ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
         if (ret)
                 gf_msg (this->name, GF_LOG_ERROR, 0,
                         GD_MSG_VOLFILE_CREATE_FAIL,
                         "Could not generate client volfiles");
 
+
+         ret = glusterd_generate_gfproxyd_volfile (volinfo);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate gfproxy volfiles");
+
 out:
         return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index aa4359c258..2a8a6e6aed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -57,7 +57,8 @@
 
 typedef enum {
         GF_CLIENT_TRUSTED,
-        GF_CLIENT_OTHER
+        GF_CLIENT_OTHER,
+        GF_CLIENT_TRUSTED_PROXY,
 } glusterd_client_type_t;
 
 /* It indicates the type of volfile that the graph is built for */
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index c4ad315272..d3f31d0c34 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -2820,6 +2820,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
           .option      = "!config",
           .op_version  = 2
         },
+        { .key         = "config.gfproxyd-remote-host",
+          .voltype     = "configuration",
+          .option      = "gfproxyd-remote-host",
+          .op_version  = 2
+        },
         { .key         = GLUSTERD_QUORUM_TYPE_KEY,
           .voltype     = "mgmt/glusterd",
           .value       = "off",
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 8beec6c29c..281a6c198f 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -254,6 +254,11 @@ struct glusterd_brick_proc {
 
 typedef struct glusterd_brick_proc glusterd_brick_proc_t;
 
+struct glusterd_gfproxyd_info {
+        short   port;
+        char    *logfile;
+};
+
 struct gf_defrag_brickinfo_ {
         char *name;
         int   files;