cluster/afr: Choose local child as source if possible

It is better to choose local brick as source if possible to prevent
over the wire read thus saving on bandwidth. Also changed code to not
attempt data-heal if 'source' is selected as arbiter.

Change-Id: I9a328d0198422280b13a30ab99545370a301dfea
BUG: 1314150
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/13585
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Krutika Dhananjay <kdhananj@redhat.com>
Tested-by: Krutika Dhananjay <kdhananj@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

9 files changed, 57 insertions, 31 deletions

diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 7f6bf74abd..e0f530f999 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1886,6 +1886,7 @@ afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
          * the slowest local subvolume is far preferable to a remote one.
          */
         if (is_local) {
+                priv->local[child_index] = 1;
                 /* Don't set arbiter as read child. */
                 if (AFR_IS_ARBITER_BRICK(priv, child_index))
                         goto out;
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 74e1a44406..f12ce81b0e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1532,3 +1532,32 @@ afr_throttled_selfheal (call_frame_t *frame, xlator_t *this)
                                       "pending, background self-heal rejected.");
         }
 }
+
+int
+afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
+                             afr_transaction_type type)
+{
+        int source = -1;
+        int i      = 0;
+
+        /* Give preference to local child to save on bandwidth */
+        for (i = 0; i < priv->child_count; i++) {
+                if (priv->local[i] && sources[i]) {
+                        if ((type == AFR_DATA_TRANSACTION) &&
+                            AFR_IS_ARBITER_BRICK (priv, i))
+                                continue;
+
+                        source = i;
+                        goto out;
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source = i;
+                        goto out;
+                }
+        }
+out:
+        return source;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 27d2849c15..a2dddafe5c 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -572,7 +572,6 @@ __afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
 				     struct afr_reply *replies,
                                      uint64_t *witness)
 {
-	int i = 0;
 	afr_private_t *priv = NULL;
 	int source = -1;
 	int sources_count = 0;
@@ -609,13 +608,9 @@ __afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
 
 out:
         afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
+        source = afr_choose_source_by_policy (priv, sources,
+                                              AFR_DATA_TRANSACTION);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
-        }
 	return source;
 }
 
@@ -729,8 +724,7 @@ __afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 		source = ret;
 
-                if (AFR_IS_ARBITER_BRICK(priv, source) &&
-                    AFR_COUNT (sources, priv->child_count) == 1) {
+                if (AFR_IS_ARBITER_BRICK(priv, source)) {
                         did_sh = _gf_false;
                         goto unlock;
                 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 42a6c0453b..e4d616ea20 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -356,7 +356,6 @@ __afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
                                       struct afr_reply *replies,
                                       uint64_t *witness)
 {
-	int i = 0;
 	afr_private_t *priv = NULL;
 	int source = -1;
 	int sources_count = 0;
@@ -373,13 +372,8 @@ __afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
 		return -1;
 	}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i]) {
-			source = i;
-			break;
-		}
-	}
-
+        source = afr_choose_source_by_policy (priv, sources,
+                                              AFR_ENTRY_TRANSACTION);
 	return source;
 }
 
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 24b139ad21..65c25abcb4 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -204,7 +204,7 @@ __afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
 {
 	int i = 0;
 	afr_private_t *priv = NULL;
-	struct iatt first = {0, };
+	struct iatt srcstat = {0, };
 	int source = -1;
 	int sources_count = 0;
 
@@ -257,23 +257,17 @@ __afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
         if (afr_dict_contains_heal_op(frame))
                 return -EIO;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i])
-			continue;
-		if (source == -1) {
-			source = i;
-			first = replies[i].poststat;
-                        break;
-		}
-	}
+        source = afr_choose_source_by_policy (priv, sources,
+                                              AFR_METADATA_TRANSACTION);
+        srcstat = replies[source].poststat;
 
 	for (i = 0; i < priv->child_count; i++) {
 		if (!sources[i] || i == source)
 			continue;
-		if (!IA_EQUAL (first, replies[i].poststat, type) ||
-		    !IA_EQUAL (first, replies[i].poststat, uid) ||
-		    !IA_EQUAL (first, replies[i].poststat, gid) ||
-		    !IA_EQUAL (first, replies[i].poststat, prot)) {
+		if (!IA_EQUAL (srcstat, replies[i].poststat, type) ||
+		    !IA_EQUAL (srcstat, replies[i].poststat, uid) ||
+		    !IA_EQUAL (srcstat, replies[i].poststat, gid) ||
+		    !IA_EQUAL (srcstat, replies[i].poststat, prot)) {
                         gf_msg_debug (this->name, 0, "%s: iatt mismatch "
                                       "for source(%d) vs (%d)",
                                       uuid_utoa
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index b298fa130c..b0f545f281 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -266,4 +266,7 @@ afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_locked_fill (call_frame_t *frame, xlator_t *this,
                  unsigned char *locked_on);
+int
+afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
+                             afr_transaction_type type);
 #endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index c539e11760..228ea38866 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -516,14 +516,17 @@ afr_shd_index_healer (void *data)
 	struct subvol_healer *healer = NULL;
 	xlator_t *this = NULL;
 	int ret = 0;
+	afr_private_t *priv = NULL;
 
 	healer = data;
 	THIS = this = healer->this;
+	priv = this->private;
 
 	for (;;) {
 		afr_shd_healer_wait (healer);
 
 		ASSERT_LOCAL(this, healer);
+		priv->local[healer->subvol] = healer->local;
 
 		do {
 		        gf_msg_debug (this->name, 0,
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 535f3e5b94..2c7e69aae3 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -387,6 +387,13 @@ init (xlator_t *this)
 
         priv->wait_count = 1;
 
+        priv->local = GF_CALLOC (sizeof (unsigned char), child_count,
+                                 gf_afr_mt_char);
+        if (!priv->local) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
                                     gf_afr_mt_char);
         if (!priv->child_up) {
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 7bb5aa8ded..41ffa4ecdb 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -61,6 +61,7 @@ typedef struct _afr_private {
         inode_t *root_inode;
 
         unsigned char *child_up;
+        unsigned char *local;
 
         char **pending_key;