dht: Handle setxattr and rm race for directory in rebalance

Problem: Selfheal as part of directory does not return an error if
the layout setxattr fails. This is because the actual lookup fop
must have been successful to proceed for layout heal. Hence, we could
not tell if fix-layout failed in rebalance.

Solution: We can check this information in the layout structure that
whether all the xlators have returned error.

fixes: #1200
Change-Id: I3e5f2a36c0d934c21476a73a9a5473d8e490cde7
Signed-off-by: Susant Palai <spalai@redhat.com>

3 files changed, 33 insertions, 0 deletions

diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index e3aed576d7..749819680e 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -11436,3 +11436,22 @@ dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
                FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
     return 0;
 }
+
+/* The job of this function is to check if all the xlators have updated
+ * error in the layout. */
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode)
+{
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    layout = dht_layout_get(this, inode);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == 0) {
+            return 0;
+        }
+    }
+
+    /* Returning the first xlator error as all xlators have errors */
+    return layout->list[0].err;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 8a49eefe9b..2c90dc9951 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1481,4 +1481,7 @@ dht_create_lock(call_frame_t *frame, xlator_t *subvol);
 int
 dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local);
 
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode);
+
 #endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 46c2f9a32a..236cae9ea6 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -3905,6 +3905,17 @@ gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
      */
 
     ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+
+    /* In case of a race where the directory is deleted just before
+     * layout setxattr, the errors are updated in the layout structure.
+     * We can use this information to make a decision whether the directory
+     * is deleted entirely.
+     */
+    if (ret == 0) {
+        ret = dht_dir_layout_error_check(this, loc->inode);
+        ret = -ret;
+    }
+
     if (ret) {
         if (-ret == ENOENT || -ret == ESTALE) {
             gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED,