From 5b290bc6e3df76bea04c11cb15643173c33455bd Mon Sep 17 00:00:00 2001
From: Pranith Kumar K <pranithk@gluster.com>
Date: Thu, 8 Mar 2012 22:17:46 +0530
Subject: syncop: Make syntask scalable

At the moment, synctask uses task->frame to perform
all the syncops, this will lead to high-memory usage if the task
crawls millions of directories. i.e millions of STACK_WINDS/UNWINDS.
To prevent this, in each task a new stack is created to perform
the fops which is reset after every syncop.

Change-Id: I53c262ec348be9b1d91af73da01f1c217f31ce6e
BUG: 798907
Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Reviewed-on: http://review.gluster.com/2850
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
---
 libglusterfs/src/stack.h  | 18 ++++++++++++++++++
 libglusterfs/src/syncop.c | 31 +++++++++++++------------------
 libglusterfs/src/syncop.h |  9 ++++++---
 3 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/libglusterfs/src/stack.h b/libglusterfs/src/stack.h
index 220eab49d0..a18ca6deb8 100644
--- a/libglusterfs/src/stack.h
+++ b/libglusterfs/src/stack.h
@@ -177,6 +177,7 @@ STACK_DESTROY (call_stack_t *stack)
         }
 
         LOCK_DESTROY (&stack->frames.lock);
+        LOCK_DESTROY (&stack->stack_lock);
 
         while (stack->frames.next) {
                 FRAME_DESTROY (stack->frames.next);
@@ -187,6 +188,23 @@ STACK_DESTROY (call_stack_t *stack)
                 mem_put (local);
 }
 
+static inline void
+STACK_RESET (call_stack_t *stack)
+{
+        void *local = NULL;
+
+        if (stack->frames.local) {
+                local = stack->frames.local;
+                stack->frames.local = NULL;
+        }
+
+        while (stack->frames.next) {
+                FRAME_DESTROY (stack->frames.next);
+        }
+
+        if (local)
+                mem_put (local);
+}
 
 #define cbk(x) cbk_##x
 
diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 4acac5f8fa..47bb0137da 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -24,22 +24,6 @@
 
 #include "syncop.h"
 
-call_frame_t *
-syncop_create_frame ()
-{
-        struct synctask *task = NULL;
-        call_frame_t *frame = NULL;
-
-        task = synctask_get ();
-
-        if (task) {
-                frame = task->frame;
-        }
-
-        return (call_frame_t *)frame;
-}
-
-
 static void
 __run (struct synctask *task)
 {
@@ -160,6 +144,9 @@ synctask_destroy (struct synctask *task)
         if (task->stack)
                 FREE (task->stack);
 
+        if (task->opframe)
+                STACK_DESTROY (task->opframe->root);
+
 	pthread_mutex_destroy (&task->mutex);
 
 	pthread_cond_destroy (&task->cond);
@@ -195,18 +182,24 @@ synctask_new (struct syncenv *env, synctask_fn_t fn, synctask_cbk_t cbk,
 
         VALIDATE_OR_GOTO (env, err);
         VALIDATE_OR_GOTO (fn, err);
-        VALIDATE_OR_GOTO (frame, err);
 
         newtask = CALLOC (1, sizeof (*newtask));
         if (!newtask)
                 return -ENOMEM;
 
+        newtask->frame      = frame;
+        if (!frame) {
+                newtask->opframe = create_frame (this, this->ctx->pool);
+        } else {
+                newtask->opframe = copy_frame (frame);
+        }
+        if (!newtask->opframe)
+                goto err;
         newtask->env        = env;
         newtask->xl         = this;
         newtask->syncfn     = fn;
 	newtask->synccbk    = cbk;
         newtask->opaque     = opaque;
-        newtask->frame      = frame;
 
         INIT_LIST_HEAD (&newtask->all_tasks);
 
@@ -260,6 +253,8 @@ err:
         if (newtask) {
                 if (newtask->stack)
                         FREE (newtask->stack);
+                if (newtask->opframe)
+                        STACK_DESTROY (newtask->opframe->root);
                 FREE (newtask);
         }
         return -1;
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index 1bea189a7d..8ec4e9e586 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -56,6 +56,7 @@ struct synctask {
         struct syncenv     *env;
         xlator_t           *xl;
         call_frame_t       *frame;
+        call_frame_t       *opframe;
         synctask_cbk_t      synccbk;
         synctask_fn_t       syncfn;
 	synctask_state_t    state;
@@ -153,12 +154,14 @@ struct syncargs {
 
 
 #define SYNCOP(subvol, stb, cbk, op, params ...) do {                   \
-                call_frame_t    *frame = NULL;                          \
+                struct  synctask        *task = NULL;                   \
                                                                         \
-                frame = syncop_create_frame ();                         \
+                task = synctask_get ();                                 \
                                                                         \
-                STACK_WIND_COOKIE (frame, cbk, (void *)stb, subvol, op, params); \
+                STACK_WIND_COOKIE (task->opframe, cbk, (void *)stb,     \
+                                   subvol, op, params);                 \
                 __yield (stb);                                          \
+                STACK_RESET (task->opframe->root);                              \
         } while (0)
 
 
-- 
cgit