r14162: Minor fixes on sort

Initial work on async paged_results
(This used to be commit 72523eae7f8925a2c23d3260875345adcf1661bb)

2 files changed, 485 insertions, 55 deletions

diff --git a/source4/lib/ldb/modules/paged_results.c b/source4/lib/ldb/modules/paged_results.c
index fd0cc7c90b0..58b066bd140 100644
--- a/source4/lib/ldb/modules/paged_results.c
+++ b/source4/lib/ldb/modules/paged_results.c
@@ -1,7 +1,7 @@
 /* 
    ldb database library
 
-   Copyright (C) Simo Sorce  2005
+   Copyright (C) Simo Sorce  2005-2006
 
      ** NOTE! The following LGPL license applies to the ldb
      ** library. This does NOT imply that all of Samba is released
@@ -23,7 +23,7 @@
 */
 
 /*
- *  Name: ldb
+ *  Name: paged_result
  *
  *  Component: ldb paged results control module
  *
@@ -36,13 +36,32 @@
 #include "includes.h"
 #include "ldb/include/includes.h"
 
+struct message_store {
+	/* keep the whole ldb_async_result as an optimization
+	 * instead of freeing and talloc-ing the container
+	 * on each result */
+	struct ldb_async_result *r;
+	struct message_store *next;
+};
+
 struct results_store {
 	char *cookie;
 	time_t timestamp;
-	int num_sent;
-	struct ldb_result *result;
+	int num_sent; /* To be removed */
+	struct ldb_result *result; /* To be removed */
 	struct results_store *prev;
 	struct results_store *next;
+	
+	struct message_store *first;
+	struct message_store *last;
+	int num_entries;
+
+	struct message_store *first_ref;
+	struct message_store *last_ref;
+
+	struct ldb_control **controls;
+
+	struct ldb_request *req;
 };
 
 struct private_data {
@@ -52,6 +71,66 @@ struct private_data {
 	
 };
 
+struct paged_async_context {
+	struct ldb_module *module;
+	void *up_context;
+	int (*up_callback)(struct ldb_context *, void *, struct ldb_async_result *);
+	int timeout;
+
+	int size;
+
+	struct results_store *store;
+};
+
+static struct ldb_async_handle *init_handle(void *mem_ctx, struct ldb_module *module,
+					    void *context,
+					    int (*callback)(struct ldb_context *, void *, struct ldb_async_result *),
+					    int timeout)
+{
+	struct paged_async_context *ac;
+	struct ldb_async_handle *h;
+
+	h = talloc_zero(mem_ctx, struct ldb_async_handle);
+	if (h == NULL) {
+		ldb_set_errstring(module->ldb, talloc_asprintf(module, "Out of Memory"));
+		return NULL;
+	}
+
+	h->module = module;
+
+	ac = talloc_zero(h, struct paged_async_context);
+	if (ac == NULL) {
+		ldb_set_errstring(module->ldb, talloc_asprintf(module, "Out of Memory"));
+		talloc_free(h);
+		return NULL;
+	}
+
+	h->private_data = (void *)ac;
+
+	h->state = LDB_ASYNC_INIT;
+	h->status = LDB_SUCCESS;
+
+	ac->module = module;
+	ac->up_context = context;
+	ac->up_callback = callback;
+	ac->timeout = timeout;
+
+	return h;
+}
+
+int store_destructor(void *data)
+{
+	struct results_store *store = talloc_get_type(data, struct results_store);
+
+	if (store->prev) {
+		store->prev->next = store->next;
+	}
+	if (store->next) {
+		store->next->prev = store->prev;
+	}
+	
+	return 0;
+}
 
 static struct results_store *new_store(struct private_data *priv)
 {
@@ -73,8 +152,13 @@ static struct results_store *new_store(struct private_data *priv)
 
 	new->timestamp = time(NULL);
 
-	new->num_sent = 0;
-	new->result = NULL;
+	new->num_sent = 0; /* To be removed */
+	new->result = NULL; /* To be removed */
+
+	new->first = NULL;
+	new->num_entries = 0;
+	new->first_ref = NULL;
+	new->controls = NULL;
 
 	/* put this entry as first */
 	new->prev = NULL;
@@ -82,40 +166,22 @@ static struct results_store *new_store(struct private_data *priv)
 	if (priv->store != NULL) priv->store->prev = new;
 	priv->store = new;
 
-	return new;
-}
+	talloc_set_destructor(new, store_destructor);
 
-static void remove_store(struct results_store *store)
-{
-	if (store->prev) {
-		store->prev->next = store->next;
-	}
-	if (store->next) {
-		store->next->prev = store->prev;
-	}
-	talloc_free(store);
+	return new;
 }
 
 /* search */
-static int paged_search(struct ldb_module *module, struct ldb_request *req)
+static int paged_search(struct ldb_module *module, struct ldb_control *control, struct ldb_request *req)
 {
 	struct private_data *private_data = talloc_get_type(module->private_data, struct private_data);
 	struct results_store *current = NULL;
 	struct ldb_result *paged_result;
 	struct ldb_control **saved_controls;
-	struct ldb_control *control;
 	struct ldb_paged_control *paged_ctrl;
 	struct ldb_paged_control *paged_ret;
 	int i, ret;
 
-	/* check if there's a paged request control */
-	control = get_control_from_list(req->controls, LDB_CONTROL_PAGED_RESULTS_OID);
-
-	if (control == NULL) {
-		/* not found go on */
-		return ldb_next_request(module, req);
-	}
-
 	paged_ctrl = talloc_get_type(control->data, struct ldb_paged_control);
 	if (!paged_ctrl) {
 		return LDB_ERR_PROTOCOL_ERROR;
@@ -176,7 +242,7 @@ static int paged_search(struct ldb_module *module, struct ldb_request *req)
 	/* check if it is an abandon */
 	if (paged_ctrl->size == 0) {
 		req->op.search.res = talloc_steal(private_data, paged_result);
-		remove_store(current);
+		talloc_free(current);
 		return LDB_SUCCESS;
 	}
 
@@ -234,15 +300,375 @@ static int paged_search(struct ldb_module *module, struct ldb_request *req)
 	return LDB_SUCCESS;	
 }
 
+static int paged_search_async_callback(struct ldb_context *ldb, void *context, struct ldb_async_result *ares)
+{
+	struct paged_async_context *ac = NULL;
+
+	if (!context || !ares) {
+		ldb_set_errstring(ldb, talloc_asprintf(ldb, "NULL Context or Result in callback"));
+		goto error;
+	}
+
+	ac = talloc_get_type(context, struct paged_async_context);
+
+	if (ares->type == LDB_REPLY_ENTRY) {
+		if (ac->store->first == NULL) {
+			ac->store->first = ac->store->last = talloc(ac->store, struct message_store);
+		} else {
+			ac->store->last->next = talloc(ac->store, struct message_store);
+			ac->store->last = ac->store->last->next;
+		}
+		if (ac->store->last == NULL) {
+			goto error;
+		}
+
+		ac->store->num_entries++;
+
+		ac->store->last->r = talloc_steal(ac->store->last, ares);
+		if (ac->store->last->r == NULL) {
+			goto error;
+		}
+		ac->store->last->next = NULL;
+	}
+
+	if (ares->type == LDB_REPLY_REFERRAL) {
+		if (ac->store->first_ref == NULL) {
+			ac->store->first_ref = ac->store->last_ref = talloc(ac->store, struct message_store);
+		} else {
+			ac->store->last_ref->next = talloc(ac->store, struct message_store);
+			ac->store->last_ref = ac->store->last_ref->next;
+		}
+		if (ac->store->last_ref == NULL) {
+			goto error;
+		}
+
+		ac->store->last_ref->r = talloc_steal(ac->store->last, ares);
+		if (ac->store->last_ref->r == NULL) {
+			goto error;
+		}
+		ac->store->last_ref->next = NULL;
+	}
+
+	if (ares->type == LDB_REPLY_DONE) {
+		if (ares->controls) {
+			ac->store->controls = talloc_steal(ac->store, ares->controls);
+			if (! ac->store->controls) {
+				goto error;
+			}
+		}
+		talloc_free(ares);
+	}
+
+	return LDB_SUCCESS;
+
+error:
+	talloc_free(ares);
+	return LDB_ERR_OPERATIONS_ERROR;
+}
+
+static int paged_search_async(struct ldb_module *module, struct ldb_control *control, struct ldb_request *req)
+{
+	struct private_data *private_data = talloc_get_type(module->private_data, struct private_data);
+	struct ldb_paged_control *paged_ctrl;
+	struct ldb_control **saved_controls;
+	struct paged_async_context *ac;
+	struct ldb_async_handle *h;
+	int ret;
+
+	req->async.handle = NULL;
+
+	if (!req->async.callback || !req->async.context) {
+		ldb_set_errstring(module->ldb, talloc_asprintf(module,
+				  "Async interface called with NULL callback function or NULL context"));
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	
+	paged_ctrl = talloc_get_type(control->data, struct ldb_paged_control);
+	if (!paged_ctrl) {
+		return LDB_ERR_PROTOCOL_ERROR;
+	}
+
+	h = init_handle(req, module, req->async.context, req->async.callback, req->async.timeout);
+	if (!h) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ac = talloc_get_type(h->private_data, struct paged_async_context);
+
+	ac->size = paged_ctrl->size;
+
+	/* check if it is a continuation search the store */
+	if (paged_ctrl->cookie_len == 0) {
+		
+		ac->store = new_store(private_data);
+		if (ac->store == NULL) {
+			talloc_free(h);
+			return LDB_ERR_UNWILLING_TO_PERFORM;
+		}
+
+		ac->store->req = talloc(ac->store, struct ldb_request);
+		if (!ac->store->req)
+			return LDB_ERR_OPERATIONS_ERROR;
+
+		ac->store->req->operation = req->operation;
+		ac->store->req->op.search.base = req->op.search.base;
+		ac->store->req->op.search.scope = req->op.search.scope;
+		ac->store->req->op.search.tree = req->op.search.tree;
+		ac->store->req->op.search.attrs = req->op.search.attrs;
+		ac->store->req->controls = req->controls;
+
+		/* save it locally and remove it from the list */
+		/* we do not need to replace them later as we
+		 * are keeping the original req intact */
+		if (!save_controls(control, ac->store->req, &saved_controls)) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+
+		ac->store->req->creds = req->creds;
+
+		ac->store->req->async.context = ac;
+		ac->store->req->async.callback = paged_search_async_callback;
+		ac->store->req->async.timeout = req->async.timeout;
+
+		ret = ldb_next_request(module, ac->store->req);
+
+	} else {
+		struct results_store *current = NULL;
+
+		for (current = private_data->store; current; current = current->next) {
+			if (strcmp(current->cookie, paged_ctrl->cookie) == 0) {
+				current->timestamp = time(NULL);
+				break;
+			}
+		}
+		if (current == NULL) {
+			talloc_free(h);
+			return LDB_ERR_UNWILLING_TO_PERFORM;
+		}
+
+		ac->store = current;
+		ret = LDB_SUCCESS;
+	}
+
+	req->async.handle = h;
+
+	/* check if it is an abandon */
+	if (ac->size == 0) {
+		talloc_free(ac->store);
+		h->status = LDB_SUCCESS;
+		h->state = LDB_ASYNC_DONE;
+		return LDB_SUCCESS;
+	}
+
+	/* TODO: age out old outstanding requests */
+
+	return ret;
+
+}
+
+static int paged_async_results(struct ldb_async_handle *handle)
+{
+	struct paged_async_context *ac;
+	struct ldb_paged_control *paged;
+	struct ldb_async_result *ares;
+	struct message_store *msg;
+	int i, num_ctrls, ret;
+
+	ac = talloc_get_type(handle->private_data, struct paged_async_context);
+
+	if (ac->store == NULL)
+		return LDB_ERR_OPERATIONS_ERROR;
+
+	while (ac->store->num_entries > 0 && ac->size > 0) {
+		msg = ac->store->first;
+		ret = ac->up_callback(ac->module->ldb, ac->up_context, msg->r);
+		if (ret != LDB_SUCCESS) {
+			handle->status = ret;
+			handle->state = LDB_ASYNC_DONE;
+			return ret;
+		}
+
+		ac->store->first = msg->next;
+		talloc_free(msg);
+		ac->store->num_entries--;
+		ac->size--;
+	}
+
+	handle->state = LDB_ASYNC_DONE;
+
+	while (ac->store->first_ref != NULL) {
+		msg = ac->store->first_ref;
+		ret = ac->up_callback(ac->module->ldb, ac->up_context, msg->r);
+		if (ret != LDB_SUCCESS) {
+			handle->status = ret;
+			handle->state = LDB_ASYNC_DONE;
+			return ret;
+		}
+
+		ac->store->first_ref = msg->next;
+		talloc_free(msg);
+	}
+
+	ares = talloc_zero(ac->store, struct ldb_async_result);
+	if (ares == NULL) {
+		handle->status = LDB_ERR_OPERATIONS_ERROR;
+		return handle->status;
+	}
+	num_ctrls = 2;
+	i = 0;
+
+	if (ac->store->controls != NULL) {
+		ares->controls = ac->store->controls;
+		while (ares->controls[i]) i++; /* counting */
+
+		ares->controls = talloc_steal(ares, ac->store->controls);
+		num_ctrls += i;
+	}
+
+	ares->controls = talloc_realloc(ares, ares->controls, struct ldb_control *, num_ctrls);
+	if (ares->controls == NULL) {
+		handle->status = LDB_ERR_OPERATIONS_ERROR;
+		return handle->status;
+	}
+
+	ares->controls[i] = talloc(ares->controls, struct ldb_control);
+	if (ares->controls[i] == NULL) {
+		handle->status = LDB_ERR_OPERATIONS_ERROR;
+		return handle->status;
+	}
+
+	ares->controls[i]->oid = talloc_strdup(ares->controls[i], LDB_CONTROL_PAGED_RESULTS_OID);
+	if (ares->controls[i]->oid == NULL) {
+		handle->status = LDB_ERR_OPERATIONS_ERROR;
+		return handle->status;
+	}
+		
+	ares->controls[i]->critical = 0;
+	ares->controls[i + 1] = NULL;
+
+	paged = talloc(ares->controls[i], struct ldb_paged_control);
+	if (paged == NULL) {
+		handle->status = LDB_ERR_OPERATIONS_ERROR;
+		return handle->status;
+	}
+	
+	ares->controls[i]->data = paged;
+
+	if (ac->size > 0) {
+		paged->size = 0;
+		paged->cookie = NULL;
+		paged->cookie_len = 0;
+	} else {
+		paged->size = ac->store->num_entries;
+		paged->cookie = talloc_strdup(paged, ac->store->cookie);
+		paged->cookie_len = strlen(paged->cookie) + 1;
+	}
+
+	ares->type = LDB_REPLY_DONE;
+
+	ret = ac->up_callback(ac->module->ldb, ac->up_context, ares);
+
+	handle->status = ret;
+
+	return ret;
+}
+
+static int paged_async_wait(struct ldb_async_handle *handle, enum ldb_async_wait_type type)
+{
+	struct paged_async_context *ac;
+	int ret;
+    
+	if (!handle || !handle->private_data) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	if (handle->state == LDB_ASYNC_DONE) {
+		return handle->status;
+	}
+
+	handle->state = LDB_ASYNC_PENDING;
+
+	ac = talloc_get_type(handle->private_data, struct paged_async_context);
+
+	if (ac->store->req->async.handle->state == LDB_ASYNC_DONE) {
+		/* if lower level is finished we do not need to call it anymore */
+		/* return all we have until size == 0 or we empty storage */
+		ret = paged_async_results(handle);
+
+		/* we are done, if num_entries is zero free the storage
+		 * as that mean we delivered the last batch */
+		if (ac->store->num_entries == 0) {
+			talloc_free(ac->store);
+		}
+
+		return ret;
+	}
+
+	if (type == LDB_WAIT_ALL) {
+		while (ac->store->req->async.handle->state != LDB_ASYNC_DONE) {
+			ret = ldb_async_wait(ac->store->req->async.handle, type);
+			if (ret != LDB_SUCCESS) {
+				handle->state = LDB_ASYNC_DONE;
+				handle->status = ret;
+				return ret;
+			}
+		}
+
+		ret = paged_async_results(handle);
+
+		/* we are done, if num_entries is zero free the storage
+		 * as that mean we delivered the last batch */
+		if (ac->store->num_entries == 0) {
+			talloc_free(ac->store);
+		}
+
+		return ret;
+	}
+
+	ret = ldb_async_wait(ac->store->req->async.handle, type);
+	if (ret != LDB_SUCCESS) {
+		handle->state = LDB_ASYNC_DONE;
+		handle->status = ret;
+		return ret;
+	}
+
+	handle->status = ret;
+
+	if (ac->store->num_entries >= ac->size ||
+	    ac->store->req->async.handle->state == LDB_ASYNC_DONE) {
+
+		ret = paged_async_results(handle);
+
+		/* we are done, if num_entries is zero free the storage
+		 * as that mean we delivered the last batch */
+		if (ac->store->num_entries == 0) {
+			talloc_free(ac->store);
+		}
+	}
+
+	return ret;
+}
+
 static int paged_request(struct ldb_module *module, struct ldb_request *req)
 {
+	struct ldb_control *control;
+
+	/* check if there's a paged request control */
+	control = get_control_from_list(req->controls, LDB_CONTROL_PAGED_RESULTS_OID);
+	if (control == NULL) {
+		/* not found go on */
+		return ldb_next_request(module, req);
+	}
+
 	switch (req->operation) {
 
 	case LDB_REQ_SEARCH:
-		return paged_search(module, req);
+		return paged_search(module, control, req);
+
+	case LDB_ASYNC_SEARCH:
+		return paged_search_async(module, control, req);
 
 	default:
-		return ldb_next_request(module, req);
+		return LDB_ERR_PROTOCOL_ERROR;
 
 	}
 }
@@ -285,6 +711,7 @@ static int paged_request_init(struct ldb_module *module)
 static const struct ldb_module_ops paged_ops = {
 	.name		   	= "paged_results",
 	.request        = paged_request,
+	.async_wait     = paged_async_wait,
 	.init_context 	= paged_request_init
 };
 
diff --git a/source4/lib/ldb/modules/sort.c b/source4/lib/ldb/modules/sort.c
index 08047c21f5d..820ba1c1f73 100644
--- a/source4/lib/ldb/modules/sort.c
+++ b/source4/lib/ldb/modules/sort.c
@@ -89,6 +89,9 @@ static struct ldb_async_handle *init_handle(void *mem_ctx, struct ldb_module *mo
 
 	h->private_data = (void *)ac;
 
+	h->state = LDB_ASYNC_INIT;
+	h->status = LDB_SUCCESS;
+
 	ac->module = module;
 	ac->up_context = context;
 	ac->up_callback = callback;
@@ -423,31 +426,6 @@ static int server_sort_search_async(struct ldb_module *module, struct ldb_contro
 	return ldb_next_request(module, ac->req);
 }
 
-static int server_sort(struct ldb_module *module, struct ldb_request *req)
-{
-	struct ldb_control *control;
-
-	/* check if there's a paged request control */
-	control = get_control_from_list(req->controls, LDB_CONTROL_SERVER_SORT_OID);
-	if (control == NULL) {
-		/* not found go on */
-		return ldb_next_request(module, req);
-	}
-
-	switch (req->operation) {
-
-	case LDB_REQ_SEARCH:
-		return server_sort_search(module, control, req);
-
-	case LDB_ASYNC_SEARCH:
-		return server_sort_search_async(module, control, req);
-
-	default:
-		return LDB_ERR_PROTOCOL_ERROR;
-
-	}
-}
-
 static int server_sort_async_results(struct ldb_async_handle *handle)
 {
 	struct sort_async_context *ac;
@@ -548,6 +526,31 @@ static int server_sort_async_wait(struct ldb_async_handle *handle, enum ldb_asyn
 	return ret;
 }
 
+static int server_sort(struct ldb_module *module, struct ldb_request *req)
+{
+	struct ldb_control *control;
+
+	/* check if there's a paged request control */
+	control = get_control_from_list(req->controls, LDB_CONTROL_SERVER_SORT_OID);
+	if (control == NULL) {
+		/* not found go on */
+		return ldb_next_request(module, req);
+	}
+
+	switch (req->operation) {
+
+	case LDB_REQ_SEARCH:
+		return server_sort_search(module, control, req);
+
+	case LDB_ASYNC_SEARCH:
+		return server_sort_search_async(module, control, req);
+
+	default:
+		return LDB_ERR_PROTOCOL_ERROR;
+
+	}
+}
+
 static int server_sort_init(struct ldb_module *module)
 {
 	struct ldb_request *req;