From 7e4bf8856e9b65e612ca195a3b4f10bfb53a8259 Mon Sep 17 00:00:00 2001 From: Simo Sorce Date: Tue, 17 Nov 2009 20:22:36 -0500 Subject: Optimize sysdb_enumgrent This brings down the time needed to enumerate my group database from 2.4 seconds to 0.15 seconds. --- server/db/sysdb.h | 10 +- server/db/sysdb_search.c | 347 +++-------------------------------------------- 2 files changed, 25 insertions(+), 332 deletions(-) (limited to 'server/db') diff --git a/server/db/sysdb.h b/server/db/sysdb.h index f94b43fdb..d9f224c1d 100644 --- a/server/db/sysdb.h +++ b/server/db/sysdb.h @@ -31,9 +31,12 @@ #define SYSDB_BASE "cn=sysdb" #define SYSDB_DOM_BASE "cn=%s,cn=sysdb" -#define SYSDB_TMPL_USER_BASE "cn=users,cn=%s,"SYSDB_BASE -#define SYSDB_TMPL_GROUP_BASE "cn=groups,cn=%s,"SYSDB_BASE -#define SYSDB_TMPL_CUSTOM_BASE "cn=custom,cn=%s,"SYSDB_BASE +#define SYSDB_USERS_CONTAINER "cn=users" +#define SYSDB_GROUPS_CONTAINER "cn=groups" +#define SYSDB_CUSTOM_CONTAINER "cn=custom" +#define SYSDB_TMPL_USER_BASE SYSDB_USERS_CONTAINER",cn=%s,"SYSDB_BASE +#define SYSDB_TMPL_GROUP_BASE SYSDB_GROUPS_CONTAINER",cn=%s,"SYSDB_BASE +#define SYSDB_TMPL_CUSTOM_BASE SYSDB_CUSTOM_CONTAINER",cn=%s,"SYSDB_BASE #define SYSDB_USER_CLASS "user" #define SYSDB_GROUP_CLASS "group" @@ -114,6 +117,7 @@ SYSDB_DEFAULT_ATTRS, \ NULL} #define SYSDB_GRSRC_ATTRS {SYSDB_NAME, SYSDB_GIDNUM, \ + SYSDB_MEMBER, \ SYSDB_DEFAULT_ATTRS, \ NULL} #define SYSDB_GRPW_ATTRS {SYSDB_NAME, SYSDB_UIDNUM, \ diff --git a/server/db/sysdb_search.c b/server/db/sysdb_search.c index 2b5dc369e..4b9470baa 100644 --- a/server/db/sysdb_search.c +++ b/server/db/sysdb_search.c @@ -35,15 +35,13 @@ struct sysdb_search_ctx { struct sss_domain_info *domain; - bool enumeration; const char *expression; sysdb_callback_t callback; void *ptr; gen_callback gen_aux_fn; - - struct get_mem_ctx *gmctx; + bool gen_conv_mpg_users; struct ldb_result *res; @@ -96,12 +94,14 @@ static void request_done(struct sysdb_search_ctx *sctx) sctx->callback(sctx->ptr, EOK, sctx->res); } +static int mpg_convert(struct ldb_message *msg); + static int get_gen_callback(struct ldb_request *req, struct ldb_reply *rep) { struct sysdb_search_ctx *sctx; struct ldb_result *res; - int n; + int n, ret; sctx = talloc_get_type(req->context, struct sysdb_search_ctx); res = sctx->res; @@ -117,6 +117,15 @@ static int get_gen_callback(struct ldb_request *req, switch (rep->type) { case LDB_REPLY_ENTRY: + + if (sctx->gen_conv_mpg_users) { + ret = mpg_convert(rep->message); + if (ret != EOK) { + request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); + return LDB_ERR_OPERATIONS_ERROR; + } + } + res->msgs = talloc_realloc(res, res->msgs, struct ldb_message *, res->count + 2); @@ -298,8 +307,6 @@ int sysdb_enumpwent(TALLOC_CTX *mem_ctx, return ENOMEM; } - sctx->enumeration = true; - if (expression) sctx->expression = expression; else @@ -320,225 +327,6 @@ int sysdb_enumpwent(TALLOC_CTX *mem_ctx, /* groups */ -struct get_mem_ctx { - struct sysdb_search_ctx *ret_sctx; - struct ldb_message **grps; - int num_grps; -}; - -static void get_members(struct sysdb_search_ctx *sctx) -{ - struct get_mem_ctx *gmctx; - struct ldb_request *req; - struct ldb_message *msg; - struct ldb_dn *dn; - static const char *attrs[] = SYSDB_GRPW_ATTRS; - int ret; - - gmctx = sctx->gmctx; - - if (gmctx->grps[0] == NULL) { - return request_done(sctx); - } - - /* fetch next group to search for members */ - gmctx->num_grps--; - msg = gmctx->grps[gmctx->num_grps]; - gmctx->grps[gmctx->num_grps] = NULL; - - /* queue the group entry on the final result structure */ - sctx->res->msgs = talloc_realloc(sctx->res, sctx->res->msgs, - struct ldb_message *, - sctx->res->count + 2); - if (!sctx->res->msgs) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - sctx->res->msgs[sctx->res->count + 1] = NULL; - sctx->res->msgs[sctx->res->count] = talloc_steal(sctx->res->msgs, msg); - sctx->res->count++; - - /* search for this group members */ - sctx->expression = talloc_asprintf(sctx, SYSDB_GRNA2_FILTER, - ldb_dn_get_linearized(msg->dn)); - if (!sctx->expression) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - - dn = ldb_dn_new_fmt(sctx, sctx->ctx->ldb, - SYSDB_TMPL_USER_BASE, sctx->domain->name); - if (!dn) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - - sctx->gen_aux_fn = get_members; - - ret = ldb_build_search_req(&req, sctx->ctx->ldb, sctx, - dn, LDB_SCOPE_SUBTREE, - sctx->expression, attrs, NULL, - sctx, get_gen_callback, - NULL); - if (ret != LDB_SUCCESS) { - return request_ldberror(sctx, ret); - } - - ret = ldb_request(sctx->ctx->ldb, req); - if (ret != LDB_SUCCESS) { - return request_ldberror(sctx, ret); - } -} - -static void match_group_members(struct sysdb_search_ctx *sctx); -static void enum_members(struct sysdb_search_ctx *sctx) -{ - static const char *attrs[] = SYSDB_GRENT_ATTRS; - struct ldb_request *req; - struct ldb_dn *dn; - int ret; - - /* search for all users that have memberof set */ - sctx->expression = talloc_asprintf(sctx, SYSDB_GRNA2_FILTER, "*"); - if (!sctx->expression) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - - dn = ldb_dn_new_fmt(sctx, sctx->ctx->ldb, - SYSDB_TMPL_USER_BASE, sctx->domain->name); - if (!dn) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - - sctx->gen_aux_fn = match_group_members; - - ret = ldb_build_search_req(&req, sctx->ctx->ldb, sctx, - dn, LDB_SCOPE_SUBTREE, - sctx->expression, attrs, NULL, - sctx, get_gen_callback, - NULL); - if (ret != LDB_SUCCESS) { - return request_ldberror(sctx, ret); - } - - ret = ldb_request(sctx->ctx->ldb, req); - if (ret != LDB_SUCCESS) { - return request_ldberror(sctx, ret); - } -} - -static void match_group_members(struct sysdb_search_ctx *sctx) -{ - struct get_mem_ctx *gmctx; - struct ldb_message **users; - size_t num_users; - size_t res_idx, grp_idx, i; - const char *grp_dn; - - gmctx = sctx->gmctx; - - /* we have groups in gmctx->grps, and users in res->msgs - * now we need to create a new set where we have each group - * followed by pointers to its users */ - users = sctx->res->msgs; - num_users = sctx->res->count; - - /* allocate initial storage all in one go */ - sctx->res->count = gmctx->num_grps + num_users; - sctx->res->msgs = talloc_array(sctx->res, struct ldb_message *, - sctx->res->count + 1); - if (!sctx->res->msgs) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - - res_idx = 0; - for (grp_idx = 0; grp_idx < gmctx->num_grps; grp_idx++) { - - /* store the group first */ - - if (res_idx == sctx->res->count) { - sctx->res->count += 10; /* allocate 10 at a time */ - sctx->res->msgs = talloc_realloc(sctx->res, sctx->res->msgs, - struct ldb_message *, - sctx->res->count + 1); - if (!sctx->res->msgs) { - return request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - } - } - - sctx->res->msgs[res_idx] = gmctx->grps[grp_idx]; - res_idx++; - - grp_dn = ldb_dn_get_linearized(gmctx->grps[grp_idx]->dn); - - /* now search for the members */ - for (i = 0; i < num_users; i++) { - struct ldb_message_element *el; - struct ldb_val *val; - int j; - - el = ldb_msg_find_element(users[i], SYSDB_MEMBEROF); - for (j = 0; j < el->num_values; j++) { - val = &(el->values[j]); - /* HACK: dn comparisons should be made with ldb_dn_compare() but - * that function requires slow conversions and memory - * allocations. ATM all DNs we use internally should be safe to - * compare directly in a case-insensitive manner */ - if (strncasecmp(grp_dn, (char *)val->data, val->length) != 0) { - continue; - } - - /* ok users belong to this group */ - if (res_idx == sctx->res->count) { - sctx->res->count += 10; /* allocate 10 at a time */ - sctx->res->msgs = talloc_realloc(sctx->res, - sctx->res->msgs, - struct ldb_message *, - sctx->res->count + 1); - if (!sctx->res->msgs) { - return request_ldberror(sctx, - LDB_ERR_OPERATIONS_ERROR); - } - } - - sctx->res->msgs[res_idx] = users[i]; - res_idx++; - - /* now remove value so that we do not parse it again, and - * completely remove the user if it has no more values */ - if (el->num_values == 1) { - - /* make sure to remove memberof as we messed it up */ - ldb_msg_remove_element(users[i], el); - - /* remove user from list by swapping it out and replacing - * it with the last on the list and shortening the list */ - num_users--; - if (i == num_users) { - users[i] = NULL; - } else { - users[i] = users[num_users]; - users[num_users] = NULL; - /* need to rewind or this user won't be checked */ - i--; - } - - } else { - /* still more values, just swap out this one */ - el->num_values--; - if (j != el->num_values) { - /* swap last in */ - el->values[j] = el->values[el->num_values]; - } - } - } - } - } - - /* count may be larger as we allocate in chuncks of 10, - * make sure we report back the real size */ - sctx->res->count = res_idx; - - return request_done(sctx); -} - static int mpg_convert(struct ldb_message *msg) { struct ldb_message_element *el; @@ -567,106 +355,6 @@ static int mpg_convert(struct ldb_message *msg) return EOK; } -static int get_grp_callback(struct ldb_request *req, - struct ldb_reply *rep) -{ - struct sysdb_search_ctx *sctx; - struct ldb_result *res; - int n, ret; - - sctx = talloc_get_type(req->context, struct sysdb_search_ctx); - res = sctx->res; - - if (!rep) { - request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - return LDB_ERR_OPERATIONS_ERROR; - } - if (rep->error != LDB_SUCCESS) { - request_ldberror(sctx, rep->error); - return rep->error; - } - - switch (rep->type) { - case LDB_REPLY_ENTRY: - - if (sctx->ctx->mpg) { - ret = mpg_convert(rep->message); - if (ret != EOK) { - request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - return LDB_ERR_OPERATIONS_ERROR; - } - } - - res->msgs = talloc_realloc(res, res->msgs, - struct ldb_message *, - res->count + 2); - if (!res->msgs) { - request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - return LDB_ERR_OPERATIONS_ERROR; - } - - res->msgs[res->count + 1] = NULL; - - res->msgs[res->count] = talloc_steal(res->msgs, rep->message); - res->count++; - break; - - case LDB_REPLY_REFERRAL: - if (res->refs) { - for (n = 0; res->refs[n]; n++) /*noop*/ ; - } else { - n = 0; - } - - res->refs = talloc_realloc(res, res->refs, char *, n + 2); - if (! res->refs) { - request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - return LDB_ERR_OPERATIONS_ERROR; - } - - res->refs[n] = talloc_steal(res->refs, rep->referral); - res->refs[n + 1] = NULL; - break; - - case LDB_REPLY_DONE: - res->controls = talloc_steal(res, rep->controls); - - /* no results, return */ - if (res->count == 0) { - request_done(sctx); - return LDB_SUCCESS; - } - - if (res->count > 0) { - - sctx->gmctx = talloc_zero(req, struct get_mem_ctx); - if (!sctx->gmctx) { - request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - return LDB_ERR_OPERATIONS_ERROR; - } - sctx->gmctx->grps = res->msgs; - sctx->gmctx->num_grps = res->count; - res->msgs = NULL; - res->count = 0; - - /* now get members */ - if (sctx->enumeration) { - enum_members(sctx); - } else { - get_members(sctx); - } - return LDB_SUCCESS; - } - - /* anything else is an error */ - request_ldberror(sctx, LDB_ERR_OPERATIONS_ERROR); - return LDB_ERR_OPERATIONS_ERROR; - } - - talloc_free(rep); - return LDB_SUCCESS; -} - static void grp_search(struct tevent_req *treq) { struct sysdb_search_ctx *sctx; @@ -682,7 +370,7 @@ static void grp_search(struct tevent_req *treq) return request_error(sctx, ret); } - if (sctx->ctx->mpg) { + if (sctx->gen_conv_mpg_users) { base_dn = ldb_dn_new_fmt(sctx, sctx->ctx->ldb, SYSDB_DOM_BASE, sctx->domain->name); } else { @@ -696,7 +384,7 @@ static void grp_search(struct tevent_req *treq) ret = ldb_build_search_req(&req, sctx->ctx->ldb, sctx, base_dn, LDB_SCOPE_SUBTREE, sctx->expression, attrs, NULL, - sctx, get_grp_callback, + sctx, get_gen_callback, NULL); if (ret != LDB_SUCCESS) { return request_ldberror(sctx, ret); @@ -727,6 +415,7 @@ int sysdb_getgrnam(TALLOC_CTX *mem_ctx, } if (ctx->mpg) { + sctx->gen_conv_mpg_users = true; sctx->expression = talloc_asprintf(sctx, SYSDB_GRNAM_MPG_FILTER, name); } else { sctx->expression = talloc_asprintf(sctx, SYSDB_GRNAM_FILTER, name); @@ -766,6 +455,7 @@ int sysdb_getgrgid(TALLOC_CTX *mem_ctx, } if (ctx->mpg) { + sctx->gen_conv_mpg_users = true; sctx->expression = talloc_asprintf(sctx, SYSDB_GRGID_MPG_FILTER, (unsigned long int)gid); @@ -807,9 +497,8 @@ int sysdb_enumgrent(TALLOC_CTX *mem_ctx, return ENOMEM; } - sctx->enumeration = true; - if (ctx->mpg) { + sctx->gen_conv_mpg_users = true; sctx->expression = SYSDB_GRENT_MPG_FILTER; } else { sctx->expression = SYSDB_GRENT_FILTER; -- cgit