LDAP: Move the ldap enum request to its own reusable module

The LDAP enumeration was too closely tied to the LDAP identity provider.
Because some providers might need special handling such as refresh the
master domain record before proceeding with the enumeration itself, this
patch splits the request itself to a separate async request and lets the
ldap_id_enum.c module only configure this new request.

Also move the enum timestamp to sdap_domain to make the enum tracking
per sdap domain. The cleanup timestamp will be moved in another patch.

1 files changed, 6 insertions, 631 deletions

diff --git a/src/providers/ldap/ldap_id_enum.c b/src/providers/ldap/ldap_id_enum.c
index 7d9ff3be2..43eb4c997 100644
--- a/src/providers/ldap/ldap_id_enum.c
+++ b/src/providers/ldap/ldap_id_enum.c
@@ -31,6 +31,7 @@
 #include "providers/ldap/ldap_common.h"
 #include "providers/ldap/sdap_async.h"
 #include "providers/ldap/sdap_idmap.h"
+#include "providers/ldap/sdap_async_enum.h"
 
 extern struct tevent_req *ldap_id_cleanup_send(TALLOC_CTX *memctx,
                                                struct tevent_context *ev,
@@ -38,9 +39,6 @@ extern struct tevent_req *ldap_id_cleanup_send(TALLOC_CTX *memctx,
 
 /* ==Enumeration-Task===================================================== */
 
-static int ldap_id_enumerate_retry(struct tevent_req *req);
-static void ldap_id_enumerate_connect_done(struct tevent_req *req);
-
 static void ldap_id_enumerate_reschedule(struct tevent_req *req);
 
 static void ldap_id_enumerate_timeout(struct tevent_context *ev,
@@ -66,7 +64,7 @@ static void ldap_id_enumerate_timer(struct tevent_context *ev,
         return;
     }
 
-    req = ldap_id_enumerate_send(ev, ctx);
+    req = sdap_dom_enum_send(ctx, ev, ctx, ctx->opts->sdom, ctx->conn);
     if (!req) {
         DEBUG(1, ("Failed to schedule enumeration, retrying later!\n"));
         /* schedule starting from now, not the last run */
@@ -129,30 +127,19 @@ static void ldap_id_enumerate_reschedule(struct tevent_req *req)
 {
     struct sdap_id_ctx *ctx = tevent_req_callback_data(req,
                                                        struct sdap_id_ctx);
-    enum tevent_req_state tstate;
-    uint64_t err;
     struct timeval tv;
     int delay;
     errno_t ret;
 
-    if (tevent_req_is_error(req, &tstate, &err)) {
+    ret = sdap_dom_enum_recv(req);
+    talloc_zfree(req);
+    if (ret != EOK) {
         /* On error schedule starting from now, not the last run */
         tv = tevent_timeval_current();
     } else {
-        tv = ctx->last_enum;
+        tv = ctx->opts->sdom->last_enum;
 
-        /* Ok, we've completed an enumeration. Save this to the
-         * sysdb so we can postpone starting up the enumeration
-         * process on the next SSSD service restart (to avoid
-         * slowing down system boot-up
-         */
-        ret = sysdb_set_enumerated(ctx->be->domain->sysdb, ctx->be->domain, true);
-        if (ret != EOK) {
-            DEBUG(1, ("Could not mark domain as having enumerated.\n"));
-            /* This error is non-fatal, so continue */
-        }
     }
-    talloc_zfree(req);
 
     delay = dp_opt_get_int(ctx->opts->basic, SDAP_ENUM_REFRESH_TIMEOUT);
     tv = tevent_timeval_add(&tv, delay, 0);
@@ -175,615 +162,3 @@ int ldap_id_enumerate_set_timer(struct sdap_id_ctx *ctx, struct timeval tv)
 
     return EOK;
 }
-
-struct global_enum_state {
-    struct tevent_context *ev;
-    struct sdap_id_ctx *ctx;
-    struct sdap_id_op *op;
-
-    bool purge;
-};
-
-static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
-                                          struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx,
-                                          struct sdap_domain *sdom,
-                                          struct sdap_id_op *op,
-                                          bool purge);
-static errno_t enum_users_recv(struct tevent_req *req);
-static void ldap_id_enum_users_done(struct tevent_req *subreq);
-static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
-                                          struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx,
-                                          struct sdap_domain *sdom,
-                                          struct sdap_id_op *op,
-                                          bool purge);
-static errno_t enum_groups_recv(struct tevent_req *req);
-static void ldap_id_enum_groups_done(struct tevent_req *subreq);
-static void ldap_id_enum_services_done(struct tevent_req *subreq);
-static void ldap_id_enum_cleanup_done(struct tevent_req *subreq);
-
-struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx)
-{
-    struct global_enum_state *state;
-    struct tevent_req *req;
-    int t;
-
-    req = tevent_req_create(ctx, &state, struct global_enum_state);
-    if (!req) return NULL;
-
-    state->ev = ev;
-    state->ctx = ctx;
-    state->op = sdap_id_op_create(state, state->ctx->conn->conn_cache);
-    if (!state->op) {
-        DEBUG(2, ("sdap_id_op_create failed\n"));
-        talloc_zfree(req);
-        return NULL;
-    }
-
-    ctx->last_enum = tevent_timeval_current();
-
-    t = dp_opt_get_int(ctx->opts->basic, SDAP_CACHE_PURGE_TIMEOUT);
-    if ((ctx->last_purge.tv_sec + t) < ctx->last_enum.tv_sec) {
-        state->purge = true;
-    } else {
-        state->purge = false;
-    }
-
-    int ret = ldap_id_enumerate_retry(req);
-    if (ret != EOK) {
-        DEBUG(2, ("ldap_id_enumerate_retry failed\n"));
-        talloc_zfree(req);
-        return NULL;
-    }
-
-    return req;
-}
-
-static int ldap_id_enumerate_retry(struct tevent_req *req)
-{
-    struct global_enum_state *state = tevent_req_data(req,
-                                                      struct global_enum_state);
-    struct tevent_req *subreq;
-    int ret;
-
-    subreq = sdap_id_op_connect_send(state->op, state, &ret);
-    if (!subreq) {
-        return ret;
-    }
-
-    tevent_req_set_callback(subreq, ldap_id_enumerate_connect_done, req);
-    return EOK;
-}
-
-static void ldap_id_enumerate_connect_done(struct tevent_req *subreq)
-{
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    struct global_enum_state *state = tevent_req_data(req,
-                                                 struct global_enum_state);
-    int ret, dp_error;
-
-    ret = sdap_id_op_connect_recv(subreq, &dp_error);
-    talloc_zfree(subreq);
-    if (ret != EOK) {
-        if (dp_error == DP_ERR_OFFLINE) {
-            tevent_req_done(req);
-        } else {
-            DEBUG(9, ("User enumeration failed to connect to LDAP server: (%d)[%s]\n",
-                      ret, strerror(ret)));
-            tevent_req_error(req, ret);
-        }
-
-        return;
-    }
-
-    subreq = enum_users_send(state, state->ev,
-                             state->ctx, state->ctx->opts->sdom,
-                             state->op, state->purge);
-    if(!subreq) {
-        tevent_req_error(req, ENOMEM);
-        return;
-    }
-
-    tevent_req_set_callback(subreq, ldap_id_enum_users_done, req);
-}
-
-static void ldap_id_enum_users_done(struct tevent_req *subreq)
-{
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    struct global_enum_state *state = tevent_req_data(req,
-                                                 struct global_enum_state);
-    uint64_t err = 0;
-    int ret, dp_error = DP_ERR_FATAL;
-
-    err = enum_users_recv(subreq);
-    talloc_zfree(subreq);
-    if (err != EOK && err != ENOENT) {
-        /* We call sdap_id_op_done only on error
-         * as the connection is reused by groups enumeration */
-        ret = sdap_id_op_done(state->op, (int)err, &dp_error);
-        if (dp_error == DP_ERR_OK) {
-            /* retry */
-            ret = ldap_id_enumerate_retry(req);
-            if (ret == EOK) {
-                return;
-            }
-
-            dp_error = DP_ERR_FATAL;
-        }
-
-        if (dp_error == DP_ERR_OFFLINE) {
-            tevent_req_done(req);
-        } else {
-            DEBUG(9, ("User enumeration failed with: (%d)[%s]\n",
-                      ret, strerror(ret)));
-            tevent_req_error(req, ret);
-        }
-        return;
-    }
-
-    subreq = enum_groups_send(state, state->ev, state->ctx,
-                              state->ctx->opts->sdom,
-                              state->op, state->purge);
-    if (!subreq) {
-        tevent_req_error(req, ENOMEM);
-        return;
-    }
-    tevent_req_set_callback(subreq, ldap_id_enum_groups_done, req);
-}
-
-static void ldap_id_enum_groups_done(struct tevent_req *subreq)
-{
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    struct global_enum_state *state = tevent_req_data(req,
-                                                 struct global_enum_state);
-    uint64_t err = 0;
-    int ret, dp_error = DP_ERR_FATAL;
-
-    err = enum_groups_recv(subreq);
-    talloc_zfree(subreq);
-    if (err != EOK && err != ENOENT) {
-        /* We call sdap_id_op_done only on error
-         * as the connection is reused by services enumeration */
-        ret = sdap_id_op_done(state->op, (int)err, &dp_error);
-        if (dp_error == DP_ERR_OK && ret != EOK) {
-            /* retry */
-            ret = ldap_id_enumerate_retry(req);
-            if (ret == EOK) {
-                return;
-            }
-
-            dp_error = DP_ERR_FATAL;
-        }
-
-        if (ret != EOK) {
-            if (dp_error == DP_ERR_OFFLINE) {
-                tevent_req_done(req);
-            } else {
-                DEBUG(9, ("Group enumeration failed with: (%d)[%s]\n",
-                          ret, strerror(ret)));
-                tevent_req_error(req, ret);
-            }
-
-            return;
-        }
-    }
-
-    subreq = enum_services_send(state, state->ev, state->ctx,
-                                state->op, state->purge);
-    if (!subreq) {
-        tevent_req_error(req, ENOMEM);
-        return;
-    }
-    tevent_req_set_callback(subreq, ldap_id_enum_services_done, req);
-}
-
-static void ldap_id_enum_services_done(struct tevent_req *subreq)
-{
-    errno_t ret;
-    int dp_error = DP_ERR_FATAL;
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    struct global_enum_state *state = tevent_req_data(req,
-                                                 struct global_enum_state);
-
-    ret = enum_services_recv(subreq);
-    talloc_zfree(subreq);
-    if (ret == ENOENT) ret = EOK;
-
-    /* All enumerations are complete, so conclude the
-     * id_op
-     */
-    ret = sdap_id_op_done(state->op, ret, &dp_error);
-    if (dp_error == DP_ERR_OK && ret != EOK) {
-        /* retry */
-        ret = ldap_id_enumerate_retry(req);
-        if (ret == EOK) {
-            return;
-        }
-
-        dp_error = DP_ERR_FATAL;
-    }
-
-    if (ret != EOK) {
-        if (dp_error == DP_ERR_OFFLINE) {
-            tevent_req_done(req);
-        } else {
-            DEBUG(SSSDBG_MINOR_FAILURE,
-                  ("Service enumeration failed with: (%d)[%s]\n",
-                   ret, strerror(ret)));
-            tevent_req_error(req, ret);
-        }
-
-        return;
-    }
-
-    if (state->purge) {
-
-        subreq = ldap_id_cleanup_send(state, state->ev, state->ctx);
-        if (!subreq) {
-            tevent_req_error(req, ENOMEM);
-            return;
-        }
-
-        tevent_req_set_callback(subreq, ldap_id_enum_cleanup_done, req);
-        return;
-    }
-
-    tevent_req_done(req);
-}
-
-static void ldap_id_enum_cleanup_done(struct tevent_req *subreq)
-{
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    talloc_zfree(subreq);
-    tevent_req_done(req);
-}
-
-/* ==User-Enumeration===================================================== */
-
-struct enum_users_state {
-    struct tevent_context *ev;
-    struct sdap_id_ctx *ctx;
-    struct sdap_domain *sdom;
-    struct sdap_id_op *op;
-
-    char *filter;
-    const char **attrs;
-};
-
-static void enum_users_op_done(struct tevent_req *subreq);
-
-static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
-                                          struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx,
-                                          struct sdap_domain *sdom,
-                                          struct sdap_id_op *op,
-                                          bool purge)
-{
-    struct tevent_req *req, *subreq;
-    struct enum_users_state *state;
-    int ret;
-    bool use_mapping;
-
-    req = tevent_req_create(memctx, &state, struct enum_users_state);
-    if (!req) return NULL;
-
-    state->ev = ev;
-    state->sdom = sdom;
-    state->ctx = ctx;
-    state->op = op;
-
-    use_mapping = sdap_idmap_domain_has_algorithmic_mapping(
-                                                          ctx->opts->idmap_ctx,
-                                                          sdom->dom->domain_id);
-
-    /* We always want to filter on objectclass and an available name */
-    state->filter = talloc_asprintf(state,
-                                    "(&(objectclass=%s)(%s=*)",
-                                    ctx->opts->user_map[SDAP_OC_USER].name,
-                                    ctx->opts->user_map[SDAP_AT_USER_NAME].name);
-    if (!state->filter) {
-        DEBUG(SSSDBG_MINOR_FAILURE,
-              ("Failed to build base filter\n"));
-        ret = ENOMEM;
-        goto fail;
-    }
-
-    if (use_mapping) {
-        /* If we're ID-mapping, check for the objectSID as well */
-        state->filter = talloc_asprintf_append_buffer(
-                state->filter, "(%s=*)",
-                ctx->opts->user_map[SDAP_AT_USER_OBJECTSID].name);
-    } else {
-        /* We're not ID-mapping, so make sure to only get entries
-         * that have UID and GID
-         */
-        state->filter = talloc_asprintf_append_buffer(
-                state->filter, "(%s=*)(%s=*)",
-                ctx->opts->user_map[SDAP_AT_USER_UID].name,
-                ctx->opts->user_map[SDAP_AT_USER_GID].name);
-    }
-    if (!state->filter) {
-        DEBUG(SSSDBG_MINOR_FAILURE,
-              ("Failed to build base filter\n"));
-        ret = ENOMEM;
-        goto fail;
-    }
-
-    if (ctx->srv_opts && ctx->srv_opts->max_user_value && !purge) {
-        /* If we have lastUSN available and we're not doing a full
-         * refresh, limit to changes with a higher entryUSN value.
-         */
-        state->filter = talloc_asprintf_append_buffer(
-                state->filter,
-                "(%s>=%s)(!(%s=%s))",
-                ctx->opts->user_map[SDAP_AT_USER_USN].name,
-                ctx->srv_opts->max_user_value,
-                ctx->opts->user_map[SDAP_AT_USER_USN].name,
-                ctx->srv_opts->max_user_value);
-
-        if (!state->filter) {
-            DEBUG(SSSDBG_MINOR_FAILURE,
-                  ("Failed to build base filter\n"));
-            ret = ENOMEM;
-            goto fail;
-        }
-    }
-
-    /* Terminate the search filter */
-    state->filter = talloc_asprintf_append_buffer(state->filter, ")");
-    if (!state->filter) {
-        DEBUG(2, ("Failed to build base filter\n"));
-        ret = ENOMEM;
-        goto fail;
-    }
-
-    /* TODO: handle attrs_type */
-    ret = build_attrs_from_map(state, ctx->opts->user_map, SDAP_OPTS_USER,
-                               NULL, &state->attrs, NULL);
-    if (ret != EOK) goto fail;
-
-    /* TODO: restrict the enumerations to using a single
-     * search base at a time.
-     */
-
-    subreq = sdap_get_users_send(state, state->ev,
-                                 state->sdom->dom,
-                                 state->sdom->dom->sysdb,
-                                 state->ctx->opts,
-                                 state->sdom->user_search_bases,
-                                 sdap_id_op_handle(state->op),
-                                 state->attrs, state->filter,
-                                 dp_opt_get_int(state->ctx->opts->basic,
-                                                SDAP_ENUM_SEARCH_TIMEOUT),
-                                 true);
-    if (!subreq) {
-        ret = ENOMEM;
-        goto fail;
-    }
-    tevent_req_set_callback(subreq, enum_users_op_done, req);
-
-    return req;
-
-fail:
-    tevent_req_error(req, ret);
-    tevent_req_post(req, ev);
-    return req;
-}
-
-static void enum_users_op_done(struct tevent_req *subreq)
-{
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    struct enum_users_state *state = tevent_req_data(req,
-                                                     struct enum_users_state);
-    char *usn_value;
-    char *endptr = NULL;
-    unsigned usn_number;
-    int ret;
-
-    ret = sdap_get_users_recv(subreq, state, &usn_value);
-    talloc_zfree(subreq);
-    if (ret) {
-        tevent_req_error(req, ret);
-        return;
-    }
-
-    if (usn_value) {
-        talloc_zfree(state->ctx->srv_opts->max_user_value);
-        state->ctx->srv_opts->max_user_value = talloc_steal(state->ctx, usn_value);
-
-        usn_number = strtoul(usn_value, &endptr, 10);
-        if ((endptr == NULL || (*endptr == '\0' && endptr != usn_value))
-            && (usn_number > state->ctx->srv_opts->last_usn)) {
-            state->ctx->srv_opts->last_usn = usn_number;
-        }
-    }
-
-    DEBUG(4, ("Users higher USN value: [%s]\n",
-              state->ctx->srv_opts->max_user_value));
-
-    tevent_req_done(req);
-}
-
-static errno_t enum_users_recv(struct tevent_req *req)
-{
-    TEVENT_REQ_RETURN_ON_ERROR(req);
-
-    return EOK;
-}
-
-/* =Group-Enumeration===================================================== */
-
-struct enum_groups_state {
-    struct tevent_context *ev;
-    struct sdap_id_ctx *ctx;
-    struct sdap_domain *sdom;
-    struct sdap_id_op *op;
-
-    char *filter;
-    const char **attrs;
-};
-
-static void enum_groups_op_done(struct tevent_req *subreq);
-
-static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
-                                          struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx,
-                                          struct sdap_domain *sdom,
-                                          struct sdap_id_op *op,
-                                          bool purge)
-{
-    struct tevent_req *req, *subreq;
-    struct enum_groups_state *state;
-    int ret;
-    bool use_mapping;
-
-    req = tevent_req_create(memctx, &state, struct enum_groups_state);
-    if (!req) return NULL;
-
-    state->ev = ev;
-    state->sdom = sdom;
-    state->ctx = ctx;
-    state->op = op;
-
-    use_mapping = sdap_idmap_domain_has_algorithmic_mapping(
-                                                          ctx->opts->idmap_ctx,
-                                                          sdom->dom->domain_id);
-
-    /* We always want to filter on objectclass and an available name */
-    state->filter = talloc_asprintf(state,
-                                    "(&(objectclass=%s)(%s=*)",
-                                    ctx->opts->group_map[SDAP_OC_GROUP].name,
-                                    ctx->opts->group_map[SDAP_AT_GROUP_NAME].name);
-    if (!state->filter) {
-        DEBUG(SSSDBG_MINOR_FAILURE,
-              ("Failed to build base filter\n"));
-        ret = ENOMEM;
-        goto fail;
-    }
-
-    if (use_mapping) {
-        /* If we're ID-mapping, check for the objectSID as well */
-        state->filter = talloc_asprintf_append_buffer(
-                state->filter, "(%s=*)",
-                ctx->opts->group_map[SDAP_AT_GROUP_OBJECTSID].name);
-    } else {
-        /* We're not ID-mapping, so make sure to only get entries
-         * that have a non-zero GID.
-         */
-        state->filter = talloc_asprintf_append_buffer(
-                state->filter, "(&(%s=*)(!(%s=0)))",
-                ctx->opts->group_map[SDAP_AT_GROUP_GID].name,
-                ctx->opts->group_map[SDAP_AT_GROUP_GID].name);
-    }
-    if (!state->filter) {
-        DEBUG(SSSDBG_MINOR_FAILURE,
-              ("Failed to build base filter\n"));
-        ret = ENOMEM;
-        goto fail;
-    }
-
-    if (ctx->srv_opts && ctx->srv_opts->max_group_value && !purge) {
-        state->filter = talloc_asprintf_append_buffer(
-                state->filter,
-                "(%s>=%s)(!(%s=%s))",
-                ctx->opts->group_map[SDAP_AT_GROUP_USN].name,
-                ctx->srv_opts->max_group_value,
-                ctx->opts->group_map[SDAP_AT_GROUP_USN].name,
-                ctx->srv_opts->max_group_value);
-        if (!state->filter) {
-            DEBUG(SSSDBG_MINOR_FAILURE,
-                  ("Failed to build base filter\n"));
-            ret = ENOMEM;
-            goto fail;
-        }
-    }
-
-    /* Terminate the search filter */
-    state->filter = talloc_asprintf_append_buffer(state->filter, ")");
-    if (!state->filter) {
-        DEBUG(SSSDBG_MINOR_FAILURE,
-              ("Failed to build base filter\n"));
-        ret = ENOMEM;
-        goto fail;
-    }
-
-    /* TODO: handle attrs_type */
-    ret = build_attrs_from_map(state, ctx->opts->group_map, SDAP_OPTS_GROUP,
-                               NULL, &state->attrs, NULL);
-    if (ret != EOK) goto fail;
-
-    /* TODO: restrict the enumerations to using a single
-     * search base at a time.
-     */
-
-    subreq = sdap_get_groups_send(state, state->ev,
-                                  state->sdom,
-                                  state->ctx->opts,
-                                  sdap_id_op_handle(state->op),
-                                  state->attrs, state->filter,
-                                  dp_opt_get_int(state->ctx->opts->basic,
-                                                 SDAP_ENUM_SEARCH_TIMEOUT),
-                                  true);
-    if (!subreq) {
-        ret = ENOMEM;
-        goto fail;
-    }
-    tevent_req_set_callback(subreq, enum_groups_op_done, req);
-
-    return req;
-
-fail:
-    tevent_req_error(req, ret);
-    tevent_req_post(req, ev);
-    return req;
-}
-
-static void enum_groups_op_done(struct tevent_req *subreq)
-{
-    struct tevent_req *req = tevent_req_callback_data(subreq,
-                                                      struct tevent_req);
-    struct enum_groups_state *state = tevent_req_data(req,
-                                                 struct enum_groups_state);
-    char *usn_value;
-    char *endptr = NULL;
-    unsigned usn_number;
-    int ret;
-
-    ret = sdap_get_groups_recv(subreq, state, &usn_value);
-    talloc_zfree(subreq);
-    if (ret) {
-        tevent_req_error(req, ret);
-        return;
-    }
-
-    if (usn_value) {
-        talloc_zfree(state->ctx->srv_opts->max_group_value);
-        state->ctx->srv_opts->max_group_value =
-                                        talloc_steal(state->ctx, usn_value);
-        usn_number = strtoul(usn_value, &endptr, 10);
-        if ((endptr == NULL || (*endptr == '\0' && endptr != usn_value))
-            && (usn_number > state->ctx->srv_opts->last_usn)) {
-            state->ctx->srv_opts->last_usn = usn_number;
-        }
-    }
-
-    DEBUG(4, ("Groups higher USN value: [%s]\n",
-              state->ctx->srv_opts->max_group_value));
-
-    tevent_req_done(req);
-}
-
-static errno_t enum_groups_recv(struct tevent_req *req)
-{
-    TEVENT_REQ_RETURN_ON_ERROR(req);
-
-    return EOK;
-}