Fix Ldap id backend offline code

After the recent changes we lost the capability to actually go offline.
Put back code that would mark the backend as offline when timeouts happen.
Make sure the enumeration code also obbeys the offline timeout, and
contributes in determining if we are offline or not.

2 files changed, 83 insertions, 14 deletions

diff --git a/server/providers/ldap/ldap_id.c b/server/providers/ldap/ldap_id.c
index 5da21cd5e..efd9e914d 100644
--- a/server/providers/ldap/ldap_id.c
+++ b/server/providers/ldap/ldap_id.c
@@ -58,12 +58,22 @@ static bool is_offline(struct sdap_id_ctx *ctx)
 {
     time_t now = time(NULL);
 
+    /* check if we are past the offline blackout timeout */
     if (ctx->went_offline + ctx->opts->offline_timeout < now) {
-        return false;
+        ctx->offline = false;
     }
+
     return ctx->offline;
 }
 
+static void mark_offline(struct sdap_id_ctx *ctx)
+{
+    DEBUG(8, ("Going offline!\n"));
+
+    ctx->went_offline = time(NULL);
+    ctx->offline = true;
+}
+
 static void sdap_check_online(struct be_req *req)
 {
     struct be_online_req *oreq;
@@ -376,16 +386,26 @@ static void users_get_op_done(struct tevent_req *subreq)
 static void users_get_done(struct tevent_req *req)
 {
     struct be_req *breq = tevent_req_callback_data(req, struct be_req);
+    struct sdap_id_ctx *ctx;
     enum tevent_req_state tstate;
     uint64_t err;
     const char *error = NULL;
     int ret = EOK;
 
+
     if (tevent_req_is_error(req, &tstate, &err)) {
         ret = err;
     }
 
-    if (ret) error = "Enum Users Failed";
+    if (ret) {
+        error = "Enum Users Failed";
+
+        if (ret == ETIMEDOUT) {
+            ctx = talloc_get_type(breq->be_ctx->bet_info[BET_ID].pvt_bet_data,
+                                  struct sdap_id_ctx);
+            mark_offline(ctx);
+        }
+    }
 
     return sdap_req_done(breq, ret, error);
 }
@@ -532,6 +552,7 @@ static void groups_get_op_done(struct tevent_req *subreq)
 static void groups_get_done(struct tevent_req *req)
 {
     struct be_req *breq = tevent_req_callback_data(req, struct be_req);
+    struct sdap_id_ctx *ctx;
     enum tevent_req_state tstate;
     uint64_t err;
     const char *error = NULL;
@@ -541,7 +562,15 @@ static void groups_get_done(struct tevent_req *req)
         ret = err;
     }
 
-    if (ret) error = "Enum Groups Failed";
+    if (ret) {
+        error = "Enum Groups Failed";
+
+        if (ret == ETIMEDOUT) {
+            ctx = talloc_get_type(breq->be_ctx->bet_info[BET_ID].pvt_bet_data,
+                                  struct sdap_id_ctx);
+            mark_offline(ctx);
+        }
+    }
 
     return sdap_req_done(breq, ret, error);
 }
@@ -663,6 +692,7 @@ static void groups_by_user_op_done(struct tevent_req *subreq)
 static void groups_by_user_done(struct tevent_req *req)
 {
     struct be_req *breq = tevent_req_callback_data(req, struct be_req);
+    struct sdap_id_ctx *ctx;
     enum tevent_req_state tstate;
     uint64_t err;
     const char *error = NULL;
@@ -672,7 +702,15 @@ static void groups_by_user_done(struct tevent_req *req)
         ret = err;
     }
 
-    if (ret) error = "Init Groups Failed";
+    if (ret) {
+        error = "Init Groups Failed";
+
+        if (ret == ETIMEDOUT) {
+            ctx = talloc_get_type(breq->be_ctx->bet_info[BET_ID].pvt_bet_data,
+                                  struct sdap_id_ctx);
+            mark_offline(ctx);
+        }
+    }
 
     return sdap_req_done(breq, ret, error);
 }
@@ -792,6 +830,13 @@ static void ldap_id_enumerate(struct tevent_context *ev,
     struct tevent_timer *timeout;
     struct tevent_req *req;
 
+    if (is_offline(ctx)) {
+        DEBUG(4, ("Backend is marked offline, retry later!\n"));
+        /* schedule starting from now, not the last run */
+        ldap_id_enumerate_set_timer(ctx, tevent_timeval_current());
+        return;
+    }
+
     ctx->last_run = tv;
 
     req = ldap_id_enumerate_send(ev, ctx);
@@ -894,7 +939,7 @@ static void ldap_id_enum_users_done(struct tevent_req *subreq)
     struct global_enum_state *state = tevent_req_data(req,
                                                  struct global_enum_state);
     enum tevent_req_state tstate;
-    uint64_t err;
+    uint64_t err = 0;
 
     if (tevent_req_is_error(subreq, &tstate, &err)) {
         goto fail;
@@ -910,6 +955,15 @@ static void ldap_id_enum_users_done(struct tevent_req *subreq)
     return;
 
 fail:
+    if (err) {
+        DEBUG(9, ("User enumeration failed with: (%d)[%s]\n",
+                  (int)err, strerror(err)));
+
+        if (err == ETIMEDOUT) {
+            mark_offline(state->ctx);
+        }
+    }
+
     DEBUG(1, ("Failed to enumerate users, retrying later!\n"));
     /* schedule starting from now, not the last run */
     ldap_id_enumerate_set_timer(state->ctx, tevent_timeval_current());
@@ -936,6 +990,10 @@ static void ldap_id_enum_groups_done(struct tevent_req *subreq)
     return;
 
 fail:
+    if (err == ETIMEDOUT) {
+        mark_offline(state->ctx);
+    }
+
     DEBUG(1, ("Failed to enumerate groups, retrying later!\n"));
     /* schedule starting from now, not the last run */
     ldap_id_enumerate_set_timer(state->ctx, tevent_timeval_current());
@@ -1118,7 +1176,7 @@ static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
     state->ev = ev;
     state->ctx = ctx;
 
-        attr_name = ctx->opts->group_map[SDAP_AT_GROUP_NAME].name;
+    attr_name = ctx->opts->group_map[SDAP_AT_GROUP_NAME].name;
 
     if (ctx->max_group_timestamp) {
         state->filter = talloc_asprintf(state,
diff --git a/server/providers/ldap/sdap_async.c b/server/providers/ldap/sdap_async.c
index 550cb9d3f..15985fffc 100644
--- a/server/providers/ldap/sdap_async.c
+++ b/server/providers/ldap/sdap_async.c
@@ -372,7 +372,7 @@ static void sdap_op_timeout(struct tevent_req *req)
     }
 
     /* signal the caller that we have a timeout */
-    op->callback(op, NULL, ETIME, op->data);
+    op->callback(op, NULL, ETIMEDOUT, op->data);
 
     /* send back to the server an abandon (see destructor) and free the op */
     talloc_free(op);
@@ -534,7 +534,7 @@ fail:
         tevent_req_error(req, ret);
     } else {
         if (lret == LDAP_SERVER_DOWN) {
-            tevent_req_error(req, EAGAIN);
+            tevent_req_error(req, ETIMEDOUT);
         } else {
             tevent_req_error(req, EIO);
         }
@@ -635,6 +635,7 @@ static struct tevent_req *simple_bind_send(TALLOC_CTX *memctx,
     struct simple_bind_state *state;
     int ret = EOK;
     int msgid;
+    int ldap_err;
 
     req = tevent_req_create(memctx, &state, struct simple_bind_state);
     if (!req) return NULL;
@@ -655,7 +656,16 @@ static struct tevent_req *simple_bind_send(TALLOC_CTX *memctx,
     ret = ldap_sasl_bind(state->sh->ldap, state->user_dn, LDAP_SASL_SIMPLE,
                          state->pw, NULL, NULL, &msgid);
     if (ret == -1 || msgid == -1) {
-        DEBUG(1, ("ldap_bind failed\n"));
+        ret = ldap_get_option(state->sh->ldap,
+                              LDAP_OPT_RESULT_CODE, &ldap_err);
+        if (ret != LDAP_OPT_SUCCESS) {
+            DEBUG(1, ("ldap_bind failed (couldn't get ldap error)\n"));
+            ret = LDAP_LOCAL_ERROR;
+        } else {
+            DEBUG(1, ("ldap_bind failed (%d)[%s]\n",
+                      ldap_err, ldap_err2string(ldap_err)));
+            ret = ldap_err;
+        }
         goto fail;
     }
     DEBUG(8, ("ldap simple bind sent, msgid = %d\n", msgid));
@@ -678,7 +688,7 @@ static struct tevent_req *simple_bind_send(TALLOC_CTX *memctx,
 
 fail:
     if (ret == LDAP_SERVER_DOWN) {
-        tevent_req_error(req, EAGAIN);
+        tevent_req_error(req, ETIMEDOUT);
     } else {
         tevent_req_error(req, EIO);
     }
@@ -726,11 +736,12 @@ static int simple_bind_recv(struct tevent_req *req, int *ldaperr)
 
     if (tevent_req_is_error(req, &tstate, &err)) {
         *ldaperr = LDAP_OTHER;
-        return -1;
+        if (err) return err;
+        return EIO;
     }
 
     *ldaperr = state->result;
-    return 0;
+    return EOK;
 }
 
 /* ==Authenticaticate-User-by-DN========================================== */
@@ -790,8 +801,8 @@ static void sdap_auth_done(struct tevent_req *subreq)
     int ret;
 
     ret = simple_bind_recv(subreq, &state->result);
-    if (ret == -1) {
-        tevent_req_error(req, EFAULT);
+    if (ret != EOK) {
+        tevent_req_error(req, ret);
         return;
     }
     tevent_req_done(req);