3 files changed, 91 insertions, 14 deletions

diff --git a/src/providers/ldap/ldap_auth.c b/src/providers/ldap/ldap_auth.c
index 83b175dc..d2840d6c 100644
--- a/src/providers/ldap/ldap_auth.c
+++ b/src/providers/ldap/ldap_auth.c
@@ -478,6 +478,7 @@ struct auth_state {
     struct fo_server *srv;
 };
 
+static struct tevent_req *auth_get_server(struct tevent_req *req);
 static void auth_resolve_done(struct tevent_req *subreq);
 static void auth_connect_done(struct tevent_req *subreq);
 static void auth_get_user_dn_done(struct tevent_req *subreq);
@@ -489,7 +490,7 @@ static struct tevent_req *auth_send(TALLOC_CTX *memctx,
                                     const char *username,
                                     struct dp_opt_blob password)
 {
-    struct tevent_req *req, *subreq;
+    struct tevent_req *req;
     struct auth_state *state;
 
     req = tevent_req_create(memctx, &state, struct auth_state);
@@ -501,10 +502,7 @@ static struct tevent_req *auth_send(TALLOC_CTX *memctx,
     state->password = password;
     state->srv = NULL;
 
-    subreq = be_resolve_server_send(state, ev, ctx->be, ctx->service->name);
-    if (!subreq) goto fail;
-
-    tevent_req_set_callback(subreq, auth_resolve_done, req);
+    if (!auth_get_server(req)) goto fail;
 
     return req;
 
@@ -513,6 +511,27 @@ fail:
     return NULL;
 }
 
+static struct tevent_req *auth_get_server(struct tevent_req *req)
+{
+    struct tevent_req *next_req;
+    struct auth_state *state = tevent_req_data(req,
+                                               struct auth_state);
+
+     /* NOTE: this call may cause service->uri to be refreshed
+      * with a new valid server. Do not use service->uri before */
+    next_req = be_resolve_server_send(state,
+                                      state->ev,
+                                      state->ctx->be,
+                                      state->ctx->service->name);
+    if (!next_req) {
+        DEBUG(1, ("be_resolve_server_send failed.\n"));
+        return NULL;
+    }
+
+    tevent_req_set_callback(next_req, auth_resolve_done, req);
+    return next_req;
+}
+
 static void auth_resolve_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
@@ -524,7 +543,9 @@ static void auth_resolve_done(struct tevent_req *subreq)
     ret = be_resolve_server_recv(subreq, &state->srv);
     talloc_zfree(subreq);
     if (ret) {
-        tevent_req_error(req, ret);
+        /* all servers have been tried and none
+         * was found good, go offline */
+        tevent_req_error(req, EIO);
         return;
     }
 
@@ -553,6 +574,12 @@ static void auth_connect_done(struct tevent_req *subreq)
             /* mark this server as bad if connection failed */
             fo_set_port_status(state->srv, PORT_NOT_WORKING);
         }
+        if (ret == ETIMEDOUT) {
+            if (auth_get_server(req) == NULL) {
+                tevent_req_error(req, ENOMEM);
+            }
+            return;
+        }
 
         tevent_req_error(req, ret);
         return;
diff --git a/src/providers/ldap/sdap_async.c b/src/providers/ldap/sdap_async.c
index 5135cb47..20828d2b 100644
--- a/src/providers/ldap/sdap_async.c
+++ b/src/providers/ldap/sdap_async.c
@@ -790,7 +790,11 @@ struct tevent_req *sdap_get_generic_send(TALLOC_CTX *memctx,
                            false, NULL, NULL, NULL, 0, &msgid);
     if (lret != LDAP_SUCCESS) {
         DEBUG(3, ("ldap_search_ext failed: %s\n", ldap_err2string(lret)));
-        ret = EIO;
+        if (lret == LDAP_SERVER_DOWN) {
+            ret = ETIMEDOUT;
+        } else {
+            ret = EIO;
+        }
         goto fail;
     }
     DEBUG(8, ("ldap_search_ext called, msgid = %d\n", msgid));
diff --git a/src/providers/ldap/sdap_async_connection.c b/src/providers/ldap/sdap_async_connection.c
index ec36a601..17cd5586 100644
--- a/src/providers/ldap/sdap_async_connection.c
+++ b/src/providers/ldap/sdap_async_connection.c
@@ -822,6 +822,7 @@ struct sdap_cli_connect_state {
     struct fo_server *srv;
 };
 
+static int sdap_cli_resolve_next(struct tevent_req *req);
 static void sdap_cli_resolve_done(struct tevent_req *subreq);
 static void sdap_cli_connect_done(struct tevent_req *subreq);
 static void sdap_cli_rootdse_step(struct tevent_req *req);
@@ -838,8 +839,9 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
                                          struct sdap_service *service,
                                          struct sysdb_attrs **rootdse)
 {
-    struct tevent_req *req, *subreq;
     struct sdap_cli_connect_state *state;
+    struct tevent_req *req;
+    int ret;
 
     req = tevent_req_create(memctx, &state, struct sdap_cli_connect_state);
     if (!req) return NULL;
@@ -847,6 +849,7 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
     state->ev = ev;
     state->opts = opts;
     state->service = service;
+    state->be = be;
     state->srv = NULL;
 
     if (rootdse) {
@@ -857,16 +860,30 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
         state->rootdse = NULL;
     }
 
+    ret = sdap_cli_resolve_next(req);
+    if (ret) {
+        tevent_req_error(req, ret);
+        tevent_req_post(req, ev);
+    }
+    return req;
+}
+
+static int sdap_cli_resolve_next(struct tevent_req *req)
+{
+    struct sdap_cli_connect_state *state = tevent_req_data(req,
+                                             struct sdap_cli_connect_state);
+    struct tevent_req *subreq;
+
     /* NOTE: this call may cause service->uri to be refreshed
      * with a new valid server. Do not use service->uri before */
-    subreq = be_resolve_server_send(state, ev, be, service->name);
+    subreq = be_resolve_server_send(state, state->ev,
+                                    state->be, state->service->name);
     if (!subreq) {
-        talloc_zfree(req);
-        return NULL;
+        return ENOMEM;
     }
-    tevent_req_set_callback(subreq, sdap_cli_resolve_done, req);
 
-    return req;
+    tevent_req_set_callback(subreq, sdap_cli_resolve_done, req);
+    return EOK;
 }
 
 static void sdap_cli_resolve_done(struct tevent_req *subreq)
@@ -889,7 +906,7 @@ static void sdap_cli_resolve_done(struct tevent_req *subreq)
     subreq = sdap_connect_send(state, state->ev, state->opts,
                                state->service->uri,
                                dp_opt_get_bool(state->opts->basic,
-                                                            SDAP_ID_TLS));
+                                               SDAP_ID_TLS));
     if (!subreq) {
         tevent_req_error(req, ENOMEM);
         return;
@@ -909,6 +926,15 @@ static void sdap_cli_connect_done(struct tevent_req *subreq)
     ret = sdap_connect_recv(subreq, state, &state->sh);
     talloc_zfree(subreq);
     if (ret) {
+        if (ret == ETIMEDOUT) { /* retry another server */
+            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            ret = sdap_cli_resolve_next(req);
+            if (ret != EOK) {
+                tevent_req_error(req, ret);
+            }
+            return;
+        }
+
         tevent_req_error(req, ret);
         return;
     }
@@ -978,6 +1004,15 @@ static void sdap_cli_rootdse_done(struct tevent_req *subreq)
     ret = sdap_get_rootdse_recv(subreq, state, &state->rootdse);
     talloc_zfree(subreq);
     if (ret) {
+        if (ret == ETIMEDOUT) { /* retry another server */
+            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            ret = sdap_cli_resolve_next(req);
+            if (ret != EOK) {
+                tevent_req_error(req, ret);
+            }
+            return;
+        }
+
         tevent_req_error(req, ret);
         return;
     }
@@ -1030,12 +1065,23 @@ static void sdap_cli_kinit_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
                                                       struct tevent_req);
+    struct sdap_cli_connect_state *state = tevent_req_data(req,
+                                             struct sdap_cli_connect_state);
     enum sdap_result result;
     int ret;
 
     ret = sdap_kinit_recv(subreq, &result);
     talloc_zfree(subreq);
     if (ret) {
+        if (ret == ETIMEDOUT) { /* child timed out, retry another server */
+            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            ret = sdap_cli_resolve_next(req);
+            if (ret != EOK) {
+                tevent_req_error(req, ret);
+            }
+            return;
+        }
+
         tevent_req_error(req, ret);
         return;
     }