Only do one cycle when resolving a server

7 files changed, 132 insertions, 59 deletions

diff --git a/src/providers/data_provider_fo.c b/src/providers/data_provider_fo.c
index 8d7cbbfd2..ada41a3ac 100644
--- a/src/providers/data_provider_fo.c
+++ b/src/providers/data_provider_fo.c
@@ -46,6 +46,7 @@ struct be_svc_data {
     bool run_callbacks;
 
     struct be_svc_callback *callbacks;
+    struct fo_server *first_resolved;
 };
 
 struct be_failover_ctx {
@@ -342,6 +343,7 @@ struct be_resolve_server_state {
     int attempts;
 
     struct fo_server *srv;
+    bool first_try;
 };
 
 static void be_resolve_server_done(struct tevent_req *subreq);
@@ -349,7 +351,8 @@ static void be_resolve_server_done(struct tevent_req *subreq);
 struct tevent_req *be_resolve_server_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
                                           struct be_ctx *ctx,
-                                          const char *service_name)
+                                          const char *service_name,
+                                          bool first_try)
 {
     struct tevent_req *req, *subreq;
     struct be_resolve_server_state *state;
@@ -370,6 +373,7 @@ struct tevent_req *be_resolve_server_send(TALLOC_CTX *memctx,
 
     state->svc = svc;
     state->attempts = 0;
+    state->first_try = first_try;
 
     subreq = fo_resolve_service_send(state, ev,
                                      ctx->be_fo->resolv,
@@ -399,42 +403,43 @@ static void be_resolve_server_done(struct tevent_req *subreq)
     switch (ret) {
     case EOK:
         if (!state->srv) {
-            tevent_req_error(req, EFAULT);
-            return;
+            ret = EFAULT;
+            goto fail;
         }
         break;
 
     case ENOENT:
         /* all servers have been tried and none
          * was found good, go offline */
-        tevent_req_error(req, EIO);
-        return;
+        ret = EIO;
+        goto fail;
 
     default:
         /* mark server as bad and retry */
         if (!state->srv) {
-            tevent_req_error(req, EFAULT);
-            return;
+            ret = EFAULT;
+            goto fail;
         }
-        DEBUG(6, ("Couldn't resolve server (%s), resolver returned (%d)\n",
-                  fo_get_server_str_name(state->srv), ret));
+        DEBUG(SSSDBG_MINOR_FAILURE,
+              ("Couldn't resolve server (%s), resolver returned (%d)\n",
+              fo_get_server_str_name(state->srv), ret));
 
         state->attempts++;
         if (state->attempts >= 10) {
             DEBUG(2, ("Failed to find a server after 10 attempts\n"));
-            tevent_req_error(req, EIO);
-            return;
+            ret = EIO;
+            goto fail;
         }
 
         /* now try next one */
-        DEBUG(6, ("Trying with the next one!\n"));
+        DEBUG(SSSDBG_TRACE_LIBS, ("Trying with the next one!\n"));
         subreq = fo_resolve_service_send(state, state->ev,
                                          state->ctx->be_fo->resolv,
                                          state->ctx->be_fo->fo_ctx,
                                          state->svc->fo_service);
         if (!subreq) {
-            tevent_req_error(req, ENOMEM);
-            return;
+            ret = ENOMEM;
+            goto fail;
         }
         tevent_req_set_callback(subreq, be_resolve_server_done, req);
 
@@ -442,24 +447,34 @@ static void be_resolve_server_done(struct tevent_req *subreq)
     }
 
     /* all fine we got the server */
+    if (state->svc->first_resolved == NULL || state->first_try == true) {
+        DEBUG(SSSDBG_TRACE_LIBS, ("Saving the first resolved server\n"));
+        state->svc->first_resolved = state->srv;
+    } else if (state->svc->first_resolved == state->srv) {
+        DEBUG(SSSDBG_OP_FAILURE,
+              ("The fail over cycled through all available servers\n"));
+        ret = ENOENT;
+        goto fail;
+    }
 
-    if (DEBUG_IS_SET(SSSDBG_CONF_SETTINGS) && fo_get_server_name(state->srv)) {
+    if (DEBUG_IS_SET(SSSDBG_FUNC_DATA) && fo_get_server_name(state->srv)) {
         struct resolv_hostent *srvaddr;
         char ipaddr[128];
         srvaddr = fo_get_server_hostent(state->srv);
         if (!srvaddr) {
-            DEBUG(3, ("FATAL: No hostent available for server (%s)\n",
-                      fo_get_server_str_name(state->srv)));
-            tevent_req_error(req, EFAULT);
-            return;
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  ("FATAL: No hostent available for server (%s)\n",
+                  fo_get_server_str_name(state->srv)));
+            ret = EFAULT;
+            goto fail;
         }
 
         inet_ntop(srvaddr->family, srvaddr->addr_list[0]->ipaddr,
                   ipaddr, 128);
 
-        DEBUG(4, ("Found address for server %s: [%s] TTL %d\n",
-                fo_get_server_str_name(state->srv), ipaddr,
-                srvaddr->addr_list[0]->ttl));
+        DEBUG(SSSDBG_FUNC_DATA, ("Found address for server %s: [%s] TTL %d\n",
+              fo_get_server_str_name(state->srv), ipaddr,
+              srvaddr->addr_list[0]->ttl));
     }
 
     srv_status_change = fo_get_server_hostname_last_change(state->srv);
@@ -479,6 +494,12 @@ static void be_resolve_server_done(struct tevent_req *subreq)
     }
 
     tevent_req_done(req);
+    return;
+
+fail:
+    DEBUG(SSSDBG_TRACE_LIBS, ("Server resolution failed: %d\n", ret));
+    state->svc->first_resolved = NULL;
+    tevent_req_error(req, ret);
 }
 
 int be_resolve_server_recv(struct tevent_req *req, struct fo_server **srv)
@@ -525,3 +546,33 @@ void reset_fo(struct be_ctx *be_ctx)
     fo_reset_services(be_ctx->be_fo->fo_ctx);
 }
 
+void be_fo_set_port_status(struct be_ctx *ctx,
+                           struct fo_server *server,
+                           enum port_status status)
+{
+    struct be_svc_data *svc;
+    struct fo_service *fsvc;
+
+    fo_set_port_status(server, status);
+
+    fsvc = fo_get_server_service(server);
+    if (!fsvc) {
+        DEBUG(SSSDBG_OP_FAILURE, ("BUG: No service associated with server\n"));
+        return;
+    }
+
+    DLIST_FOR_EACH(svc, ctx->be_fo->svcs) {
+        if (svc->fo_service == fsvc) break;
+    }
+
+    if (!svc) {
+        DEBUG(SSSDBG_OP_FAILURE, ("BUG: Unknown service\n"));
+        return;
+    }
+
+    if (status == PORT_WORKING) {
+        /* We were successful in connecting to the server. Cycle through all
+         * available servers next time */
+        svc->first_resolved = NULL;
+    }
+}
diff --git a/src/providers/dp_backend.h b/src/providers/dp_backend.h
index b6b3db50a..6e98e7ef3 100644
--- a/src/providers/dp_backend.h
+++ b/src/providers/dp_backend.h
@@ -207,8 +207,14 @@ int be_fo_add_server(struct be_ctx *ctx, const char *service_name,
 struct tevent_req *be_resolve_server_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
                                           struct be_ctx *ctx,
-                                          const char *service_name);
+                                          const char *service_name,
+                                          bool first_try);
 int be_resolve_server_recv(struct tevent_req *req, struct fo_server **srv);
+
+void be_fo_set_port_status(struct be_ctx *ctx,
+                           struct fo_server *server,
+                           enum port_status status);
+
 /*
  * Instruct fail-over to try next server on the next connect attempt.
  * Should be used after connection to service was unexpectedly dropped
diff --git a/src/providers/fail_over.c b/src/providers/fail_over.c
index 637b5b4d3..b83106840 100644
--- a/src/providers/fail_over.c
+++ b/src/providers/fail_over.c
@@ -1476,3 +1476,10 @@ void fo_reset_services(struct fo_ctx *fo_ctx)
     }
 }
 
+struct fo_service *
+fo_get_server_service(struct fo_server *server)
+{
+    if (!server) return NULL;
+    return server->service;
+}
+
diff --git a/src/providers/fail_over.h b/src/providers/fail_over.h
index fee64e82c..8fbbe251b 100644
--- a/src/providers/fail_over.h
+++ b/src/providers/fail_over.h
@@ -185,4 +185,7 @@ time_t fo_get_server_hostname_last_change(struct fo_server *server);
 int fo_is_srv_lookup(struct fo_server *s);
 
 void fo_reset_services(struct fo_ctx *fo_ctx);
+
+struct fo_service *fo_get_server_service(struct fo_server *server);
+
 #endif /* !__FAIL_OVER_H__ */
diff --git a/src/providers/krb5/krb5_auth.c b/src/providers/krb5/krb5_auth.c
index 14a7c547f..0306426cc 100644
--- a/src/providers/krb5/krb5_auth.c
+++ b/src/providers/krb5/krb5_auth.c
@@ -320,6 +320,9 @@ int krb5_auth_recv(struct tevent_req *req, int *pam_status, int *dp_err)
     return EOK;
 }
 
+static struct tevent_req *krb5_next_kdc(struct tevent_req *req);
+static struct tevent_req *krb5_next_kpasswd(struct tevent_req *req);
+
 struct tevent_req *krb5_auth_send(TALLOC_CTX *mem_ctx,
                                   struct tevent_context *ev,
                                   struct be_ctx *be_ctx,
@@ -507,16 +510,14 @@ struct tevent_req *krb5_auth_send(TALLOC_CTX *mem_ctx,
 
     kr->srv = NULL;
     kr->kpasswd_srv = NULL;
-    subreq = be_resolve_server_send(state, state->ev, state->be_ctx,
-                                    krb5_ctx->service->name);
-    if (subreq == NULL) {
-        DEBUG(1, ("be_resolve_server_send failed.\n"));
-        ret = ENOMEM;
+
+    subreq = krb5_next_kdc(req);
+    if (!subreq) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("krb5_next_kdc failed.\n"));
+        ret = EIO;
         goto done;
     }
 
-    tevent_req_set_callback(subreq, krb5_resolve_kdc_done, req);
-
     return req;
 
 done:
@@ -557,16 +558,12 @@ static void krb5_resolve_kdc_done(struct tevent_req *subreq)
         }
     } else {
         if (kr->krb5_ctx->kpasswd_service != NULL) {
-            subreq = be_resolve_server_send(state, state->ev, state->be_ctx,
-                                            kr->krb5_ctx->kpasswd_service->name);
+            subreq = krb5_next_kpasswd(req);
             if (subreq == NULL) {
-                DEBUG(1, ("be_resolve_server_send failed.\n"));
-                ret = ENOMEM;
+                DEBUG(SSSDBG_CRIT_FAILURE, ("krb5_next_kpasswd failed.\n"));
+                ret = EIO;
                 goto failed;
             }
-
-            tevent_req_set_callback(subreq, krb5_resolve_kpasswd_done, req);
-
             return;
         }
     }
@@ -718,7 +715,6 @@ done:
 }
 
 static struct tevent_req *krb5_next_server(struct tevent_req *req);
-static struct tevent_req *krb5_next_kdc(struct tevent_req *req);
 static struct tevent_req *krb5_next_kpasswd(struct tevent_req *req);
 
 static void krb5_child_done(struct tevent_req *subreq)
@@ -883,14 +879,16 @@ static void krb5_child_done(struct tevent_req *subreq)
     if (kr->kpasswd_srv != NULL) {
         /* ..which is unreachable by now.. */
         if (msg_status == PAM_AUTHTOK_LOCK_BUSY) {
-            fo_set_port_status(kr->kpasswd_srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be_ctx,
+                                  kr->kpasswd_srv, PORT_NOT_WORKING);
             /* ..try to resolve next kpasswd server */
             if (krb5_next_kpasswd(req) == NULL) {
                 tevent_req_error(req, ENOMEM);
             }
             return;
         } else {
-            fo_set_port_status(kr->kpasswd_srv, PORT_WORKING);
+            be_fo_set_port_status(state->be_ctx,
+                                  kr->kpasswd_srv, PORT_WORKING);
         }
     }
 
@@ -900,7 +898,7 @@ static void krb5_child_done(struct tevent_req *subreq)
     if (msg_status == PAM_AUTHINFO_UNAVAIL ||
         (kr->kpasswd_srv == NULL && msg_status == PAM_AUTHTOK_LOCK_BUSY)) {
         if (kr->srv != NULL) {
-            fo_set_port_status(kr->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be_ctx, kr->srv, PORT_NOT_WORKING);
             /* ..try to resolve next KDC */
             if (krb5_next_kdc(req) == NULL) {
                 tevent_req_error(req, ENOMEM);
@@ -908,7 +906,7 @@ static void krb5_child_done(struct tevent_req *subreq)
             return;
         }
     } else if (kr->srv != NULL) {
-        fo_set_port_status(kr->srv, PORT_WORKING);
+        be_fo_set_port_status(state->be_ctx, kr->srv, PORT_WORKING);
     }
 
     /* Now only a successful authentication or password change is left.
@@ -971,17 +969,20 @@ static struct tevent_req *krb5_next_server(struct tevent_req *req)
     switch (pd->cmd) {
         case SSS_PAM_AUTHENTICATE:
         case SSS_CMD_RENEW:
-            fo_set_port_status(state->kr->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be_ctx,
+                                  state->kr->srv, PORT_NOT_WORKING);
             next_req = krb5_next_kdc(req);
             break;
         case SSS_PAM_CHAUTHTOK:
         case SSS_PAM_CHAUTHTOK_PRELIM:
             if (state->kr->kpasswd_srv) {
-                fo_set_port_status(state->kr->kpasswd_srv, PORT_NOT_WORKING);
+                be_fo_set_port_status(state->be_ctx,
+                                      state->kr->kpasswd_srv, PORT_NOT_WORKING);
                 next_req = krb5_next_kpasswd(req);
                 break;
             } else {
-                fo_set_port_status(state->kr->srv, PORT_NOT_WORKING);
+                be_fo_set_port_status(state->be_ctx,
+                                      state->kr->srv, PORT_NOT_WORKING);
                 next_req = krb5_next_kdc(req);
                 break;
             }
@@ -999,7 +1000,8 @@ static struct tevent_req *krb5_next_kdc(struct tevent_req *req)
 
     next_req = be_resolve_server_send(state, state->ev,
                                       state->be_ctx,
-                                      state->krb5_ctx->service->name);
+                                      state->krb5_ctx->service->name,
+                                      state->kr->srv == NULL ? true : false);
     if (next_req == NULL) {
         DEBUG(1, ("be_resolve_server_send failed.\n"));
         return NULL;
@@ -1016,7 +1018,8 @@ static struct tevent_req *krb5_next_kpasswd(struct tevent_req *req)
 
     next_req = be_resolve_server_send(state, state->ev,
                                 state->be_ctx,
-                                state->krb5_ctx->kpasswd_service->name);
+                                state->krb5_ctx->kpasswd_service->name,
+                                state->kr->kpasswd_srv == NULL ? true : false);
     if (next_req == NULL) {
         DEBUG(1, ("be_resolve_server_send failed.\n"));
         return NULL;
diff --git a/src/providers/ldap/ldap_auth.c b/src/providers/ldap/ldap_auth.c
index 170a34b94..734249ced 100644
--- a/src/providers/ldap/ldap_auth.c
+++ b/src/providers/ldap/ldap_auth.c
@@ -517,7 +517,8 @@ static struct tevent_req *auth_get_server(struct tevent_req *req)
     next_req = be_resolve_server_send(state,
                                       state->ev,
                                       state->ctx->be,
-                                      state->sdap_service->name);
+                                      state->sdap_service->name,
+                                      state->srv == NULL ? true : false);
     if (!next_req) {
         DEBUG(1, ("be_resolve_server_send failed.\n"));
         return NULL;
@@ -588,7 +589,8 @@ static void auth_connect_done(struct tevent_req *subreq)
     if (ret) {
         if (state->srv) {
             /* mark this server as bad if connection failed */
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->ctx->be,
+                                  state->srv, PORT_NOT_WORKING);
         }
         if (ret == ETIMEDOUT) {
             if (auth_get_server(req) == NULL) {
@@ -600,7 +602,7 @@ static void auth_connect_done(struct tevent_req *subreq)
         tevent_req_error(req, ret);
         return;
     } else if (state->srv) {
-        fo_set_port_status(state->srv, PORT_WORKING);
+        be_fo_set_port_status(state->ctx->be, state->srv, PORT_WORKING);
     }
 
     ret = get_user_dn(state, state->ctx->be->sysdb, state->ctx->opts,
diff --git a/src/providers/ldap/sdap_async_connection.c b/src/providers/ldap/sdap_async_connection.c
index ca1cd6e12..2b7f8c93d 100644
--- a/src/providers/ldap/sdap_async_connection.c
+++ b/src/providers/ldap/sdap_async_connection.c
@@ -884,7 +884,8 @@ static struct tevent_req *sdap_kinit_next_kdc(struct tevent_req *req)
 
     next_req = be_resolve_server_send(state, state->ev,
                                       state->be,
-                                      state->krb_service_name);
+                                      state->krb_service_name,
+                                      state->kdc_srv == NULL ? true : false);
     if (next_req == NULL) {
         DEBUG(1, ("be_resolve_server_send failed.\n"));
         return NULL;
@@ -962,7 +963,7 @@ static void sdap_kinit_done(struct tevent_req *subreq)
         return;
     } else {
         if (kerr == KRB5_KDC_UNREACH) {
-            fo_set_port_status(state->kdc_srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->kdc_srv, PORT_NOT_WORKING);
             nextreq = sdap_kinit_next_kdc(req);
             if (!nextreq) {
                 tevent_req_error(req, ENOMEM);
@@ -1191,7 +1192,6 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
     state->be = be;
     state->srv = NULL;
     state->srv_opts = NULL;
-    state->be = be;
     state->use_rootdse = !skip_rootdse;
     state->force_tls = force_tls;
     state->do_auth = !skip_auth;
@@ -1216,7 +1216,8 @@ static int sdap_cli_resolve_next(struct tevent_req *req)
     /* NOTE: this call may cause service->uri to be refreshed
      * with a new valid server. Do not use service->uri before */
     subreq = be_resolve_server_send(state, state->ev,
-                                    state->be, state->service->name);
+                                    state->be, state->service->name,
+                                    state->srv == NULL ? true : false);
     if (!subreq) {
         return ENOMEM;
     }
@@ -1290,7 +1291,7 @@ static void sdap_cli_connect_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
     if (ret) {
         /* retry another server */
-        fo_set_port_status(state->srv, PORT_NOT_WORKING);
+        be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
         ret = sdap_cli_resolve_next(req);
         if (ret != EOK) {
             tevent_req_error(req, ret);
@@ -1364,7 +1365,7 @@ static void sdap_cli_rootdse_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
     if (ret) {
         if (ret == ETIMEDOUT) { /* retry another server */
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
             ret = sdap_cli_resolve_next(req);
             if (ret != EOK) {
                 tevent_req_error(req, ret);
@@ -1483,7 +1484,7 @@ static void sdap_cli_kinit_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
     if (ret) {
         if (ret == ETIMEDOUT) { /* child timed out, retry another server */
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
             ret = sdap_cli_resolve_next(req);
             if (ret != EOK) {
                 tevent_req_error(req, ret);
@@ -1585,7 +1586,7 @@ int sdap_cli_connect_recv(struct tevent_req *req,
     if (tevent_req_is_error(req, &tstate, &err)) {
         /* mark the server as bad if connection failed */
         if (state->srv) {
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
         } else {
             if (can_retry) {
                 *can_retry = false;
@@ -1597,7 +1598,7 @@ int sdap_cli_connect_recv(struct tevent_req *req,
         }
         return EIO;
     } else if (state->srv) {
-        fo_set_port_status(state->srv, PORT_WORKING);
+        be_fo_set_port_status(state->be, state->srv, PORT_WORKING);
     }
 
     if (gsh) {