FO: Use SRV TTL in fail over code

Resolves:
https://fedorahosted.org/sssd/ticket/1884

Removes the hardcoded SRV TTL timeout and uses TTL from the DNS instead.

Reviewed-by: Pavel Březina <pbrezina@redhat.com>
(cherry picked from commit 8df69bbc58c2f4d3f0b34be9756d9ddf24b1db6d)

10 files changed, 647 insertions, 9 deletions

diff --git a/src/providers/ad/ad_srv.c b/src/providers/ad/ad_srv.c
index ac9dfa187..910f82dd0 100644
--- a/src/providers/ad/ad_srv.c
+++ b/src/providers/ad/ad_srv.c
@@ -176,7 +176,7 @@ static void ad_get_dc_servers_done(struct tevent_req *subreq)
     req = tevent_req_callback_data(subreq, struct tevent_req);
     state = tevent_req_data(req, struct ad_get_dc_servers_state);
 
-    ret = fo_discover_srv_recv(state, subreq, &domain,
+    ret = fo_discover_srv_recv(state, subreq, &domain, NULL,
                                &state->servers, &state->num_servers);
     talloc_zfree(subreq);
     if (ret != EOK) {
@@ -616,6 +616,7 @@ struct ad_srv_plugin_state {
 
     const char *site;
     char *dns_domain;
+    uint32_t ttl;
     char *forest;
     struct fo_server_info *primary_servers;
     size_t num_primary_servers;
@@ -837,6 +838,7 @@ static void ad_srv_plugin_servers_done(struct tevent_req *subreq)
     state = tevent_req_data(req, struct ad_srv_plugin_state);
 
     ret = fo_discover_servers_recv(state, subreq, &state->dns_domain,
+                                   &state->ttl,
                                    &state->primary_servers,
                                    &state->num_primary_servers,
                                    &state->backup_servers,
@@ -874,6 +876,7 @@ static void ad_srv_plugin_servers_done(struct tevent_req *subreq)
 errno_t ad_srv_plugin_recv(TALLOC_CTX *mem_ctx,
                            struct tevent_req *req,
                            char **_dns_domain,
+                           uint32_t *_ttl,
                            struct fo_server_info **_primary_servers,
                            size_t *_num_primary_servers,
                            struct fo_server_info **_backup_servers,
@@ -904,6 +907,9 @@ errno_t ad_srv_plugin_recv(TALLOC_CTX *mem_ctx,
         *_dns_domain = talloc_steal(mem_ctx, state->dns_domain);
     }
 
+    if (_ttl) {
+        *_ttl = state->ttl;
+    }
 
     return EOK;
 }
diff --git a/src/providers/ad/ad_srv.h b/src/providers/ad/ad_srv.h
index be3ac2826..ae5efe447 100644
--- a/src/providers/ad/ad_srv.h
+++ b/src/providers/ad/ad_srv.h
@@ -42,6 +42,7 @@ struct tevent_req *ad_srv_plugin_send(TALLOC_CTX *mem_ctx,
 errno_t ad_srv_plugin_recv(TALLOC_CTX *mem_ctx,
                             struct tevent_req *req,
                             char **_dns_domain,
+                            uint32_t *_ttl,
                             struct fo_server_info **_primary_servers,
                             size_t *_num_primary_servers,
                             struct fo_server_info **_backup_servers,
diff --git a/src/providers/data_provider_fo.c b/src/providers/data_provider_fo.c
index ef1f0b2ed..dab796d46 100644
--- a/src/providers/data_provider_fo.c
+++ b/src/providers/data_provider_fo.c
@@ -70,7 +70,6 @@ static int be_fo_get_options(struct be_ctx *ctx,
     opts->service_resolv_timeout = dp_opt_get_int(ctx->be_res->opts,
                                                   DP_RES_OPT_RESOLVER_TIMEOUT);
     opts->retry_timeout = 30;
-    opts->srv_retry_timeout = 14400;
     opts->srv_retry_neg_timeout = 15;
     opts->family_order = ctx->be_res->family_order;
 
diff --git a/src/providers/fail_over.c b/src/providers/fail_over.c
index c16c70ddb..814208152 100644
--- a/src/providers/fail_over.c
+++ b/src/providers/fail_over.c
@@ -117,6 +117,7 @@ struct srv_data {
     struct fo_server *meta;
 
     int srv_lookup_status;
+    int ttl;
     struct timeval last_status_change;
 };
 
@@ -149,7 +150,6 @@ fo_context_init(TALLOC_CTX *mem_ctx, struct fo_options *opts)
         return NULL;
     }
 
-    ctx->opts->srv_retry_timeout = opts->srv_retry_timeout;
     ctx->opts->srv_retry_neg_timeout = opts->srv_retry_neg_timeout;
     ctx->opts->retry_timeout = opts->retry_timeout;
     ctx->opts->family_order  = opts->family_order;
@@ -272,7 +272,7 @@ get_srv_data_status(struct srv_data *data)
     if (data->srv_lookup_status == SRV_RESOLVE_ERROR) {
         timeout = data->meta->service->ctx->opts->srv_retry_neg_timeout;
     } else {
-        timeout = data->meta->service->ctx->opts->srv_retry_timeout;
+        timeout = data->ttl;
     }
 
     if (timeout && STATUS_DIFF(data, tv) > timeout) {
@@ -1285,8 +1285,9 @@ resolve_srv_done(struct tevent_req *subreq)
     size_t num_backup_servers = 0;
     char *dns_domain = NULL;
     int ret;
+    uint32_t ttl;
 
-    ret = state->fo_ctx->srv_recv_fn(state, subreq, &dns_domain,
+    ret = state->fo_ctx->srv_recv_fn(state, subreq, &dns_domain, &ttl,
                                      &primary_servers, &num_primary_servers,
                                      &backup_servers, &num_backup_servers);
     talloc_free(subreq);
@@ -1300,6 +1301,7 @@ resolve_srv_done(struct tevent_req *subreq)
             goto done;
         }
 
+        state->meta->srv_data->ttl = ttl;
         talloc_zfree(state->meta->srv_data->dns_domain);
         state->meta->srv_data->dns_domain = talloc_steal(state->meta->srv_data,
                                                          dns_domain);
diff --git a/src/providers/fail_over.h b/src/providers/fail_over.h
index b1ec6a23c..d44ad2ff1 100644
--- a/src/providers/fail_over.h
+++ b/src/providers/fail_over.h
@@ -78,7 +78,6 @@ struct fo_server;
  * try when looking up the service.
  */
 struct fo_options {
-    time_t srv_retry_timeout;
     time_t srv_retry_neg_timeout;
     time_t retry_timeout;
     int service_resolv_timeout;
diff --git a/src/providers/fail_over_srv.c b/src/providers/fail_over_srv.c
index 5c06d2876..5f474eaee 100644
--- a/src/providers/fail_over_srv.c
+++ b/src/providers/fail_over_srv.c
@@ -30,6 +30,7 @@ struct fo_discover_srv_state {
     char *dns_domain;
     struct fo_server_info *servers;
     size_t num_servers;
+    uint32_t ttl;
 };
 
 static void fo_discover_srv_done(struct tevent_req *subreq);
@@ -83,7 +84,7 @@ static void fo_discover_srv_done(struct tevent_req *subreq)
     state = tevent_req_data(req, struct fo_discover_srv_state);
 
     ret = resolv_discover_srv_recv(state, subreq,
-                                   &reply_list, NULL, &state->dns_domain);
+                                   &reply_list, &state->ttl, &state->dns_domain);
     talloc_zfree(subreq);
     if (ret == ENOENT) {
         ret = ERR_SRV_NOT_FOUND;
@@ -143,6 +144,7 @@ done:
 errno_t fo_discover_srv_recv(TALLOC_CTX *mem_ctx,
                              struct tevent_req *req,
                              char **_dns_domain,
+                             uint32_t *_ttl,
                              struct fo_server_info **_servers,
                              size_t *_num_servers)
 {
@@ -159,6 +161,10 @@ errno_t fo_discover_srv_recv(TALLOC_CTX *mem_ctx,
         *_servers = talloc_steal(mem_ctx, state->servers);
     }
 
+    if (_ttl != NULL) {
+        *_ttl = state->ttl;
+    }
+
     if (_num_servers != NULL) {
         *_num_servers = state->num_servers;
     }
@@ -175,6 +181,7 @@ struct fo_discover_servers_state {
     const char *backup_domain;
 
     char *dns_domain;
+    uint32_t ttl;
     struct fo_server_info *primary_servers;
     size_t num_primary_servers;
     struct fo_server_info *backup_servers;
@@ -212,6 +219,7 @@ struct tevent_req *fo_discover_servers_send(TALLOC_CTX *mem_ctx,
             state->backup_servers = NULL;
             state->num_backup_servers = 0;
             state->dns_domain = NULL;
+            state->ttl = 0;
 
             ret = EOK;
             goto immediately;
@@ -289,6 +297,7 @@ static void fo_discover_servers_primary_done(struct tevent_req *subreq)
 
     ret = fo_discover_srv_recv(state, subreq,
                                &state->dns_domain,
+                               &state->ttl,
                                &state->primary_servers,
                                &state->num_primary_servers);
     talloc_zfree(subreq);
@@ -361,7 +370,7 @@ static void fo_discover_servers_backup_done(struct tevent_req *subreq)
     state = tevent_req_data(req, struct fo_discover_servers_state);
 
     ret = fo_discover_srv_recv(state, subreq, NULL,
-                               &state->backup_servers,
+                               NULL, &state->backup_servers,
                                &state->num_backup_servers);
     talloc_zfree(subreq);
     if (ret != EOK) {
@@ -385,6 +394,7 @@ static void fo_discover_servers_backup_done(struct tevent_req *subreq)
 errno_t fo_discover_servers_recv(TALLOC_CTX *mem_ctx,
                                  struct tevent_req *req,
                                  char **_dns_domain,
+                                 uint32_t *_ttl,
                                  struct fo_server_info **_primary_servers,
                                  size_t *_num_primary_servers,
                                  struct fo_server_info **_backup_servers,
@@ -415,6 +425,10 @@ errno_t fo_discover_servers_recv(TALLOC_CTX *mem_ctx,
         *_dns_domain = talloc_steal(mem_ctx, state->dns_domain);
     }
 
+    if (_ttl) {
+        *_ttl = state->ttl;
+    }
+
 
     return EOK;
 }
@@ -436,6 +450,7 @@ struct fo_resolve_srv_dns_state {
     const char *discovery_domain;
 
     char *dns_domain;
+    uint32_t ttl;
     struct fo_server_info *servers;
     size_t num_servers;
 };
@@ -644,7 +659,8 @@ static void fo_resolve_srv_dns_done(struct tevent_req *subreq)
     req = tevent_req_callback_data(subreq, struct tevent_req);
     state = tevent_req_data(req, struct fo_resolve_srv_dns_state);
 
-    ret = fo_discover_srv_recv(state, subreq, &state->dns_domain,
+    ret = fo_discover_srv_recv(state, subreq,
+                               &state->dns_domain, &state->ttl,
                                &state->servers, &state->num_servers);
     talloc_zfree(subreq);
     if (ret != EOK) {
@@ -663,6 +679,7 @@ done:
 errno_t fo_resolve_srv_dns_recv(TALLOC_CTX *mem_ctx,
                                 struct tevent_req *req,
                                 char **_dns_domain,
+                                uint32_t *_ttl,
                                 struct fo_server_info **_primary_servers,
                                 size_t *_num_primary_servers,
                                 struct fo_server_info **_backup_servers,
@@ -694,5 +711,9 @@ errno_t fo_resolve_srv_dns_recv(TALLOC_CTX *mem_ctx,
         *_dns_domain = talloc_steal(mem_ctx, state->dns_domain);
     }
 
+    if (_ttl) {
+        *_ttl = state->ttl;
+    }
+
     return EOK;
 }
diff --git a/src/providers/fail_over_srv.h b/src/providers/fail_over_srv.h
index c4b50ba33..fe4088e02 100644
--- a/src/providers/fail_over_srv.h
+++ b/src/providers/fail_over_srv.h
@@ -65,6 +65,7 @@ typedef errno_t
 (*fo_srv_lookup_plugin_recv_t)(TALLOC_CTX *mem_ctx,
                                struct tevent_req *req,
                                char **_dns_domain,
+                               uint32_t *_ttl,
                                struct fo_server_info **_primary_servers,
                                size_t *_num_primary_servers,
                                struct fo_server_info **_backup_servers,
@@ -80,6 +81,7 @@ struct tevent_req *fo_discover_srv_send(TALLOC_CTX *mem_ctx,
 errno_t fo_discover_srv_recv(TALLOC_CTX *mem_ctx,
                              struct tevent_req *req,
                              char **_dns_domain,
+                             uint32_t *_ttl,
                              struct fo_server_info **_servers,
                              size_t *_num_servers);
 
@@ -94,6 +96,7 @@ struct tevent_req *fo_discover_servers_send(TALLOC_CTX *mem_ctx,
 errno_t fo_discover_servers_recv(TALLOC_CTX *mem_ctx,
                                  struct tevent_req *req,
                                  char **_dns_domain,
+                                 uint32_t *_ttl,
                                  struct fo_server_info **_primary_servers,
                                  size_t *_num_primary_servers,
                                  struct fo_server_info **_backup_servers,
@@ -121,6 +124,7 @@ struct tevent_req *fo_resolve_srv_dns_send(TALLOC_CTX *mem_ctx,
 errno_t fo_resolve_srv_dns_recv(TALLOC_CTX *mem_ctx,
                                 struct tevent_req *req,
                                 char **_dns_domain,
+                                uint32_t *_ttl,
                                 struct fo_server_info **_primary_servers,
                                 size_t *_num_primary_servers,
                                 struct fo_server_info **_backup_servers,
diff --git a/src/providers/ipa/ipa_srv.c b/src/providers/ipa/ipa_srv.c
index 4be5f3317..747771134 100644
--- a/src/providers/ipa/ipa_srv.c
+++ b/src/providers/ipa/ipa_srv.c
@@ -69,6 +69,7 @@ fail:
 
 struct ipa_srv_plugin_state {
     char *dns_domain;
+    uint32_t ttl;
     struct fo_server_info *primary_servers;
     size_t num_primary_servers;
     struct fo_server_info *backup_servers;
@@ -164,6 +165,7 @@ static void ipa_srv_plugin_done(struct tevent_req *subreq)
     state = tevent_req_data(req, struct ipa_srv_plugin_state);
 
     ret = fo_discover_servers_recv(state, subreq, &state->dns_domain,
+                                   &state->ttl,
                                    &state->primary_servers,
                                    &state->num_primary_servers,
                                    &state->backup_servers,
@@ -183,6 +185,7 @@ static void ipa_srv_plugin_done(struct tevent_req *subreq)
 errno_t ipa_srv_plugin_recv(TALLOC_CTX *mem_ctx,
                             struct tevent_req *req,
                             char **_dns_domain,
+                            uint32_t *_ttl,
                             struct fo_server_info **_primary_servers,
                             size_t *_num_primary_servers,
                             struct fo_server_info **_backup_servers,
@@ -213,6 +216,9 @@ errno_t ipa_srv_plugin_recv(TALLOC_CTX *mem_ctx,
         *_dns_domain = talloc_steal(mem_ctx, state->dns_domain);
     }
 
+    if (_ttl) {
+        *_ttl = state->ttl;
+    }
 
     return EOK;
 }
diff --git a/src/providers/ipa/ipa_srv.h b/src/providers/ipa/ipa_srv.h
index 1bfd2eeae..d089c9f64 100644
--- a/src/providers/ipa/ipa_srv.h
+++ b/src/providers/ipa/ipa_srv.h
@@ -39,6 +39,7 @@ struct tevent_req *ipa_srv_plugin_send(TALLOC_CTX *mem_ctx,
 errno_t ipa_srv_plugin_recv(TALLOC_CTX *mem_ctx,
                             struct tevent_req *req,
                             char **_dns_domain,
+                            uint32_t *_ttl,
                             struct fo_server_info **_primary_servers,
                             size_t *_num_primary_servers,
                             struct fo_server_info **_backup_servers,
diff --git a/src/tests/cmocka/test_fo_srv.c b/src/tests/cmocka/test_fo_srv.c
new file mode 100644
index 000000000..3ee308bd5
--- /dev/null
+++ b/src/tests/cmocka/test_fo_srv.c
@@ -0,0 +1,599 @@
+/*
+    Authors:
+        Jakub Hrozek <jhrozek@redhat.com>
+
+    Copyright (C) 2014 Red Hat
+
+    SSSD tests: Resolver tests using a fake resolver library
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <talloc.h>
+#include <tevent.h>
+#include <errno.h>
+#include <popt.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <sys/types.h>
+#include <stdarg.h>
+#include <stdlib.h>
+
+#include "providers/fail_over_srv.h"
+#include "tests/cmocka/common_mock.h"
+#include "tests/cmocka/common_mock_resp.h"
+
+#define TEST_RESOLV_TIMEOUT 5
+#define TEST_FO_TIMEOUT     3000
+#define TEST_SRV_TTL        500
+#define TEST_SRV_SHORT_TTL  2
+
+static TALLOC_CTX *global_mock_context = NULL;
+
+enum host_database default_host_dbs[] = { DB_FILES, DB_DNS, DB_SENTINEL };
+
+struct resolv_ctx {
+    int foo;
+};
+
+/* mock resolver interface. The resolver test is separate */
+int resolv_init(TALLOC_CTX *mem_ctx, struct tevent_context *ev_ctx,
+                int timeout, struct resolv_ctx **ctxp)
+{
+    *ctxp = talloc(mem_ctx, struct resolv_ctx);
+    return EOK;
+}
+
+struct tevent_req *
+resolv_gethostbyname_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+                          struct resolv_ctx *ctx, const char *name,
+                          enum restrict_family family_order,
+                          enum host_database *db)
+{
+    return test_req_succeed_send(mem_ctx, ev);
+}
+
+int resolv_gethostbyname_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx,
+                              int *status, int *timeouts,
+                              struct resolv_hostent **rhostent)
+{
+    return test_request_recv(req);
+}
+
+const char *resolv_strerror(int ares_code)
+{
+    return NULL;
+}
+
+struct tevent_req *resolv_discover_srv_send(TALLOC_CTX *mem_ctx,
+                                            struct tevent_context *ev,
+                                            struct resolv_ctx *resolv_ctx,
+                                            const char *service,
+                                            const char *protocol,
+                                            const char **discovery_domains)
+{
+    return test_req_succeed_send(mem_ctx, ev);
+}
+
+errno_t resolv_discover_srv_recv(TALLOC_CTX *mem_ctx,
+                                 struct tevent_req *req,
+                                 struct ares_srv_reply **_reply_list,
+                                 uint32_t *_ttl,
+                                 char **_dns_domain)
+{
+    struct ares_srv_reply *reply_list;
+    uint32_t ttl;
+    char *dns_domain;
+
+    /* Need to always consume all mocked values */
+    reply_list = sss_mock_ptr_type(struct ares_srv_reply *);
+    ttl = sss_mock_ptr_type(uint32_t);
+    dns_domain = sss_mock_ptr_type(char *);
+
+    if (_reply_list != NULL) {
+        *_reply_list = reply_list;
+    }
+
+    if (_ttl != NULL) {
+        *_ttl = ttl;
+    }
+
+    if (_dns_domain != NULL) {
+        *_dns_domain = dns_domain;
+    }
+
+    return test_request_recv(req);
+}
+
+struct ares_srv_reply *pop_lowest_prio(struct ares_srv_reply **r)
+{
+    struct ares_srv_reply *lowest;
+    struct ares_srv_reply *iter;
+    struct ares_srv_reply *prev;
+
+    lowest = *r;
+    iter = lowest;
+    while (iter != NULL) {
+        if (iter->priority < lowest->priority) {
+            lowest = iter;
+        }
+
+        iter = iter->next;
+    }
+
+    prev = NULL;
+    iter = *r;
+    while (iter != lowest) {
+        prev = iter;
+        iter = iter->next;
+    }
+
+    /* iter points to the lowest prio. Prev points to the item before */
+    if (prev) {
+        prev->next = lowest->next;
+    } else {
+        *r = lowest->next;
+    }
+
+    return lowest;
+}
+
+int resolv_sort_srv_reply(struct ares_srv_reply **reply)
+{
+    struct ares_srv_reply *r;
+    struct ares_srv_reply *lowest;
+    struct ares_srv_reply *sorted = NULL;
+    struct ares_srv_reply *sorted_head = NULL;
+
+    r = *reply;
+    if (r == NULL || r->next == NULL) {
+        return EOK;
+    }
+
+    do {
+        lowest = pop_lowest_prio(&r);
+        if (sorted) {
+            sorted->next = lowest;
+            sorted = sorted->next;
+        } else {
+            sorted = lowest;
+            sorted_head = sorted;
+        }
+    } while (r != NULL);
+
+    *reply = sorted_head;
+    return EOK;
+}
+
+struct tevent_req *resolv_get_domain_send(TALLOC_CTX *mem_ctx,
+                                          struct tevent_context *ev,
+                                          struct resolv_ctx *resolv_ctx,
+                                          const char *hostname,
+                                          enum host_database *host_dbs,
+                                          enum restrict_family family_order)
+{
+    return test_req_succeed_send(mem_ctx, ev);
+}
+
+errno_t resolv_get_domain_recv(TALLOC_CTX *mem_ctx,
+                               struct tevent_req *req,
+                               char **_dns_domain)
+{
+    return test_request_recv(req);
+}
+
+/* The unit test */
+struct test_fo_srv_ctx {
+    struct resolv_ctx *resolv;
+    struct fo_ctx *fo_ctx;
+    struct fo_resolve_srv_dns_ctx *srv_ctx;
+    struct fo_service *fo_svc;
+    struct sss_test_ctx *ctx;
+};
+
+int test_fo_srv_data_cmp(void *ud1, void *ud2)
+{
+    return strcasecmp((char*) ud1, (char*) ud2);
+}
+
+void test_fo_srv_setup(void **state)
+{
+    struct test_fo_srv_ctx *test_ctx;
+    errno_t ret;
+    struct fo_options fopts;
+    bool ok;
+
+    assert_true(leak_check_setup());
+    global_mock_context = talloc_new(global_talloc_context);
+    assert_non_null(global_mock_context);
+
+    test_ctx = talloc_zero(global_mock_context,
+                           struct test_fo_srv_ctx);
+    assert_non_null(test_ctx);
+
+    test_ctx->ctx = create_ev_test_ctx(test_ctx);
+    assert_non_null(test_ctx->ctx);
+
+    ret = resolv_init(test_ctx, test_ctx->ctx->ev,
+                      TEST_RESOLV_TIMEOUT, &test_ctx->resolv);
+    assert_non_null(test_ctx->resolv);
+
+    memset(&fopts, 0, sizeof(fopts));
+    fopts.retry_timeout = TEST_FO_TIMEOUT;
+    fopts.family_order  = IPV4_FIRST;
+
+    test_ctx->fo_ctx = fo_context_init(test_ctx, &fopts);
+    assert_non_null(test_ctx->fo_ctx);
+
+    test_ctx->srv_ctx = fo_resolve_srv_dns_ctx_init(test_ctx, test_ctx->resolv,
+                                                    IPV4_FIRST, default_host_dbs,
+                                                    "client.sssd.com", "sssd.local");
+    assert_non_null(test_ctx->srv_ctx);
+
+    ok = fo_set_srv_lookup_plugin(test_ctx->fo_ctx,
+                                  fo_resolve_srv_dns_send,
+                                  fo_resolve_srv_dns_recv,
+                                  test_ctx->srv_ctx);
+    assert_true(ok);
+
+    ret = fo_new_service(test_ctx->fo_ctx, "ldap",
+                         test_fo_srv_data_cmp,
+                         &test_ctx->fo_svc);
+    assert_int_equal(ret, ERR_OK);
+
+    *state = test_ctx;
+}
+
+void test_fo_srv_teardown(void **state)
+{
+    struct test_fo_srv_ctx *test_ctx =
+        talloc_get_type(*state, struct test_fo_srv_ctx);
+
+    talloc_free(test_ctx);
+    talloc_free(global_mock_context);
+    assert_true(leak_check_teardown());
+}
+
+/* reply_list and dns_domain must be a talloc context so it can be used as
+ * talloc_steal argument later
+ */
+static void mock_srv_results(struct ares_srv_reply *reply_list,
+                             uint32_t ttl,
+                             char *dns_domain)
+{
+    will_return(resolv_discover_srv_recv, reply_list);
+    will_return(resolv_discover_srv_recv, ttl);
+    will_return(resolv_discover_srv_recv, dns_domain);
+}
+
+static void check_server(struct fo_server *srv, int port, const char *name)
+{
+    assert_non_null(srv);
+    assert_true(fo_is_srv_lookup(srv));
+    assert_int_equal(fo_get_server_port(srv), port);
+    assert_string_equal(fo_get_server_name(srv), name);
+}
+
+static void test_fo_srv_done1(struct tevent_req *req);
+static void test_fo_srv_done2(struct tevent_req *req);
+static void test_fo_srv_done3(struct tevent_req *req);
+static void test_fo_srv_done4(struct tevent_req *req);
+
+void test_fo_srv(void **state)
+{
+    errno_t ret;
+    struct tevent_req *req;
+    struct test_fo_srv_ctx *test_ctx =
+        talloc_get_type(*state, struct test_fo_srv_ctx);
+    struct ares_srv_reply *s1;
+    struct ares_srv_reply *s2;
+    char *dns_domain;
+
+    s1 = talloc_zero(test_ctx, struct ares_srv_reply);
+    assert_non_null(s1);
+    s1->host = talloc_strdup(s1, "ldap1.sssd.com");
+    assert_non_null(s1->host);
+    s1->weight = 100;
+    s1->priority = 1;
+    s1->port = 389;
+
+    s2 = talloc_zero(test_ctx, struct ares_srv_reply);
+    assert_non_null(s2);
+    s2->host = talloc_strdup(s2, "ldap2.sssd.com");
+    assert_non_null(s2->host);
+    s2->weight = 100;
+    s2->priority = 2;
+    s2->port = 389;
+
+    s1->next = s2;
+
+    dns_domain = talloc_strdup(test_ctx, "sssd.com");
+    assert_non_null(dns_domain);
+
+    mock_srv_results(s1, TEST_SRV_TTL, dns_domain);
+
+    ret = fo_add_srv_server(test_ctx->fo_svc, "_ldap", "sssd.com",
+                            "sssd.local", "tcp", test_ctx);
+    assert_int_equal(ret, ERR_OK);
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ctx->ev,
+                                  test_ctx->resolv, test_ctx->fo_ctx,
+                                  test_ctx->fo_svc);
+    assert_non_null(req);
+    tevent_req_set_callback(req, test_fo_srv_done1, test_ctx);
+
+    ret = test_ev_loop(test_ctx->ctx);
+    assert_int_equal(ret, ERR_OK);
+}
+
+static void test_fo_srv_done1(struct tevent_req *req)
+{
+    struct test_fo_srv_ctx *test_ctx = \
+        tevent_req_callback_data(req, struct test_fo_srv_ctx);
+    struct fo_server *srv;
+    errno_t ret;
+
+    ret = fo_resolve_service_recv(req, &srv);
+    talloc_zfree(req);
+    assert_int_equal(ret, ERR_OK);
+
+    /* ldap1.sssd.com has lower priority, it must always be first */
+    check_server(srv, 389, "ldap1.sssd.com");
+
+    /* Mark the server as working and request the service again. The same server
+     * must be returned */
+    fo_set_server_status(srv, SERVER_WORKING);
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ctx->ev,
+                                  test_ctx->resolv, test_ctx->fo_ctx,
+                                  test_ctx->fo_svc);
+    assert_non_null(req);
+    tevent_req_set_callback(req, test_fo_srv_done2, test_ctx);
+}
+
+static void test_fo_srv_done2(struct tevent_req *req)
+{
+    struct test_fo_srv_ctx *test_ctx = \
+        tevent_req_callback_data(req, struct test_fo_srv_ctx);
+    struct fo_server *srv;
+    errno_t ret;
+
+    ret = fo_resolve_service_recv(req, &srv);
+    talloc_zfree(req);
+    assert_int_equal(ret, ERR_OK);
+
+    /* Must be ldap1 again */
+    check_server(srv, 389, "ldap1.sssd.com");
+
+    /* Mark it at wrong, next lookup should yield ldap2 */
+    fo_set_server_status(srv, SERVER_NOT_WORKING);
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ctx->ev,
+                                  test_ctx->resolv, test_ctx->fo_ctx,
+                                  test_ctx->fo_svc);
+    assert_non_null(req);
+    tevent_req_set_callback(req, test_fo_srv_done3, test_ctx);
+}
+
+static void test_fo_srv_done3(struct tevent_req *req)
+{
+    struct test_fo_srv_ctx *test_ctx = \
+        tevent_req_callback_data(req, struct test_fo_srv_ctx);
+    struct fo_server *srv;
+    errno_t ret;
+
+    ret = fo_resolve_service_recv(req, &srv);
+    talloc_zfree(req);
+    assert_int_equal(ret, ERR_OK);
+
+    /* Must be ldap2 now */
+    check_server(srv, 389, "ldap2.sssd.com");
+
+    /* Mark is at wrong, next lookup must reach the end of the server list */
+    fo_set_server_status(srv, SERVER_NOT_WORKING);
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ctx->ev,
+                                  test_ctx->resolv, test_ctx->fo_ctx,
+                                  test_ctx->fo_svc);
+    assert_non_null(req);
+    tevent_req_set_callback(req, test_fo_srv_done4, test_ctx);
+}
+
+static void test_fo_srv_done4(struct tevent_req *req)
+{
+    struct test_fo_srv_ctx *test_ctx = \
+        tevent_req_callback_data(req, struct test_fo_srv_ctx);
+    struct fo_server *srv;
+    errno_t ret;
+
+    ret = fo_resolve_service_recv(req, &srv);
+    talloc_zfree(req);
+    /* No servers are left..*/
+    assert_int_equal(ret, ENOENT);
+
+    test_ctx->ctx->error = ERR_OK;
+    test_ctx->ctx->done = true;
+}
+
+/* Make sure that two queries more than TTL seconds apart resolve
+ * into two different lists
+ */
+static void test_fo_srv_before(struct tevent_req *req);
+static void test_fo_srv_after(struct tevent_req *req);
+
+void test_fo_srv_ttl_change(void **state)
+{
+    errno_t ret;
+    struct tevent_req *req;
+    struct test_fo_srv_ctx *test_ctx =
+        talloc_get_type(*state, struct test_fo_srv_ctx);
+    struct ares_srv_reply *s1;
+    struct ares_srv_reply *s2;
+    char *dns_domain;
+
+    s1 = talloc_zero(test_ctx, struct ares_srv_reply);
+    assert_non_null(s1);
+    s1->host = talloc_strdup(s1, "ldap1.sssd.com");
+    assert_non_null(s1->host);
+    s1->weight = 100;
+    s1->priority = 1;
+    s1->port = 389;
+
+    s2 = talloc_zero(test_ctx, struct ares_srv_reply);
+    assert_non_null(s2);
+    s2->host = talloc_strdup(s2, "ldap2.sssd.com");
+    assert_non_null(s2->host);
+    s2->weight = 100;
+    s2->priority = 2;
+    s2->port = 389;
+
+    s1->next = s2;
+
+    dns_domain = talloc_strdup(test_ctx, "sssd.com");
+    assert_non_null(dns_domain);
+
+    mock_srv_results(s1, TEST_SRV_SHORT_TTL, dns_domain);
+
+    ret = fo_add_srv_server(test_ctx->fo_svc, "_ldap", "sssd.com",
+                            "sssd.local", "tcp", test_ctx);
+    assert_int_equal(ret, ERR_OK);
+
+    ret = fo_add_server(test_ctx->fo_svc, "ldap1.sssd.com",
+                        389, (void *) discard_const("ldap://ldap1.sssd.com"),
+                        true);
+    assert_int_equal(ret, ERR_OK);
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ctx->ev,
+                                  test_ctx->resolv, test_ctx->fo_ctx,
+                                  test_ctx->fo_svc);
+    assert_non_null(req);
+    tevent_req_set_callback(req, test_fo_srv_before, test_ctx);
+
+    ret = test_ev_loop(test_ctx->ctx);
+    assert_int_equal(ret, ERR_OK);
+}
+
+static void test_fo_srv_before(struct tevent_req *req)
+{
+    struct test_fo_srv_ctx *test_ctx = \
+        tevent_req_callback_data(req, struct test_fo_srv_ctx);
+    struct fo_server *srv;
+    struct ares_srv_reply *s1;
+    struct ares_srv_reply *s2;
+    char *dns_domain;
+    errno_t ret;
+
+    ret = fo_resolve_service_recv(req, &srv);
+    talloc_zfree(req);
+    assert_int_equal(ret, ERR_OK);
+
+    DEBUG(SSSDBG_TRACE_FUNC, "Before TTL change\n");
+
+    check_server(srv, 389, "ldap1.sssd.com");
+    fo_set_server_status(srv, SERVER_WORKING);
+
+    /* Simulate changing the DNS environment. Change the host names */
+    s1 = talloc_zero(test_ctx, struct ares_srv_reply);
+    assert_non_null(s1);
+    s1->host = talloc_strdup(s1, "ldap2.sssd.com");
+    assert_non_null(s1->host);
+    s1->weight = 100;
+    s1->priority = 2;
+    s1->port = 389;
+
+    s2 = talloc_zero(test_ctx, struct ares_srv_reply);
+    assert_non_null(s2);
+    s2->host = talloc_strdup(s2, "ldap3.sssd.com");
+    assert_non_null(s2->host);
+    s2->weight = 100;
+    s2->priority = 1;
+    s2->port = 389;
+
+    s1->next = s2;
+
+    dns_domain = talloc_strdup(test_ctx, "sssd.com");
+    assert_non_null(dns_domain);
+
+    mock_srv_results(s1, TEST_SRV_SHORT_TTL, dns_domain);
+    sleep(TEST_SRV_SHORT_TTL + 1);
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ctx->ev,
+                                  test_ctx->resolv, test_ctx->fo_ctx,
+                                  test_ctx->fo_svc);
+    assert_non_null(req);
+    tevent_req_set_callback(req, test_fo_srv_after, test_ctx);
+}
+
+static void test_fo_srv_after(struct tevent_req *req)
+{
+    struct test_fo_srv_ctx *test_ctx = \
+        tevent_req_callback_data(req, struct test_fo_srv_ctx);
+    struct fo_server *srv;
+    errno_t ret;
+
+    ret = fo_resolve_service_recv(req, &srv);
+    talloc_zfree(req);
+    assert_int_equal(ret, ERR_OK);
+
+    /* Must be a different server now */
+    check_server(srv, 389, "ldap3.sssd.com");
+
+    test_ctx->ctx->error = ERR_OK;
+    test_ctx->ctx->done = true;
+}
+
+int main(int argc, const char *argv[])
+{
+    int rv;
+    poptContext pc;
+    int opt;
+    struct poptOption long_options[] = {
+        POPT_AUTOHELP
+        SSSD_DEBUG_OPTS
+        POPT_TABLEEND
+    };
+
+    const UnitTest tests[] = {
+        unit_test_setup_teardown(test_fo_srv,
+                                 test_fo_srv_setup,
+                                 test_fo_srv_teardown),
+        unit_test_setup_teardown(test_fo_srv_ttl_change,
+                                 test_fo_srv_setup,
+                                 test_fo_srv_teardown),
+    };
+
+    /* Set debug level to invalid value so we can deside if -d 0 was used. */
+    debug_level = SSSDBG_INVALID;
+
+    pc = poptGetContext(argv[0], argc, argv, long_options, 0);
+    while((opt = poptGetNextOpt(pc)) != -1) {
+        switch(opt) {
+        default:
+            fprintf(stderr, "\nInvalid option %s: %s\n\n",
+                    poptBadOption(pc, 0), poptStrerror(opt));
+            poptPrintUsage(pc, stderr, 0);
+            return 1;
+        }
+    }
+    poptFreeContext(pc);
+
+    DEBUG_CLI_INIT(debug_level);
+
+    /* Even though normally the tests should clean up after themselves
+     * they might not after a failed run. Remove the old db to be sure */
+    tests_set_cwd();
+
+    rv = run_tests(tests);
+    return rv;
+}