Add fail over utility functions

These functions should be used by providers to centrally manage lists of
servers. Servers are grouped into services and each service has it's own
list of servers. If, however, you will try to add a same server into two
different services, they will share a common structure. This means that
a host will only be resolved once.

4 files changed, 971 insertions, 1 deletions

diff --git a/server/Makefile.am b/server/Makefile.am
index ca793394c..88053fc59 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -69,7 +69,8 @@ if HAVE_CHECK
         krb5-utils-tests \
         check_and_open-tests \
         files-tests \
-        refcount-tests
+        refcount-tests \
+        fail_over-tests
 endif
 
 check_PROGRAMS = \
@@ -209,6 +210,9 @@ if BUILD_ARES_PARSE_TXT
     SSSD_RESOLV_OBJ += resolv/ares/ares_parse_txt_reply.c
 endif
 
+SSSD_FAILOVER_OBJ = \
+    providers/fail_over.c \
+    $(SSSD_RESOLV_OBJ)
 
 SSSD_LIBS = \
     $(TALLOC_LIBS) \
@@ -279,6 +283,7 @@ dist_noinst_HEADERS = \
     confdb/confdb_setup.h \
     providers/data_provider.h \
     providers/dp_backend.h \
+    providers/fail_over.h \
     providers/providers.h \
     providers/krb5/krb5_auth.h \
     providers/krb5/krb5_common.h \
@@ -478,6 +483,19 @@ refcount_tests_CFLAGS = \
 refcount_tests_LDADD = \
     $(SSSD_LIBS) \
     $(CHECK_LIBS)
+
+fail_over_tests_SOURCES = \
+    tests/fail_over-tests.c \
+    tests/common.c \
+    $(SSSD_FAILOVER_OBJ) \
+    $(CHECK_OBJ) \
+    $(SSSD_UTIL_OBJ)
+fail_over_tests_CFLAGS = \
+    $(CHECK_CFLAGS)
+fail_over_tests_LDADD = \
+    $(SSSD_LIBS) \
+    $(CHECK_LIBS) \
+    $(CARES_LIBS)
 endif
 
 stress_tests_SOURCES = \
diff --git a/server/providers/fail_over.c b/server/providers/fail_over.c
new file mode 100644
index 000000000..ce94ac30f
--- /dev/null
+++ b/server/providers/fail_over.c
@@ -0,0 +1,543 @@
+/*
+   SSSD
+
+   Fail over helper functions.
+
+   Authors:
+        Martin Nagy <mnagy@redhat.com>
+
+   Copyright (C) Red Hat, Inc 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <sys/time.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <strings.h>
+#include <talloc.h>
+
+#include "util/dlinklist.h"
+#include "util/refcount.h"
+#include "util/util.h"
+#include "providers/fail_over.h"
+#include "resolv/async_resolv.h"
+
+#define STATUS_DIFF(p, tv2) ((p)->last_status_change.tv_sec - (tv2).tv_sec)
+
+#define DEFAULT_PORT_STATUS PORT_NEUTRAL
+#define DEFAULT_SERVER_STATUS SERVER_NAME_NOT_RESOLVED
+
+struct fo_ctx {
+    struct fo_service *service_list;
+    struct server_common *server_common_list;
+
+    /* Settings. */
+    time_t retry_timeout;
+};
+
+struct fo_service {
+    struct fo_service *prev;
+    struct fo_service *next;
+
+    struct fo_ctx *ctx;
+    char *name;
+    struct fo_server *active_server;
+    struct fo_server *server_list;
+};
+
+struct fo_server {
+    struct fo_server *prev;
+    struct fo_server *next;
+
+    void *user_data;
+    int port;
+    int port_status;
+    struct fo_service *service;
+    struct timeval last_status_change;
+    struct server_common *common;
+};
+
+struct server_common {
+    REFCOUNT_COMMON;
+
+    struct server_common *prev;
+    struct server_common *next;
+
+    char *name;
+    struct hostent *hostent;
+    struct resolve_service_request *request_list;
+    int server_status;
+    struct timeval last_status_change;
+};
+
+struct resolve_service_request {
+    struct resolve_service_request *prev;
+    struct resolve_service_request *next;
+
+    struct server_common *server_common;
+    struct tevent_req *req;
+};
+
+struct status {
+    int value;
+    struct timeval last_change;
+};
+
+struct fo_ctx *
+fo_context_init(TALLOC_CTX *mem_ctx, time_t retry_timeout)
+{
+    struct fo_ctx *ctx;
+
+    ctx = talloc_zero(mem_ctx, struct fo_ctx);
+    if (ctx == NULL) {
+        DEBUG(1, ("No memory\n"));
+        return NULL;
+    }
+
+    ctx->retry_timeout = retry_timeout;
+
+    return ctx;
+}
+
+/*
+ * This function will return the status of the server. If the status was
+ * last updated a long time ago, we will first reset the status.
+ */
+static enum server_status
+get_server_status(struct fo_server *server)
+{
+    struct timeval tv;
+    time_t timeout;
+
+    if (server->common == NULL)
+        return SERVER_NAME_RESOLVED;
+
+    timeout = server->service->ctx->retry_timeout;
+    if (timeout != 0 && server->common->server_status == SERVER_NOT_WORKING) {
+        gettimeofday(&tv, NULL);
+        if (STATUS_DIFF(server->common, tv) > timeout) {
+            server->common->server_status = SERVER_NAME_RESOLVED;
+            server->last_status_change.tv_sec = tv.tv_sec;
+        }
+    }
+
+    return server->common->server_status;
+}
+
+/*
+ * This function will return the status of the service. If the status was
+ * last updated a long time ago, we will first reset the status.
+ */
+static enum port_status
+get_port_status(struct fo_server *server)
+{
+    struct timeval tv;
+    time_t timeout;
+
+    timeout = server->service->ctx->retry_timeout;
+    if (timeout != 0 && server->port_status == PORT_NOT_WORKING) {
+        gettimeofday(&tv, NULL);
+        if (STATUS_DIFF(server, tv) > timeout) {
+            server->port_status = PORT_NEUTRAL;
+            server->last_status_change.tv_sec = tv.tv_sec;
+        }
+    }
+
+    return server->port_status;
+}
+
+static int
+server_works(struct fo_server *server)
+{
+    if (get_server_status(server) == SERVER_NOT_WORKING)
+        return 0;
+
+    return 1;
+}
+
+static int
+service_works(struct fo_server *server)
+{
+    if (!server_works(server))
+        return 0;
+    if (get_port_status(server) == PORT_NOT_WORKING)
+        return 0;
+
+    return 1;
+}
+
+static int
+service_destructor(struct fo_service *service)
+{
+    DLIST_REMOVE(service->ctx->service_list, service);
+    return 0;
+}
+
+int
+fo_new_service(struct fo_ctx *ctx, const char *name,
+               struct fo_service **_service)
+{
+    struct fo_service *service;
+    int ret;
+
+    ret = fo_get_service(ctx, name, &service);
+    if (ret == EOK) {
+        DEBUG(1, ("Service %s already exists\n", name));
+        return EEXIST;
+    } else if (ret != ENOENT) {
+        return ret;
+    }
+
+    service = talloc_zero(ctx, struct fo_service);
+    if (service == NULL)
+        return ENOMEM;
+
+    service->name = talloc_strdup(service, name);
+    if (service->name == NULL) {
+        talloc_free(service);
+        return ENOMEM;
+    }
+
+    service->ctx = ctx;
+    DLIST_ADD(ctx->service_list, service);
+
+    talloc_set_destructor(service, service_destructor);
+    *_service = service;
+
+    return EOK;
+}
+
+int
+fo_get_service(struct fo_ctx *ctx, const char *name,
+               struct fo_service **_service)
+{
+    struct fo_service *service;
+
+    DLIST_FOR_EACH(service, ctx->service_list) {
+        if (!strcasecmp(name, service->name)) {
+            *_service = service;
+            return EOK;
+        }
+    }
+
+    return ENOENT;
+}
+
+static int
+get_server_common(TALLOC_CTX *mem_ctx, struct fo_ctx *ctx, const char *name,
+                  struct server_common **_common)
+{
+    struct server_common *common;
+
+    DLIST_FOR_EACH(common, ctx->server_common_list) {
+        if (!strcmp(name, common->name)) {
+            *_common = rc_reference(mem_ctx, struct server_common, common);
+            if (_common == NULL)
+                return ENOMEM;
+            return EOK;
+        }
+    }
+
+    return ENOENT;
+}
+
+static struct server_common *
+create_server_common(TALLOC_CTX *mem_ctx, struct fo_ctx *ctx, const char *name)
+{
+    struct server_common *common;
+
+    common = rc_alloc(mem_ctx, struct server_common);
+    if (common == NULL)
+        return NULL;
+
+    common->name = talloc_strdup(common, name);
+    if (common->name == NULL) {
+        talloc_free(common);
+        return NULL;
+    }
+
+    common->prev = NULL;
+    common->next = NULL;
+    common->hostent = NULL;
+    common->request_list = NULL;
+    common->server_status = DEFAULT_SERVER_STATUS;
+    common->last_status_change.tv_sec = 0;
+    common->last_status_change.tv_usec = 0;
+
+    DLIST_ADD_END(ctx->server_common_list, common, struct server_common *);
+    return common;
+}
+
+int
+fo_add_server(struct fo_service *service, const char *name, int port,
+              void *user_data)
+{
+    struct fo_server *server;
+    int ret;
+
+    DLIST_FOR_EACH(server, service->server_list) {
+        if (server->port != port || server->user_data != user_data)
+            continue;
+        if (name == NULL && server->common == NULL) {
+            return EEXIST;
+        } else if (name != NULL && server->common != NULL) {
+            if (!strcmp(name, server->common->name))
+                return EEXIST;
+        }
+    }
+
+    server = talloc_zero(service, struct fo_server);
+    if (server == NULL)
+        return ENOMEM;
+
+    server->port = port;
+    server->user_data = user_data;
+    server->service = service;
+    server->port_status = DEFAULT_PORT_STATUS;
+
+    if (name != NULL) {
+        ret = get_server_common(server, service->ctx, name, &server->common);
+        if (ret == ENOENT) {
+            server->common = create_server_common(server, service->ctx, name);
+            if (server->common == NULL) {
+                talloc_free(server);
+                return ENOMEM;
+            }
+        } else if (ret != EOK) {
+            talloc_free(server);
+            return ret;
+        }
+    }
+
+    DLIST_ADD_END(service->server_list, server, struct fo_server *);
+
+    return EOK;
+}
+
+static int
+get_first_server_entity(struct fo_service *service, struct fo_server **_server)
+{
+    struct fo_server *server;
+
+    /* If we already have a working server, use that one. */
+    server = service->active_server;
+    if (server != NULL) {
+        if (service_works(server)) {
+            *_server = server;
+            return EOK;
+        }
+        service->active_server = NULL;
+    }
+
+    /* Otherwise iterate through the server list. */
+    DLIST_FOR_EACH(server, service->server_list) {
+        if (service_works(server)) {
+            *_server = server;
+            return EOK;
+        }
+    }
+
+    return ENOENT;
+}
+
+static int
+resolve_service_request_destructor(struct resolve_service_request *request)
+{
+    DLIST_REMOVE(request->server_common->request_list, request);
+    return 0;
+}
+
+static int
+set_lookup_hook(struct fo_server *server, struct tevent_req *req)
+{
+    struct resolve_service_request *request;
+
+    request = talloc(req, struct resolve_service_request);
+    if (request == NULL) {
+        DEBUG(1, ("No memory\n"));
+        talloc_free(request);
+        return ENOMEM;
+    }
+    request->server_common = server->common;
+    request->req = req;
+    DLIST_ADD(server->common->request_list, request);
+    talloc_set_destructor(request, resolve_service_request_destructor);
+
+    return EOK;
+}
+
+/*******************************************************************
+ * Get server to connect to.                                       *
+ *******************************************************************/
+
+struct resolve_service_state {
+    struct fo_server *server;
+};
+
+static void fo_resolve_service_done(struct tevent_req *subreq);
+
+struct tevent_req *
+fo_resolve_service_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+                   struct resolv_ctx *resolv, struct fo_service *service)
+{
+    int ret;
+    struct fo_server *server;
+    struct tevent_req *req;
+    struct tevent_req *subreq;
+    struct resolve_service_state *state;
+
+    req = tevent_req_create(mem_ctx, &state, struct resolve_service_state);
+    if (req == NULL)
+        return NULL;
+
+    ret = get_first_server_entity(service, &server);
+    if (ret != EOK) {
+        DEBUG(1, ("No available servers for service '%s'\n", service->name));
+        goto done;
+    }
+
+    state->server = server;
+
+    if (server->common == NULL) {
+        /* This server doesn't have a name, we don't do name resolution. */
+        tevent_req_done(req);
+        tevent_req_post(req, ev);
+        return req;
+    }
+
+    switch (get_server_status(server)) {
+    case SERVER_NAME_NOT_RESOLVED: /* Request name resolution. */
+        /* TODO: Enable IPv6. */
+        subreq = resolv_gethostbyname_send(server->common, ev, resolv,
+                                           server->common->name, AF_INET);
+        if (subreq == NULL) {
+            ret = ENOMEM;
+            goto done;
+        }
+        tevent_req_set_callback(subreq, fo_resolve_service_done, server->common);
+        /* FALLTHROUGH */
+    case SERVER_RESOLVING_NAME:
+        /* Name resolution is already under way. Just add ourselves into the
+         * waiting queue so we get notified after the operation is finished. */
+        ret = set_lookup_hook(server, req);
+        if (ret != EOK)
+            goto done;
+        break;
+    default: /* The name is already resolved. Return immediately. */
+        tevent_req_done(req);
+        tevent_req_post(req, ev);
+        break;
+    }
+
+done:
+    if (ret != EOK) {
+        tevent_req_error(req, ret);
+        tevent_req_post(req, ev);
+    }
+    return req;
+}
+
+static void
+fo_resolve_service_done(struct tevent_req *subreq)
+{
+    int resolv_status;
+    struct resolve_service_request *request;
+    struct server_common *common;
+
+    common = tevent_req_callback_data(subreq, struct server_common);
+
+    if (common->hostent != NULL)
+        talloc_free(common->hostent);
+    resolv_gethostbyname_recv(common, subreq, &resolv_status, NULL,
+                              &common->hostent);
+    talloc_free(subreq);
+
+    if (common->hostent == NULL) {
+        DEBUG(1, ("Failed to resolve %s: %s\n", common->name,
+              resolv_strerror(resolv_status)));
+    }
+
+    /* Take care of all requests for this server. */
+    while ((request = common->request_list) != NULL) {
+        DLIST_REMOVE(common->request_list, request);
+        if (resolv_status)
+            tevent_req_error(request->req, resolv_status);
+        else
+            tevent_req_done(request->req);
+    }
+}
+
+int
+fo_resolve_service_recv(struct tevent_req *req, struct fo_server **server)
+{
+    struct resolve_service_state *state;
+    enum tevent_req_state tstate;
+    uint64_t err = EIO;
+
+    state = tevent_req_data(req, struct resolve_service_state);
+
+    if (tevent_req_is_error(req, &tstate, &err))
+        return err;
+
+    if (server)
+        *server = state->server;
+
+    return EOK;
+}
+
+void
+fo_set_server_status(struct fo_server *server, enum server_status status)
+{
+    if (server->common == NULL) {
+        DEBUG(1, ("Bug: Trying to set server status of a name-less server\n"));
+        return;
+    }
+    server->common->server_status = status;
+    gettimeofday(&server->common->last_status_change, NULL);
+}
+
+void
+fo_set_port_status(struct fo_server *server, enum port_status status)
+{
+    server->port_status = status;
+    gettimeofday(&server->last_status_change, NULL);
+    if (status == PORT_WORKING) {
+        fo_set_server_status(server, SERVER_WORKING);
+        server->service->active_server = server;
+    }
+}
+
+void *
+fo_get_server_user_data(struct fo_server *server)
+{
+    return server->user_data;
+}
+
+int
+fo_get_server_port(struct fo_server *server)
+{
+    return server->port;
+}
+
+struct hostent *
+fo_get_server_hostent(struct fo_server *server)
+{
+    if (server->common == NULL) {
+        DEBUG(1, ("Bug: Trying to get hostent from a name-less server\n"));
+        return NULL;
+    }
+    return server->common->hostent;
+}
diff --git a/server/providers/fail_over.h b/server/providers/fail_over.h
new file mode 100644
index 000000000..b6e2f2f7b
--- /dev/null
+++ b/server/providers/fail_over.h
@@ -0,0 +1,106 @@
+#ifndef __FAIL_OVER_H__
+#define __FAIL_OVER_H__
+
+#include <stdbool.h>
+#include <talloc.h>
+
+/* Some forward declarations that don't have to do anything with fail over. */
+struct hostent;
+struct resolv_ctx;
+struct tevent_context;
+struct tevent_req;
+
+enum port_status {
+    PORT_NEUTRAL,    /* We didn't try this port yet. */
+    PORT_WORKING,    /* This port was reported to work. */
+    PORT_NOT_WORKING /* This port was reported to not work. */
+};
+
+enum server_status {
+    SERVER_NAME_NOT_RESOLVED, /* We didn't yet resolved the host name. */
+    SERVER_RESOLVING_NAME,    /* Name resolving is in progress. */
+    SERVER_NAME_RESOLVED,     /* We resolved the host name but didn't try to connect. */
+    SERVER_WORKING,           /* We successfully connected to the server. */
+    SERVER_NOT_WORKING        /* We tried and failed to connect to the server. */
+};
+
+struct fo_ctx;
+struct fo_service;
+struct fo_server;
+
+/*
+ * Create a new fail over context. The 'retry_timeout' argument specifies the
+ * duration in seconds of how long a server or port will be considered
+ * non-working after being marked as such.
+ */
+struct fo_ctx *fo_context_init(TALLOC_CTX *mem_ctx,
+                               time_t retry_timeout);
+
+/*
+ * Create a new service structure for 'ctx', saving it to the location pointed
+ * to by '_service'. The needed memory will be allocated from 'ctx'.
+ * Service name will be set to 'name'.
+ */
+int fo_new_service(struct fo_ctx *ctx,
+                   const char *name,
+                   struct fo_service **_service);
+
+/*
+ * Look up service named 'name' from the 'ctx' service list. Target of
+ * '_service' will be set to the service if it was found.
+ */
+int fo_get_service(struct fo_ctx *ctx,
+                   const char *name,
+                   struct fo_service **_service);
+
+/*
+ * Adds a server 'name' to the 'service'. Port 'port' will be used for
+ * connection. If 'name' is NULL, no server resolution will be done.
+ */
+int fo_add_server(struct fo_service *service,
+                  const char *name,
+                  int port,
+                  void *user_data);
+
+/*
+ * Request the first server from the service's list of servers. It is only
+ * considered if it is not marked as not working (or the retry interval already
+ * passed). If the server address wasn't resolved yet, it will be done.
+ */
+struct tevent_req *fo_resolve_service_send(TALLOC_CTX *mem_ctx,
+                                           struct tevent_context *ev,
+                                           struct resolv_ctx *resolv,
+                                           struct fo_service *service);
+
+int fo_resolve_service_recv(struct tevent_req *req,
+                            struct fo_server **server);
+
+/*
+ * Set feedback about 'server'. Caller should use this to indicate a problem
+ * with the server itself, not only with the service on that server. This
+ * should be used, for example, when the IP address of the server can't be
+ * reached. This setting can affect other services as well, since they can
+ * share the same server.
+ */
+void fo_set_server_status(struct fo_server *server,
+                          enum server_status status);
+
+/*
+ * Set feedback about the port status. This function should be used when
+ * the server itself is working but the service is not. When status is set
+ * to PORT_WORKING, 'server' is also marked as an "active server" for it's
+ * service. When the next fo_resolve_service_send() function is called, this
+ * server will be preferred. This will hold as long as it is not marked as
+ * not-working.
+ */
+void fo_set_port_status(struct fo_server *server,
+                        enum port_status status);
+
+
+void *fo_get_server_user_data(struct fo_server *server);
+
+int fo_get_server_port(struct fo_server *server);
+
+struct hostent *fo_get_server_hostent(struct fo_server *server);
+
+#endif /* !__FAIL_OVER_H__ */
diff --git a/server/tests/fail_over-tests.c b/server/tests/fail_over-tests.c
new file mode 100644
index 000000000..49375a005
--- /dev/null
+++ b/server/tests/fail_over-tests.c
@@ -0,0 +1,303 @@
+/*
+   SSSD
+
+   Fail over tests.
+
+   Authors:
+        Martin Nagy <mnagy@redhat.com>
+
+   Copyright (C) Red Hat, Inc 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <arpa/inet.h>
+
+#include <check.h>
+#include <popt.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "resolv/async_resolv.h"
+#include "tests/common.h"
+#include "util/util.h"
+
+/* Interface under test */
+#include "providers/fail_over.h"
+
+int use_net_test;
+
+struct test_ctx {
+    struct tevent_context *ev;
+    struct resolv_ctx *resolv;
+    struct fo_ctx *fo_ctx;
+    int tasks;
+};
+
+struct task {
+    struct test_ctx *test_ctx;
+    const char *location;
+    int recv;
+    int port;
+    int new_server_status;
+    int new_port_status;
+};
+
+static struct test_ctx *
+setup_test(void)
+{
+    struct test_ctx *ctx;
+    int ret;
+
+    ctx = talloc_zero(NULL, struct test_ctx);
+    fail_if(ctx == NULL, "Could not allocate memory for test context");
+
+    ctx->ev = tevent_context_init(ctx);
+    if (ctx->ev == NULL) {
+        talloc_free(ctx);
+        fail("Could not init tevent context");
+    }
+
+    ret = resolv_init(ctx, ctx->ev, &ctx->resolv);
+    if (ret != EOK) {
+        talloc_free(ctx);
+        fail("Could not init resolv context");
+    }
+
+    ctx->fo_ctx = fo_context_init(ctx, 5 * 60);
+    if (ctx->fo_ctx == NULL) {
+        talloc_free(ctx);
+        fail("Could not init fail over context");
+    }
+
+    return ctx;
+}
+
+static void
+test_loop(struct test_ctx *data)
+{
+    while (data->tasks != 0)
+        tevent_loop_once(data->ev);
+}
+
+START_TEST(test_fo_new_service)
+{
+    int i;
+    int ret;
+    struct test_ctx *ctx;
+    struct fo_service *service;
+    struct fo_service *services[10];
+
+    ctx = setup_test();
+    check_leaks_push(ctx);
+
+    for (i = 0; i < 10; i++) {
+        char buf[16];
+        sprintf(buf, "service_%d", i);
+
+        check_leaks_push(ctx);
+        ret = fo_new_service(ctx->fo_ctx, buf, &services[i]);
+        fail_if(ret != EOK);
+    }
+
+    ret = fo_new_service(ctx->fo_ctx, "service_3", &service);
+    fail_if(ret != EEXIST);
+
+    for (i = 9; i >= 0; i--) {
+        char buf[16];
+        sprintf(buf, "service_%d", i);
+
+        ret = fo_get_service(ctx->fo_ctx, buf, &service);
+        fail_if(ret != EOK);
+        fail_if(service != services[i]);
+        talloc_free(service);
+        check_leaks_pop(ctx);
+
+        ret = fo_get_service(ctx->fo_ctx, buf, &service);
+        fail_if(ret != ENOENT);
+    }
+
+    check_leaks_pop(ctx);
+    talloc_free(ctx);
+}
+END_TEST
+
+static void
+test_resolve_service_callback(struct tevent_req *req)
+{
+    uint64_t recv_status;
+    int port;
+    struct task *task;
+    struct fo_server *server = NULL;
+    struct hostent *he;
+    int i;
+
+    task = tevent_req_callback_data(req, struct task);
+
+    task->test_ctx->tasks--;
+
+    recv_status = fo_resolve_service_recv(req, &server);
+    talloc_free(req);
+    fail_if(recv_status != task->recv, "%s: Expected return of %d, got %d",
+            task->location, task->recv, recv_status);
+    if (recv_status != EOK)
+        return;
+    fail_if(server == NULL);
+    port = fo_get_server_port(server);
+    fail_if(port != task->port, "%s: Expected port %d, got %d", task->location,
+            task->port, port);
+
+    if (task->new_port_status >= 0)
+        fo_set_port_status(server, task->new_port_status);
+    if (task->new_server_status >= 0)
+        fo_set_server_status(server, task->new_server_status);
+
+    he = fo_get_server_hostent(server);
+    fail_if(he == NULL, "%s: fo_get_server_hostent() returned NULL");
+    for (i = 0; he->h_addr_list[i]; i++) {
+        char buf[256];
+
+        inet_ntop(he->h_addrtype, he->h_addr_list[i], buf, sizeof(buf));
+        fail_if(strcmp(buf, "127.0.0.1") != 0 && strcmp(buf, "::1") != 0);
+    }
+
+}
+
+#define get_request(a, b, c, d, e, f) \
+       _get_request(a, b, c, d, e, f, __location__)
+
+static void
+_get_request(struct test_ctx *test_ctx, struct fo_service *service,
+             int expected_recv, int expected_port, int new_port_status,
+             int new_server_status, const char *location)
+{
+    struct tevent_req *req;
+    struct task *task;
+
+    task = talloc(test_ctx, struct task);
+    fail_if(task == NULL);
+
+    task->test_ctx = test_ctx;
+    task->recv = expected_recv;
+    task->port = expected_port;
+    task->new_port_status = new_port_status;
+    task->new_server_status = new_server_status;
+    task->location = location;
+    test_ctx->tasks++;
+
+    req = fo_resolve_service_send(test_ctx, test_ctx->ev, test_ctx->resolv, service);
+    fail_if(req == NULL, "%s: fo_resolve_service_send() failed", location);
+
+    tevent_req_set_callback(req, test_resolve_service_callback, task);
+    test_loop(test_ctx);
+}
+
+START_TEST(test_fo_resolve_service)
+{
+    struct test_ctx *ctx;
+    struct fo_service *service[2];
+
+    ctx = setup_test();
+    fail_if(ctx == NULL);
+
+    /* Add service. */
+    fail_if(fo_new_service(ctx->fo_ctx, "http", &service[0]) != EOK);
+
+    fail_if(fo_new_service(ctx->fo_ctx, "ldap", &service[1]) != EOK);
+
+    /* Add servers. */
+    fail_if(fo_add_server(service[0], "localhost", 20, NULL) != EOK);
+    fail_if(fo_add_server(service[0], "127.0.0.1", 80, NULL) != EOK);
+
+    fail_if(fo_add_server(service[1], "localhost", 30, NULL) != EOK);
+    fail_if(fo_add_server(service[1], "127.0.0.1", 389, NULL) != EOK);
+    fail_if(fo_add_server(service[1], "127.0.0.1", 389, NULL) != EEXIST);
+
+    /* Make requests. */
+    get_request(ctx, service[0], EOK, 20, PORT_WORKING, -1);
+    get_request(ctx, service[0], EOK, 20, -1, SERVER_NOT_WORKING);
+    get_request(ctx, service[0], EOK, 80, PORT_WORKING, -1);
+    get_request(ctx, service[0], EOK, 80, PORT_NOT_WORKING, -1);
+    get_request(ctx, service[0], ENOENT, 0, -1, -1);
+
+    get_request(ctx, service[1], EOK, 389, PORT_WORKING, -1);
+    get_request(ctx, service[1], EOK, 389, -1, SERVER_NOT_WORKING);
+    get_request(ctx, service[1], ENOENT, 0, -1, -1);
+
+    talloc_free(ctx);
+}
+END_TEST
+
+Suite *
+create_suite(void)
+{
+    Suite *s = suite_create("fail_over");
+
+    TCase *tc = tcase_create("FAIL_OVER Tests");
+
+    tcase_add_checked_fixture(tc, leak_check_setup, leak_check_teardown);
+    /* Do some testing */
+    tcase_add_test(tc, test_fo_new_service);
+    tcase_add_test(tc, test_fo_resolve_service);
+    if (use_net_test) {
+    }
+    /* Add all test cases to the test suite */
+    suite_add_tcase(s, tc);
+
+    return s;
+}
+
+int
+main(int argc, const char *argv[])
+{
+    int opt;
+    poptContext pc;
+    int failure_count;
+    Suite *suite;
+    SRunner *sr;
+    int debug = 0;
+
+    struct poptOption long_options[] = {
+        POPT_AUTOHELP
+        { "debug-level", 'd', POPT_ARG_INT, &debug, 0, "Set debug level", NULL },
+        { "use-net-test", 'n', POPT_ARG_NONE, 0, 'n', "Run tests that need an active internet connection", NULL },
+        POPT_TABLEEND
+    };
+
+    pc = poptGetContext(argv[0], argc, argv, long_options, 0);
+    while((opt = poptGetNextOpt(pc)) != -1) {
+        switch(opt) {
+        case 'n':
+            use_net_test = 1;
+            break;
+
+        default:
+            fprintf(stderr, "\nInvalid option %s: %s\n\n",
+                    poptBadOption(pc, 0), poptStrerror(opt));
+            poptPrintUsage(pc, stderr, 0);
+            return 1;
+        }
+    }
+    poptFreeContext(pc);
+    debug_level = debug;
+
+    suite = create_suite();
+    sr = srunner_create(suite);
+    srunner_run_all(sr, CK_VERBOSE);
+    failure_count = srunner_ntests_failed(sr);
+    srunner_free(sr);
+    return (failure_count == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
+}