Add code to make it easier to reconnect in case the server

is not available immediately or drops the dbus connection.
First step is the nss connection to the data provider.

7 files changed, 134 insertions, 33 deletions

diff --git a/server/monitor.c b/server/monitor.c
index 921e8b59a..369047913 100644
--- a/server/monitor.c
+++ b/server/monitor.c
@@ -37,7 +37,7 @@
 
 /* ping time cannot be less then once every few seconds or the
  * monitor will get crazy hammering children with messages */
-#define MONITOR_MIN_PING_TIME 10
+#define MONITOR_DEF_PING_TIME 10
 
 struct mt_conn {
     struct sbus_conn_ctx *conn_ctx;
@@ -55,6 +55,8 @@ struct mt_svc {
     char *name;
     pid_t pid;
 
+    int ping_time;
+
     int restarts;
     time_t last_restart;
     time_t last_pong;
@@ -68,7 +70,6 @@ struct mt_ctx {
     struct sbus_srv_ctx *sbus_srv;
 
     int service_id_timeout;
-    int service_ping_time;
 };
 
 static int start_service(struct mt_svc *mt_svc);
@@ -260,7 +261,7 @@ static void set_tasks_checker(struct mt_svc *svc)
     struct timeval tv;
 
     gettimeofday(&tv, NULL);
-    tv.tv_sec += svc->mt_ctx->service_ping_time;
+    tv.tv_sec += svc->ping_time;
     tv.tv_usec = 0;
     te = event_add_timed(svc->mt_ctx->ev, svc, tv, tasks_check_handler, svc);
     if (te == NULL) {
@@ -281,15 +282,6 @@ int get_monitor_config(struct mt_ctx *ctx)
         return ret;
     }
 
-    ret = confdb_get_int(ctx->cdb, ctx,
-                         "config/services/monitor", "servicePingTime",
-                         MONITOR_MIN_PING_TIME, &ctx->service_ping_time);
-    if (ret != EOK) {
-        return ret;
-    }
-    if (ctx->service_ping_time < MONITOR_MIN_PING_TIME)
-        ctx->service_ping_time = MONITOR_MIN_PING_TIME;
-
     ret = confdb_get_param(ctx->cdb, ctx,
                            "config/services", "activeServices",
                            &ctx->services);
@@ -355,6 +347,15 @@ int monitor_process_init(TALLOC_CTX *mem_ctx,
             talloc_free(svc);
             continue;
         }
+
+        ret = confdb_get_int(cdb, svc, path, "timeout",
+                             MONITOR_DEF_PING_TIME, &svc->ping_time);
+        if (ret != EOK) {
+            DEBUG(0,("Failed to start service '%s'\n", svc->name));
+            talloc_free(svc);
+            continue;
+        }
+
         talloc_free(path);
 
         /* Add this service to the queue to be started once the monitor
@@ -397,6 +398,16 @@ int monitor_process_init(TALLOC_CTX *mem_ctx,
             continue;
         }
 
+        ret = confdb_get_int(cdb, svc, path, "timeout",
+                             MONITOR_DEF_PING_TIME, &svc->ping_time);
+        if (ret != EOK) {
+            DEBUG(0,("Failed to start service '%s'\n", svc->name));
+            talloc_free(svc);
+            continue;
+        }
+
+        talloc_free(path);
+
         /* if no command is present do not run the domain */
         if (svc->command == NULL) {
             /* the LOCAL domain does not need a backend at the moment */
diff --git a/server/nss/nsssrv_dp.c b/server/nss/nsssrv_dp.c
index 5b1092acd..567f8e9b7 100644
--- a/server/nss/nsssrv_dp.c
+++ b/server/nss/nsssrv_dp.c
@@ -20,6 +20,7 @@
 */
 
 #include <sys/time.h>
+#include <time.h>
 #include "util/util.h"
 #include "nss/nsssrv.h"
 #include "providers/data_provider.h"
@@ -292,19 +293,101 @@ struct sbus_method nss_dp_methods[] = {
     { NULL, NULL }
 };
 
-int nss_dp_init(struct nss_ctx *nctx)
+struct nss_dp_pvt_ctx {
+    struct nss_ctx *nctx;
+    struct sbus_method *methods;
+    time_t last_retry;
+    int retries;
+};
+
+static int nss_dp_conn_destructor(void *data);
+static void nss_dp_reconnect(struct event_context *ev,
+                             struct timed_event *te,
+                             struct timeval tv, void *data);
+
+static void nss_dp_conn_reconnect(struct nss_dp_pvt_ctx *pvt)
 {
+    struct nss_ctx *nctx;
+    struct timed_event *te;
+    struct timeval tv;
+    time_t now;
     int ret;
 
-    /* Set up SBUS connection to the data provider */
+    now = time(NULL);
+
+    /* reset retry if last reconnect was > 60 sec. ago */
+    if (pvt->last_retry + 60 < now) pvt->retries = 0;
+    if (pvt->retries >= 3) {
+        DEBUG(4, ("Too many reconnect retries! Giving up\n"));
+        return;
+    }
+
+    pvt->last_retry = now;
+    pvt->retries++;
+
+    nctx = pvt->nctx;
+
     ret = dp_sbus_cli_init(nctx, nctx->ev, nctx->cdb,
-                           nss_dp_methods, &nctx->dp_ctx);
+                           pvt->methods, pvt,
+                           nss_dp_conn_destructor,
+                           &nctx->dp_ctx);
     if (ret != EOK) {
-        return ret;
+        DEBUG(4, ("Failed to reconnect [%d(%s)]!\n", ret, strerror(ret)));
+
+        tv.tv_sec = now +5;
+        tv.tv_usec = 0;
+        te = event_add_timed(nctx->ev, nctx, tv, nss_dp_reconnect, pvt);
+        if (te == NULL) {
+            DEBUG(4, ("Failed to add timed event! Giving up\n"));
+        } else {
+            DEBUG(4, ("Retrying in 5 seconds\n"));
+        }
     }
+}
+
+static void nss_dp_reconnect(struct event_context *ev,
+                             struct timed_event *te,
+                             struct timeval tv, void *data)
+{
+    struct nss_dp_pvt_ctx *pvt;
+
+    pvt = talloc_get_type(data, struct nss_dp_pvt_ctx);
+
+    nss_dp_conn_reconnect(pvt);
+}
+
+int nss_dp_conn_destructor(void *data)
+{
+    struct nss_dp_pvt_ctx *pvt;
+    struct sbus_conn_ctx *scon;
+
+    scon = talloc_get_type(data, struct sbus_conn_ctx);
+    if (!scon) return 0;
+
+    /* if this is a regular disconnect just quit */
+    if (sbus_conn_disconnecting(scon)) return 0;
+
+    pvt = talloc_get_type(sbus_conn_get_private_data(scon),
+                          struct nss_dp_pvt_ctx);
+    if (pvt) return 0;
+
+    nss_dp_conn_reconnect(pvt);
+
+    return 0;
+}
+
+int nss_dp_init(struct nss_ctx *nctx)
+{
+    struct nss_dp_pvt_ctx *pvt;
+    int ret;
+
+    pvt = talloc_zero(nctx, struct nss_dp_pvt_ctx);
+    if (!pvt) return ENOMEM;
+
+    pvt->nctx = nctx;
+    pvt->methods = nss_dp_methods;
 
-    /* attach context to the connection */
-    sbus_conn_set_private_data(nctx->dp_ctx->scon_ctx, nctx);
+    nss_dp_conn_reconnect(pvt);
 
     return EOK;
 }
diff --git a/server/providers/data_provider.h b/server/providers/data_provider.h
index bd610a539..a7311e5ee 100644
--- a/server/providers/data_provider.h
+++ b/server/providers/data_provider.h
@@ -77,6 +77,8 @@ int dp_sbus_cli_init(TALLOC_CTX *mem_ctx,
                      struct event_context *ev,
                      struct confdb_ctx *cdb,
                      struct sbus_method *methods,
+                     void *conn_pvt_data,
+                     sbus_conn_destructor_fn destructor,
                      struct service_sbus_ctx **srvs_ctx);
 
 #endif /* __DATA_PROVIDER_ */
diff --git a/server/providers/data_provider_be.c b/server/providers/data_provider_be.c
index 75e99bdc5..ba9ea466c 100644
--- a/server/providers/data_provider_be.c
+++ b/server/providers/data_provider_be.c
@@ -308,21 +308,9 @@ static int mon_cli_init(struct be_ctx *ctx)
  * sbus channel to the data provider daemon */
 static int be_cli_init(struct be_ctx *ctx)
 {
-    struct service_sbus_ctx *dp_ctx;
-    int ret;
-
-    ret = dp_sbus_cli_init(ctx, ctx->ev, ctx->cdb,
-                           be_methods, &dp_ctx);
-    if (ret != EOK) {
-        return ret;
-    }
-
-    ctx->dp_ctx = dp_ctx;
-
-    /* attach be context to the connection */
-    sbus_conn_set_private_data(dp_ctx->scon_ctx, ctx);
-
-    return EOK;
+    return dp_sbus_cli_init(ctx, ctx->ev, ctx->cdb,
+                            be_methods, ctx, NULL,
+                            &ctx->dp_ctx);
 }
 
 static int load_backend(struct be_ctx *ctx)
diff --git a/server/providers/dp_helpers.c b/server/providers/dp_helpers.c
index 99d65f45e..52dc69e18 100644
--- a/server/providers/dp_helpers.c
+++ b/server/providers/dp_helpers.c
@@ -25,6 +25,8 @@ int dp_sbus_cli_init(TALLOC_CTX *mem_ctx,
                      struct event_context *ev,
                      struct confdb_ctx *cdb,
                      struct sbus_method *methods,
+                     void *conn_pvt_data,
+                     sbus_conn_destructor_fn destructor,
                      struct service_sbus_ctx **srvs_ctx)
 {
     struct service_sbus_ctx *ss_ctx;
@@ -87,6 +89,14 @@ int dp_sbus_cli_init(TALLOC_CTX *mem_ctx,
     sm_ctx->message_handler = sbus_message_handler;
     sbus_conn_add_method_ctx(ss_ctx->scon_ctx, sm_ctx);
 
+    if (conn_pvt_data) {
+        sbus_conn_set_private_data(ss_ctx->scon_ctx, conn_pvt_data);
+    }
+
+    if (destructor) {
+        sbus_conn_set_destructor(ss_ctx->scon_ctx, destructor);
+    }
+
     talloc_steal(mem_ctx, ss_ctx);
     *srvs_ctx = ss_ctx;
     ret = EOK;
diff --git a/server/sbus/sssd_dbus.h b/server/sbus/sssd_dbus.h
index cf59a22f9..51a16e203 100644
--- a/server/sbus/sssd_dbus.h
+++ b/server/sbus/sssd_dbus.h
@@ -89,6 +89,7 @@ void sbus_conn_set_private_data(struct sbus_conn_ctx *conn_ctx, void *pvt_data);
 void *sbus_conn_get_private_data(struct sbus_conn_ctx *conn_ctx);
 int sbus_conn_add_method_ctx(struct sbus_conn_ctx *conn_ctx,
                              struct sbus_method_ctx *method_ctx);
+bool sbus_conn_disconnecting(struct sbus_conn_ctx *conn_ctx);
 
 /* Default message handler
  * Should be usable for most cases */
diff --git a/server/sbus/sssd_dbus_connection.c b/server/sbus/sssd_dbus_connection.c
index 81d8aee1b..2d2758146 100644
--- a/server/sbus/sssd_dbus_connection.c
+++ b/server/sbus/sssd_dbus_connection.c
@@ -620,3 +620,9 @@ void *sbus_conn_get_private_data(struct sbus_conn_ctx *conn_ctx)
 {
     return conn_ctx->pvt_data;
 }
+
+bool sbus_conn_disconnecting(struct sbus_conn_ctx *conn_ctx)
+{
+    if (conn_ctx->disconnect == 1) return true;
+    return false;
+}