Enable autoreconnection of Data Provider Backends to the Data Provider

4 files changed, 124 insertions, 7 deletions

diff --git a/server/providers/data_provider_be.c b/server/providers/data_provider_be.c
index f427ac8c6..8796ec14b 100644
--- a/server/providers/data_provider_be.c
+++ b/server/providers/data_provider_be.c
@@ -646,11 +646,13 @@ static int mon_cli_init(struct be_ctx *ctx)
     return EOK;
 }
 
+static void be_cli_reconnect_init(struct sbus_conn_ctx *sconn, int status, void *pvt);
+
 /* be_cli_init
  * sbus channel to the data provider daemon */
 static int be_cli_init(struct be_ctx *ctx)
 {
-    int ret;
+    int ret, max_retries;
     char *sbus_address;
     struct sbus_method_ctx *sm_ctx;
 
@@ -677,9 +679,92 @@ static int be_cli_init(struct be_ctx *ctx)
         return ret;
     }
 
+    /* Enable automatic reconnection to the Data Provider */
+    ret = confdb_get_int(ctx->cdb, ctx, ctx->conf_path,
+                         "retries", 3, &max_retries);
+    if (ret != EOK) {
+        DEBUG(0, ("Failed to set up automatic reconnection\n"));
+        return ret;
+    }
+
+    sbus_reconnect_init(ctx->dp_ctx->scon_ctx, max_retries,
+                        be_cli_reconnect_init, ctx);
+
     return EOK;
 }
 
+static int be_finalize(struct be_ctx *ctx);
+static void be_shutdown(struct be_req *req, int status,
+                        const char *errstr);
+
+static void be_cli_reconnect_init(struct sbus_conn_ctx *sconn, int status, void *pvt)
+{
+    int ret;
+    struct be_ctx *be_ctx = talloc_get_type(pvt, struct be_ctx);
+
+    /* Did we reconnect successfully? */
+    if (status == SBUS_RECONNECT_SUCCESS) {
+        /* Add the methods back to the new connection */
+        ret = sbus_conn_add_method_ctx(be_ctx->dp_ctx->scon_ctx,
+                                       be_ctx->dp_ctx->sm_ctx);
+        if (ret != EOK) {
+            DEBUG(0, ("Could not re-add methods on reconnection.\n"));
+            ret = be_finalize(be_ctx);
+            if (ret != EOK) {
+                DEBUG(0, ("Finalizing back-end failed with error [%d] [%s]", ret, strerror(ret)));
+                be_shutdown(NULL, ret, NULL);
+            }
+            return;
+        }
+
+        DEBUG(1, ("Reconnected to the Data Provider.\n"));
+        return;
+    }
+
+    /* Handle failure */
+    DEBUG(0, ("Could not reconnect to data provider.\n"));
+    /* Kill the backend and let the monitor restart it */
+    ret = be_finalize(be_ctx);
+    if (ret != EOK) {
+        DEBUG(0, ("Finalizing back-end failed with error [%d] [%s]", ret, strerror(ret)));
+        be_shutdown(NULL, ret, NULL);
+    }
+}
+
+static void be_shutdown(struct be_req *req, int status,
+                        const char *errstr)
+{
+    /* Nothing left to do but exit() */
+    if (status == EOK)
+        exit(0);
+
+    /* Something went wrong in finalize */
+    exit(1);
+}
+
+static int be_finalize(struct be_ctx *ctx)
+{
+    int ret;
+    struct be_req *shutdown_req = talloc_zero(ctx, struct be_req);
+    if (!shutdown_req) {
+        ret = ENOMEM;
+        goto fail;
+    }
+
+    shutdown_req->be_ctx = ctx;
+    shutdown_req->fn = be_shutdown;
+
+    shutdown_req->pvt = ctx->pvt_data;
+
+    ret = be_file_request(ctx, ctx->ops->finalize, shutdown_req);
+    if (ret == EOK) return EOK;
+
+fail:
+    /* If we got here, we couldn't shut down cleanly. */
+    DEBUG(0, ("ERROR: could not shut down cleanly.\n"));
+    return ret;
+}
+
 static int load_backend(struct be_ctx *ctx)
 {
     TALLOC_CTX *tmp_ctx;
@@ -827,13 +912,19 @@ int main(int argc, const char *argv[])
     if (!srv_name) return 2;
 
     ret = server_setup(srv_name, 0, &main_ctx);
-    if (ret != EOK) return 2;
+    if (ret != EOK) {
+        DEBUG(0, ("Could not set up mainloop [%d]\n", ret));
+        return 2;
+    }
 
     ret = be_process_init(main_ctx,
                           be_name, be_domain,
                           main_ctx->event_ctx,
                           main_ctx->confdb_ctx);
-    if (ret != EOK) return 3;
+    if (ret != EOK) {
+        DEBUG(0, ("Could not initialize backend [%d]\n", ret));
+        return 3;
+    }
 
     DEBUG(1, ("Backend provider %s(%s) started!\n", be_name, be_domain));
 
diff --git a/server/providers/dp_backend.h b/server/providers/dp_backend.h
index 4dd2ee476..12cfb3a51 100644
--- a/server/providers/dp_backend.h
+++ b/server/providers/dp_backend.h
@@ -26,6 +26,10 @@
 #include "db/sysdb.h"
 #include "responder/pam/pamsrv.h"
 
+struct be_ctx;
+
+typedef void (*be_shutdown_fn)(void *);
+
 struct be_mod_ops;
 
 struct be_ctx {
@@ -40,6 +44,7 @@ struct be_ctx {
     const char *conf_path;
     struct be_mod_ops *ops;
     void *pvt_data;
+    be_shutdown_fn shutdown;
 };
 
 struct be_req;
@@ -65,12 +70,19 @@ struct be_online_req {
     int online;
 };
 
+struct be_pam_handler {
+    int pam_status;
+    const char *domain;
+    struct pam_data *pd;
+};
+
 typedef void (*be_req_fn_t)(struct be_req *);
 
 struct be_mod_ops {
     be_req_fn_t check_online;
     be_req_fn_t get_account_info;
     be_req_fn_t pam_handler;
+    be_req_fn_t finalize;
 };
 
 #endif /* __DP_BACKEND_H___ */
diff --git a/server/providers/ldap_be.c b/server/providers/ldap_be.c
index d91e07924..89da07153 100644
--- a/server/providers/ldap_be.c
+++ b/server/providers/ldap_be.c
@@ -622,10 +622,17 @@ done:
     req->fn(req, pam_status, NULL);
 }
 
+static void ldap_shutdown(struct be_req *req)
+{
+    /* TODO: Clean up any internal data */
+    req->fn(req, EOK, NULL);
+}
+
 struct be_mod_ops ldap_mod_ops = {
     .check_online = NULL,
     .get_account_info = NULL,
-    .pam_handler = ldap_pam_handler
+    .pam_handler = ldap_pam_handler,
+    .finalize = ldap_shutdown
 };
 
 
diff --git a/server/providers/proxy.c b/server/providers/proxy.c
index 4b2ed7d65..cc1da1691 100644
--- a/server/providers/proxy.c
+++ b/server/providers/proxy.c
@@ -85,8 +85,8 @@ static int proxy_internal_conv(int num_msg, const struct pam_message **msgm,
         switch( msgm[i]->msg_style ) {
             case PAM_PROMPT_ECHO_OFF:
                 DEBUG(4, ("Conversation message: %s.\n", msgm[i]->msg));
-                reply[i].resp_retcode = 0; 
-                reply[i].resp = strdup(auth_data->authtok); 
+                reply[i].resp_retcode = 0;
+                reply[i].resp = strdup(auth_data->authtok);
                 break;
             default:
                 DEBUG(1, ("Conversation style %d not supported.\n",
@@ -1050,10 +1050,17 @@ static void proxy_get_account_info(struct be_req *req)
     }
 }
 
+static void proxy_shutdown(struct be_req *req)
+{
+    /* TODO: Clean up any internal data */
+    req->fn(req, EOK, NULL);
+}
+
 struct be_mod_ops proxy_mod_ops = {
     .check_online = proxy_check_online,
     .get_account_info = proxy_get_account_info,
-    .pam_handler = proxy_pam_handler
+    .pam_handler = proxy_pam_handler,
+    .finalize = proxy_shutdown
 };
 
 static void *proxy_dlsym(void *handle, const char *functemp, char *libname)