Clean up cache on server reinitialization

https://fedorahosted.org/sssd/ticket/734

We successfully detect when the server is reinitialized by testing
the new lastUSN value. The maximum USN values are set to zero, but
the current cache content remains.

This patch removes records that were deleted from the server.
It uses the following approach:
  1. remove entryUSN attribute from all entries
  2. run enumeration
  3. remove records that doesn't have entryUSN attribute updated

We don't need to do this for sudo rules, they will be refreshed
automatically during next smart/full refresh, or when an expired rule
is deleted.

5 files changed, 403 insertions, 4 deletions

diff --git a/src/providers/ldap/ldap_common.h b/src/providers/ldap/ldap_common.h
index 1773f37e7..034dc995b 100644
--- a/src/providers/ldap/ldap_common.h
+++ b/src/providers/ldap/ldap_common.h
@@ -76,6 +76,12 @@ int sssm_ldap_id_init(struct be_ctx *bectx,
 void sdap_check_online(struct be_req *breq);
 void sdap_do_online_check(struct be_req *be_req, struct sdap_id_ctx *ctx);
 
+struct tevent_req* sdap_reinit_cleanup_send(TALLOC_CTX *mem_ctx,
+                                            struct be_ctx *be_ctx,
+                                            struct sdap_id_ctx *id_ctx);
+
+errno_t sdap_reinit_cleanup_recv(struct tevent_req *req);
+
 /* id */
 void sdap_account_info_handler(struct be_req *breq);
 void sdap_handle_account_info(struct be_req *breq, struct sdap_id_ctx *ctx);
@@ -140,6 +146,9 @@ int ldap_get_autofs_options(TALLOC_CTX *memctx,
 int ldap_id_enumerate_set_timer(struct sdap_id_ctx *ctx, struct timeval tv);
 int ldap_id_cleanup_set_timer(struct sdap_id_ctx *ctx, struct timeval tv);
 
+struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
+                                          struct sdap_id_ctx *ctx);
+
 void sdap_mark_offline(struct sdap_id_ctx *ctx);
 
 struct tevent_req *users_get_send(TALLOC_CTX *memctx,
diff --git a/src/providers/ldap/ldap_id.c b/src/providers/ldap/ldap_id.c
index 9515219cb..b8520df83 100644
--- a/src/providers/ldap/ldap_id.c
+++ b/src/providers/ldap/ldap_id.c
@@ -797,6 +797,8 @@ fail:
     sdap_handler_done(be_req, DP_ERR_FATAL, ret, NULL);
 }
 
+static void sdap_check_online_reinit_done(struct tevent_req *req);
+
 static void sdap_check_online_done(struct tevent_req *req)
 {
     struct sdap_online_check_ctx *check_ctx = tevent_req_callback_data(req,
@@ -806,6 +808,9 @@ static void sdap_check_online_done(struct tevent_req *req)
     bool can_retry;
     struct sdap_server_opts *srv_opts;
     struct be_req *be_req;
+    struct sdap_id_ctx *id_ctx;
+    struct tevent_req *reinit_req = NULL;
+    bool reinit = false;
 
     ret = sdap_cli_connect_recv(req, NULL, &can_retry, NULL, &srv_opts);
     talloc_zfree(req);
@@ -830,16 +835,56 @@ static void sdap_check_online_done(struct tevent_req *req)
             check_ctx->id_ctx->srv_opts->max_service_value = 0;
             check_ctx->id_ctx->srv_opts->max_sudo_value = 0;
             check_ctx->id_ctx->srv_opts->last_usn = srv_opts->last_usn;
+
+            reinit = true;
         }
 
         sdap_steal_server_opts(check_ctx->id_ctx, &srv_opts);
     }
 
     be_req = check_ctx->be_req;
+    id_ctx = check_ctx->id_ctx;
     talloc_free(check_ctx);
+
+    if (reinit) {
+        DEBUG(SSSDBG_TRACE_FUNC, ("Server reinitialization detected. "
+                                  "Cleaning cache.\n"));
+        reinit_req = sdap_reinit_cleanup_send(be_req, be_req->be_ctx, id_ctx);
+        if (reinit_req == NULL) {
+            DEBUG(SSSDBG_CRIT_FAILURE, ("Unable to perform reinitialization "
+                                        "clean up.\n"));
+            /* not fatal */
+            goto done;
+        }
+
+        tevent_req_set_callback(reinit_req, sdap_check_online_reinit_done,
+                                be_req);
+        return;
+    }
+
+done:
     sdap_handler_done(be_req, dp_err, 0, NULL);
 }
 
+static void sdap_check_online_reinit_done(struct tevent_req *req)
+{
+    struct be_req *be_req = NULL;
+    errno_t ret;
+
+    be_req = tevent_req_callback_data(req, struct be_req);
+    ret = sdap_reinit_cleanup_recv(req);
+    talloc_zfree(req);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("Unable to perform reinitialization "
+              "clean up [%d]: %s\n", ret, strerror(ret)));
+        /* not fatal */
+    } else {
+        DEBUG(SSSDBG_TRACE_FUNC, ("Reinitialization clean up completed\n"));
+    }
+
+    sdap_handler_done(be_req, DP_ERR_OK, 0, NULL);
+}
+
 /* =Get-Account-Info-Call================================================= */
 
 /* FIXME: embed this function in sssd_be and only call out
diff --git a/src/providers/ldap/ldap_id_enum.c b/src/providers/ldap/ldap_id_enum.c
index 53fc99fba..88036cc26 100644
--- a/src/providers/ldap/ldap_id_enum.c
+++ b/src/providers/ldap/ldap_id_enum.c
@@ -37,8 +37,6 @@ extern struct tevent_req *ldap_id_cleanup_send(TALLOC_CTX *memctx,
 
 /* ==Enumeration-Task===================================================== */
 
-static struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
-                                                 struct sdap_id_ctx *ctx);
 static int ldap_id_enumerate_retry(struct tevent_req *req);
 static void ldap_id_enumerate_connect_done(struct tevent_req *req);
 
@@ -202,8 +200,8 @@ static void ldap_id_enum_groups_done(struct tevent_req *subreq);
 static void ldap_id_enum_services_done(struct tevent_req *subreq);
 static void ldap_id_enum_cleanup_done(struct tevent_req *subreq);
 
-static struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
-                                                 struct sdap_id_ctx *ctx)
+struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
+                                          struct sdap_id_ctx *ctx)
 {
     struct global_enum_state *state;
     struct tevent_req *req;
diff --git a/src/providers/ldap/sdap_id_op.c b/src/providers/ldap/sdap_id_op.c
index 3036d0cc1..034813012 100644
--- a/src/providers/ldap/sdap_id_op.c
+++ b/src/providers/ldap/sdap_id_op.c
@@ -519,6 +519,8 @@ done:
     return ret;
 }
 
+static void sdap_id_op_connect_reinit_done(struct tevent_req *req);
+
 /* Subrequest callback for connection completion */
 static void sdap_id_op_connect_done(struct tevent_req *subreq)
 {
@@ -529,6 +531,8 @@ static void sdap_id_op_connect_done(struct tevent_req *subreq)
     struct sdap_server_opts *current_srv_opts = NULL;
     bool can_retry = false;
     bool is_offline = false;
+    struct tevent_req *reinit_req = NULL;
+    bool reinit = false;
     int ret;
 
     ret = sdap_cli_connect_recv(subreq, conn_data, &can_retry,
@@ -570,6 +574,8 @@ static void sdap_id_op_connect_done(struct tevent_req *subreq)
                 current_srv_opts->max_service_value = 0;
                 current_srv_opts->max_sudo_value = 0;
                 current_srv_opts->last_usn = srv_opts->last_usn;
+
+                reinit = true;
             }
         }
         ret = sdap_id_conn_data_set_expire_timer(conn_data);
@@ -694,6 +700,38 @@ static void sdap_id_op_connect_done(struct tevent_req *subreq)
 
         sdap_id_release_conn_data(conn_data);
     }
+
+    if (reinit) {
+        DEBUG(SSSDBG_TRACE_FUNC, ("Server reinitialization detected. "
+                                  "Cleaning cache.\n"));
+        reinit_req = sdap_reinit_cleanup_send(conn_cache->id_ctx->be,
+                                              conn_cache->id_ctx->be,
+                                              conn_cache->id_ctx);
+        if (reinit_req == NULL) {
+            DEBUG(SSSDBG_CRIT_FAILURE, ("Unable to perform reinitialization "
+                                        "clean up.\n"));
+            return;
+        }
+
+        tevent_req_set_callback(reinit_req, sdap_id_op_connect_reinit_done,
+                                NULL);
+    }
+}
+
+static void sdap_id_op_connect_reinit_done(struct tevent_req *req)
+{
+    errno_t ret;
+
+    ret = sdap_reinit_cleanup_recv(req);
+    talloc_zfree(req);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("Unable to perform reinitialization "
+              "clean up [%d]: %s\n", ret, strerror(ret)));
+        /* not fatal */
+        return;
+    }
+
+    DEBUG(SSSDBG_TRACE_FUNC, ("Reinitialization clean up completed\n"));
 }
 
 /* Mark operation connection request as complete */
diff --git a/src/providers/ldap/sdap_reinit.c b/src/providers/ldap/sdap_reinit.c
new file mode 100644
index 000000000..0200de0a2
--- /dev/null
+++ b/src/providers/ldap/sdap_reinit.c
@@ -0,0 +1,309 @@
+/*
+    Authors:
+        Pavel B??ezina <pbrezina@redhat.com>
+
+    Copyright (C) 2012 Red Hat
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <talloc.h>
+#include <tevent.h>
+#include <string.h>
+#include <ldb.h>
+
+#include "util/util.h"
+#include "providers/ldap/ldap_common.h"
+#include "db/sysdb.h"
+#include "db/sysdb_services.h"
+#include "db/sysdb_private.h"
+
+struct sdap_reinit_cleanup_state {
+    struct sysdb_ctx *sysdb;
+};
+
+static errno_t sdap_reinit_clear_usn(struct sysdb_ctx *sysdb);
+static void sdap_reinit_cleanup_done(struct tevent_req *subreq);
+static errno_t sdap_reinit_delete_records(struct sysdb_ctx *sysdb);
+
+struct tevent_req* sdap_reinit_cleanup_send(TALLOC_CTX *mem_ctx,
+                                            struct be_ctx *be_ctx,
+                                            struct sdap_id_ctx *id_ctx)
+{
+    struct tevent_req *req = NULL;
+    struct tevent_req *subreq = NULL;
+    struct sdap_reinit_cleanup_state *state;
+    int ret;
+
+    /*
+     * 1. remove entryUSN attribute from all entries
+     * 2. run enumeration
+     * 3. remove records that doesn't have entryUSN attribute updated
+     *
+     * We don't need to do this for sudo rules, they will be refreshed
+     * automatically during next smart/full refresh, or when an expired rule
+     * is deleted.
+     */
+
+    req = tevent_req_create(mem_ctx, &state, struct sdap_reinit_cleanup_state);
+    if (req == NULL) {
+        return NULL;
+    }
+
+    if (!be_ctx->domain->enumerate) {
+        /* enumeration is disabled, this whole process is meaningless */
+        ret = EOK;
+        goto immediately;
+    }
+
+    ret = sdap_reinit_clear_usn(be_ctx->domain->sysdb);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("Unable to clear USN attributes [%d]: %s\n",
+                                    ret, strerror(ret)));
+        goto immediately;
+    }
+
+    req = ldap_id_enumerate_send(be_ctx->ev, id_ctx);
+    if (req == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("Unable to issue enumeration request\n"));
+        ret = ENOMEM;
+        goto immediately;
+    }
+
+    tevent_req_set_callback(subreq, sdap_reinit_cleanup_done, req);
+
+    return req;
+
+immediately:
+    if (ret != EOK) {
+        tevent_req_error(req, ret);
+    } else {
+        tevent_req_done(req);
+    }
+    tevent_req_post(req, be_ctx->ev);
+
+    return req;
+}
+
+static errno_t sdap_reinit_clear_usn(struct sysdb_ctx *sysdb)
+{
+    TALLOC_CTX *tmp_ctx = NULL;
+    bool in_transaction = false;
+    struct ldb_result *result = NULL;
+    struct ldb_message **messages = NULL;
+    struct ldb_message *msg = NULL;
+    int messages_num = 0;
+    struct ldb_dn *base_dn = NULL;
+    const char *base[] = { SYSDB_TMPL_USER_BASE,
+                           SYSDB_TMPL_GROUP_BASE,
+                           SYSDB_TMPL_SVC_BASE,
+                           NULL };
+    const char *attrs[] = { "dn", NULL };
+    int i, j;
+    int sret;
+    int lret;
+    errno_t ret;
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("talloc_new() failed\n"));
+        return ENOMEM;
+    }
+
+    for (i = 0; base[i] != NULL; i++) {
+        lret = ldb_search(sysdb->ldb, tmp_ctx, &result, base_dn,
+                          LDB_SCOPE_SUBTREE, attrs, NULL);
+        if (lret != LDB_SUCCESS) {
+            ret = sysdb_error_to_errno(lret);
+            goto done;
+        }
+
+        if (result->count == 0) {
+            talloc_zfree(result);
+            continue;
+        }
+
+        messages = talloc_realloc(tmp_ctx, messages, struct ldb_message*,
+                                  messages_num + result->count);
+
+        for (j = 0; j < result->count; j++) {
+            msg = ldb_msg_new(messages);
+            if (msg == NULL) {
+                ret = ENOMEM;
+                goto done;
+            }
+            msg->dn = talloc_move(tmp_ctx, &result->msgs[j]->dn);
+
+            lret = ldb_msg_add_empty(msg, SYSDB_USN, LDB_FLAG_MOD_DELETE, NULL);
+            if (lret != LDB_SUCCESS) {
+                ret = sysdb_error_to_errno(lret);
+                goto done;
+            }
+
+            messages[messages_num + j] = msg;
+        }
+
+        messages_num += result->count;
+        talloc_zfree(result);
+    }
+
+    ret = sysdb_transaction_start(sysdb);
+    if (ret != EOK) {
+        goto done;
+    }
+    in_transaction = true;
+
+    for (i = 0; i < messages_num; i++) {
+        lret = ldb_modify(sysdb->ldb, messages[i]);
+        if (lret != LDB_SUCCESS) {
+            ret = sysdb_error_to_errno(lret);
+            goto done;
+        }
+    }
+
+    ret = sysdb_transaction_commit(sysdb);
+    if (ret == EOK) {
+        in_transaction = false;
+    } else {
+        DEBUG(SSSDBG_MINOR_FAILURE, ("Could not commit transaction\n"));
+    }
+
+done:
+    if (in_transaction) {
+        sret = sysdb_transaction_cancel(sysdb);
+        if (sret != EOK) {
+            DEBUG(SSSDBG_OP_FAILURE, ("Could not cancel transaction\n"));
+        }
+    }
+
+    talloc_free(tmp_ctx);
+
+    return ret;
+}
+
+static void sdap_reinit_cleanup_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = NULL;
+    struct sdap_reinit_cleanup_state *state = NULL;
+    enum tevent_req_state tstate;
+    uint64_t err;
+    errno_t ret;
+
+    req = tevent_req_callback_data(subreq, struct tevent_req);
+    state = tevent_req_data(req, struct sdap_reinit_cleanup_state);
+
+    if (tevent_req_is_error(subreq, &tstate, &err)) {
+        ret = err;
+        DEBUG(SSSDBG_CRIT_FAILURE, ("Domain enumeration failed [%d]: %s\n",
+                                    err, strerror(err)));
+        goto fail;
+    }
+
+    /* Ok, we've completed an enumeration. Save this to the
+     * sysdb so we can postpone starting up the enumeration
+     * process on the next SSSD service restart (to avoid
+     * slowing down system boot-up
+     */
+    ret = sysdb_set_enumerated(state->sysdb, true);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_MINOR_FAILURE, ("Could not mark domain as having "
+                                     "enumerated.\n"));
+        /* This error is non-fatal, so continue */
+    }
+
+    ret = sdap_reinit_delete_records(state->sysdb);
+    if (ret != EOK) {
+        goto fail;
+    }
+
+    tevent_req_done(req);
+    return;
+
+fail:
+    tevent_req_error(req, ret);
+}
+
+static errno_t sdap_reinit_delete_records(struct sysdb_ctx *sysdb)
+{
+    TALLOC_CTX *tmp_ctx = NULL;
+    bool in_transaction = false;
+    struct ldb_result *result = NULL;
+    struct ldb_dn *base_dn = NULL;
+    const char *base[] = { SYSDB_TMPL_USER_BASE,
+                           SYSDB_TMPL_GROUP_BASE,
+                           SYSDB_TMPL_SVC_BASE,
+                           NULL };
+    const char *attrs[] = { "dn", NULL };
+    int i, j;
+    int sret;
+    int lret;
+    errno_t ret;
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("talloc_new() failed\n"));
+        return ENOMEM;
+    }
+
+    ret = sysdb_transaction_start(sysdb);
+    if (ret != EOK) {
+        goto done;
+    }
+    in_transaction = true;
+
+    for (i = 0; base[i] != NULL; i++) {
+        lret = ldb_search(sysdb->ldb, tmp_ctx, &result, base_dn,
+                          LDB_SCOPE_SUBTREE, attrs, "(!("SYSDB_USN"=*))");
+        if (lret != LDB_SUCCESS) {
+            ret = sysdb_error_to_errno(lret);
+            goto done;
+        }
+
+        for (j = 0; j < result->count; j++) {
+            ret = ldb_delete(sysdb->ldb, result->msgs[i]->dn);
+            if (ret != LDB_SUCCESS) {
+                ret = sysdb_error_to_errno(ret);
+                goto done;
+            }
+        }
+
+        talloc_zfree(result);
+    }
+
+    ret = sysdb_transaction_commit(sysdb);
+    if (ret == EOK) {
+        in_transaction = false;
+    } else {
+        DEBUG(SSSDBG_MINOR_FAILURE, ("Could not commit transaction\n"));
+    }
+
+done:
+    if (in_transaction) {
+        sret = sysdb_transaction_cancel(sysdb);
+        if (sret != EOK) {
+            DEBUG(SSSDBG_OP_FAILURE, ("Could not cancel transaction\n"));
+        }
+    }
+
+    talloc_free(tmp_ctx);
+
+    return ret;
+}
+
+errno_t sdap_reinit_cleanup_recv(struct tevent_req *req)
+{
+    TEVENT_REQ_RETURN_ON_ERROR(req);
+
+    return EOK;
+}