Better cleanup task handling

Implements a different mechanism for cleanup task. Instead of just
deleting expired entries, this patch adds a new option
account_cache_expiration for domains. If an entry is expired and the last
login was more days in the past that account_cache_expiration, the entry is
deleted.

Groups are deleted if they are expired and and no user references them
(no user has memberof: attribute pointing at that group).

The parameter account_cache_expiration is not LDAP-specific, so that other
future backends might use the same timeout setting.

Fixes: #391

11 files changed, 263 insertions, 39 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index a37cf7db1..fa62702bc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -664,6 +664,7 @@ dist_noinst_DATA += \
 # Plugin Libraries #
 ####################
 libsss_ldap_la_SOURCES = \
+    util/find_uid.c \
     providers/child_common.c \
     providers/ldap/ldap_id.c \
     providers/ldap/ldap_id_enum.c \
diff --git a/src/confdb/confdb.h b/src/confdb/confdb.h
index 66576c354..06faa43ba 100644
--- a/src/confdb/confdb.h
+++ b/src/confdb/confdb.h
@@ -101,6 +101,7 @@
 #define CONFDB_DOMAIN_FQ "use_fully_qualified_names"
 #define CONFDB_DOMAIN_ENTRY_CACHE_TIMEOUT "entry_cache_timeout"
 #define CONFDB_DOMAIN_FAMILY_ORDER "lookup_family_order"
+#define CONFDB_DOMAIN_ACCOUNT_CACHE_EXPIRATION "account_cache_expiration"
 
 /* Local Provider */
 #define CONFDB_LOCAL_DEFAULT_SHELL   "default_shell"
diff --git a/src/config/SSSDConfig.py b/src/config/SSSDConfig.py
index 471ecb6c5..2697c71ba 100644
--- a/src/config/SSSDConfig.py
+++ b/src/config/SSSDConfig.py
@@ -80,6 +80,7 @@ option_strings = {
     'use_fully_qualified_names' : _('Display users/groups in fully-qualified form'),
     'entry_cache_timeout' : _('Entry cache timeout length (seconds)'),
     'lookup_family_order' : _('Restrict or prefer a specific address family when performing DNS lookups'),
+    'account_cache_expiration' : _('How long to keep cached entries after last successful login (days)'),
 
     # [provider/ipa]
     'ipa_domain' : _('IPA domain'),
diff --git a/src/config/SSSDConfigTest.py b/src/config/SSSDConfigTest.py
index eed1de311..9f9e75f56 100755
--- a/src/config/SSSDConfigTest.py
+++ b/src/config/SSSDConfigTest.py
@@ -415,6 +415,7 @@ class SSSDConfigTestSSSDDomain(unittest.TestCase):
             'use_fully_qualified_names',
             'entry_cache_timeout',
             'lookup_family_order',
+            'account_cache_expiration',
             'id_provider',
             'auth_provider',
             'access_provider',
@@ -725,6 +726,7 @@ class SSSDConfigTestSSSDDomain(unittest.TestCase):
             'store_legacy_passwords',
             'use_fully_qualified_names',
             'entry_cache_timeout',
+            'account_cache_expiration',
             'lookup_family_order',
             'id_provider',
             'auth_provider',
diff --git a/src/config/etc/sssd.api.conf b/src/config/etc/sssd.api.conf
index 35890acc0..14ec30835 100644
--- a/src/config/etc/sssd.api.conf
+++ b/src/config/etc/sssd.api.conf
@@ -55,6 +55,7 @@ store_legacy_passwords = bool, None, false
 use_fully_qualified_names = bool, None, false
 entry_cache_timeout = int, None, false
 lookup_family_order = str, None, false
+account_cache_expiration = int, None, false
 
 # Special providers
 [provider/permit]
diff --git a/src/man/sssd.conf.5.xml b/src/man/sssd.conf.5.xml
index 665fa79ee..171d261b6 100644
--- a/src/man/sssd.conf.5.xml
+++ b/src/man/sssd.conf.5.xml
@@ -460,6 +460,21 @@
                         </para>
                     </listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term>account_cache_expiration (integer)</term>
+                    <listitem>
+                        <para>
+                            Number of days entries are left in cache after
+                            last successful login before being removed during
+                            a cleanup of the cache. 0 means keep forever.
+                            The value of this parameter must be bigger than
+                            offline_credentials_expiration.
+                        </para>
+                        <para>
+                            Default: 0 (unlimited)
+                        </para>
+                    </listitem>
+                </varlistentry>
 
                 <varlistentry>
                     <term>id_provider (string)</term>
diff --git a/src/providers/ipa/ipa_common.c b/src/providers/ipa/ipa_common.c
index 7686227a5..92da9d170 100644
--- a/src/providers/ipa/ipa_common.c
+++ b/src/providers/ipa/ipa_common.c
@@ -64,7 +64,8 @@ struct dp_option ipa_def_ldap_opts[] = {
     /* use the same parm name as the krb5 module so we set it only once */
     { "krb5_realm", DP_OPT_STRING, NULL_STRING, NULL_STRING },
     { "ldap_pwd_policy", DP_OPT_STRING, { "none" } , NULL_STRING },
-    { "ldap_referrals", DP_OPT_BOOL, BOOL_TRUE, BOOL_TRUE }
+    { "ldap_referrals", DP_OPT_BOOL, BOOL_TRUE, BOOL_TRUE },
+    { "account_cache_expiration", DP_OPT_NUMBER, { .number = 0 }, NULL_NUMBER }
 };
 
 struct sdap_attr_map ipa_attr_map[] = {
diff --git a/src/providers/ipa/ipa_common.h b/src/providers/ipa/ipa_common.h
index 60c7313f0..75be55e2d 100644
--- a/src/providers/ipa/ipa_common.h
+++ b/src/providers/ipa/ipa_common.h
@@ -35,7 +35,7 @@ struct ipa_service {
 /* the following define is used to keep track of the options in the ldap
  * module, so that if they change and ipa is not updated correspondingly
  * this will trigger a runtime abort error */
-#define IPA_OPTS_BASIC_TEST 31
+#define IPA_OPTS_BASIC_TEST 32
 
 enum ipa_basic_opt {
     IPA_DOMAIN = 0,
diff --git a/src/providers/ldap/ldap_common.c b/src/providers/ldap/ldap_common.c
index 61cba03e7..a67ea3626 100644
--- a/src/providers/ldap/ldap_common.c
+++ b/src/providers/ldap/ldap_common.c
@@ -62,7 +62,8 @@ struct dp_option default_basic_opts[] = {
     /* use the same parm name as the krb5 module so we set it only once */
     { "krb5_realm", DP_OPT_STRING, NULL_STRING, NULL_STRING },
     { "ldap_pwd_policy", DP_OPT_STRING, { "none" } , NULL_STRING },
-    { "ldap_referrals", DP_OPT_BOOL, BOOL_TRUE, BOOL_TRUE }
+    { "ldap_referrals", DP_OPT_BOOL, BOOL_TRUE, BOOL_TRUE },
+    { "account_cache_expiration", DP_OPT_NUMBER, { .number = 0 }, NULL_NUMBER }
 };
 
 struct sdap_attr_map generic_attr_map[] = {
@@ -166,6 +167,8 @@ int ldap_get_options(TALLOC_CTX *memctx,
     char *schema;
     const char *pwd_policy;
     int ret;
+    int account_cache_expiration;
+    int offline_credentials_expiration;
 
     opts = talloc_zero(memctx, struct sdap_options);
     if (!opts) return ENOMEM;
@@ -217,6 +220,48 @@ int ldap_get_options(TALLOC_CTX *memctx,
         goto done;
     }
 
+    /* account_cache_expiration must be >= than offline_credentials_expiration */
+    ret = confdb_get_int(cdb, memctx, CONFDB_PAM_CONF_ENTRY,
+                         CONFDB_PAM_CRED_TIMEOUT, 0,
+                         &offline_credentials_expiration);
+    if (ret != EOK) {
+        DEBUG(1, ("Cannot get value of %s from confdb \n",
+                  CONFDB_PAM_CRED_TIMEOUT));
+        goto done;
+    }
+
+    account_cache_expiration = dp_opt_get_int(opts->basic,
+                                              SDAP_ACCOUNT_CACHE_EXPIRATION);
+
+    /* account cache_expiration must not be smaller than
+     * offline_credentials_expiration to prevent deleting entries that
+     * still contain credentials valid for offline login.
+     *
+     * offline_credentials_expiration == 0 is a special case that says
+     * that the cached credentials are valid forever. Therefore, the cached
+     * entries must not be purged from cache.
+     */
+    if (!offline_credentials_expiration && account_cache_expiration) {
+        DEBUG(1, ("Conflicting values for options %s (unlimited) "
+                  "and %s (%d)\n",
+                  opts->basic[SDAP_ACCOUNT_CACHE_EXPIRATION].opt_name,
+                  CONFDB_PAM_CRED_TIMEOUT,
+                  offline_credentials_expiration));
+        ret = EINVAL;
+        goto done;
+    }
+    if (offline_credentials_expiration && account_cache_expiration &&
+        offline_credentials_expiration >= account_cache_expiration) {
+        DEBUG(1, ("Value of %s (now %d) must be larger "
+                  "than value of %s (now %d)\n",
+                  opts->basic[SDAP_ACCOUNT_CACHE_EXPIRATION].opt_name,
+                  account_cache_expiration,
+                  CONFDB_PAM_CRED_TIMEOUT,
+                  offline_credentials_expiration));
+        ret = EINVAL;
+        goto done;
+    }
+
 
 #ifndef HAVE_LDAP_CONNCB
     bool ldap_referrals;
diff --git a/src/providers/ldap/ldap_id_cleanup.c b/src/providers/ldap/ldap_id_cleanup.c
index 02b750bca..64de54033 100644
--- a/src/providers/ldap/ldap_id_cleanup.c
+++ b/src/providers/ldap/ldap_id_cleanup.c
@@ -27,6 +27,7 @@
 #include <sys/time.h>
 
 #include "util/util.h"
+#include "util/find_uid.h"
 #include "db/sysdb.h"
 #include "providers/ldap/ldap_common.h"
 #include "providers/ldap/sdap_async.h"
@@ -143,14 +144,12 @@ int ldap_id_cleanup_set_timer(struct sdap_id_ctx *ctx, struct timeval tv)
 
 struct global_cleanup_state {
     struct tevent_context *ev;
-    struct sysdb_ctx *sysdb;
-    struct sss_domain_info *domain;
+    struct sdap_id_ctx *ctx;
 };
 
 static struct tevent_req *cleanup_users_send(TALLOC_CTX *memctx,
                                              struct tevent_context *ev,
-                                             struct sysdb_ctx *sysdb,
-                                             struct sss_domain_info *domain);
+                                             struct sdap_id_ctx *ctx);
 static void ldap_id_cleanup_users_done(struct tevent_req *subreq);
 static struct tevent_req *cleanup_groups_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
@@ -169,10 +168,9 @@ struct tevent_req *ldap_id_cleanup_send(TALLOC_CTX *memctx,
     if (!req) return NULL;
 
     state->ev = ev;
-    state->sysdb = ctx->be->sysdb;
-    state->domain = ctx->be->domain;
+    state->ctx = ctx;
 
-    subreq = cleanup_users_send(state, ev, state->sysdb, state->domain);
+    subreq = cleanup_users_send(state, ev, state->ctx);
     if (!subreq) {
         talloc_zfree(req);
         return NULL;
@@ -204,7 +202,8 @@ static void ldap_id_cleanup_users_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
 
     subreq = cleanup_groups_send(state, state->ev,
-                                 state->sysdb, state->domain);
+                                 state->ctx->be->sysdb,
+                                 state->ctx->be->domain);
     if (!subreq) {
         err = ENOMEM;
         goto fail;
@@ -252,28 +251,34 @@ struct cleanup_users_state {
     struct tevent_context *ev;
     struct sysdb_ctx *sysdb;
     struct sss_domain_info *domain;
+    struct sdap_id_ctx *ctx;
 
     struct sysdb_handle *handle;
 
+    hash_table_t *uid_table;
+
     struct ldb_message **msgs;
     size_t count;
     int cur;
 };
 
 static void cleanup_users_process(struct tevent_req *subreq);
+static int cleanup_users_logged_in(hash_table_t *table,
+                                   const struct ldb_message *msg);
 static void cleanup_users_delete(struct tevent_req *req);
+static void cleanup_users_next(struct tevent_req *req);
 static void cleanup_users_delete_done(struct tevent_req *subreq);
 
 static struct tevent_req *cleanup_users_send(TALLOC_CTX *memctx,
                                              struct tevent_context *ev,
-                                             struct sysdb_ctx *sysdb,
-                                             struct sss_domain_info *domain)
+                                             struct sdap_id_ctx *ctx)
 {
     struct tevent_req *req, *subreq;
     struct cleanup_users_state *state;
-    static const char *attrs[] = { SYSDB_NAME, NULL };
+    static const char *attrs[] = { SYSDB_NAME, SYSDB_UIDNUM, NULL };
     time_t now = time(NULL);
     char *subfilter;
+    int account_cache_expiration;
 
     req = tevent_req_create(memctx, &state, struct cleanup_users_state);
     if (!req) {
@@ -281,15 +286,41 @@ static struct tevent_req *cleanup_users_send(TALLOC_CTX *memctx,
     }
 
     state->ev = ev;
-    state->sysdb = sysdb;
-    state->domain = domain;
+    state->sysdb = ctx->be->sysdb;
+    state->domain = ctx->be->domain;
+    state->ctx  = ctx;
     state->msgs = NULL;
     state->count = 0;
     state->cur = 0;
 
-    subfilter = talloc_asprintf(state, "(&(!(%s=0))(%s<=%ld))",
-                                SYSDB_CACHE_EXPIRE,
-                                SYSDB_CACHE_EXPIRE, (long)now);
+    account_cache_expiration = dp_opt_get_int(state->ctx->opts->basic,
+                                           SDAP_ACCOUNT_CACHE_EXPIRATION);
+    DEBUG(9, ("Cache expiration is set to %d days\n",
+              account_cache_expiration));
+
+    if (!subfilter) {
+        DEBUG(2, ("Failed to build filter\n"));
+        talloc_zfree(req);
+        return NULL;
+    }
+
+    if (account_cache_expiration > 0) {
+        subfilter = talloc_asprintf(state,
+                                    "(&(!(%s=0))(%s<=%ld)(|(!(%s=*))(%s<=%ld)))",
+                                    SYSDB_CACHE_EXPIRE,
+                                    SYSDB_CACHE_EXPIRE,
+                                    (long) now,
+                                    SYSDB_LAST_LOGIN,
+                                    SYSDB_LAST_LOGIN,
+                                    (long) (now - (account_cache_expiration * 86400)));
+    } else {
+        subfilter = talloc_asprintf(state,
+                                    "(&(!(%s=0))(%s<=%ld)(!(%s=*)))",
+                                    SYSDB_CACHE_EXPIRE,
+                                    SYSDB_CACHE_EXPIRE,
+                                    (long) now,
+                                    SYSDB_LAST_LOGIN);
+    }
     if (!subfilter) {
         DEBUG(2, ("Failed to build filter\n"));
         talloc_zfree(req);
@@ -305,7 +336,6 @@ static struct tevent_req *cleanup_users_send(TALLOC_CTX *memctx,
         return NULL;
     }
     tevent_req_set_callback(subreq, cleanup_users_process, req);
-
     return req;
 }
 
@@ -334,6 +364,15 @@ static void cleanup_users_process(struct tevent_req *subreq)
         tevent_req_done(req);
     }
 
+    ret = get_uid_table(state, &state->uid_table);
+    /* get_uid_table returns ENOSYS on non-Linux platforms. We proceed with
+     * the cleanup in that case
+     */
+    if (ret != EOK && ret != ENOSYS) {
+        tevent_req_error(req, ret);
+        return;
+    }
+
     cleanup_users_delete(req);
 }
 
@@ -343,6 +382,7 @@ static void cleanup_users_delete(struct tevent_req *req)
     struct cleanup_users_state *state = tevent_req_data(req,
                                                struct cleanup_users_state);
     const char *name;
+    int ret;
 
     name = ldb_msg_find_attr_as_string(state->msgs[state->cur],
                                       SYSDB_NAME, NULL);
@@ -353,6 +393,21 @@ static void cleanup_users_delete(struct tevent_req *req)
         return;
     }
 
+    if (state->uid_table) {
+        ret = cleanup_users_logged_in(state->uid_table, state->msgs[state->cur]);
+        if (ret == EOK) {
+            /* If the user is logged in, proceed to the next one */
+            DEBUG(5, ("User %s is still logged in, keeping his data\n", name));
+            cleanup_users_next(req);
+            return;
+        } else if (ret != ENOENT) {
+            tevent_req_error(req, ret);
+            return;
+        }
+    }
+
+    /* If not logged in or cannot check the table, delete him */
+    DEBUG(9, ("About to delete user %s\n", name));
     subreq = sysdb_delete_user_send(state, state->ev,
                                     state->sysdb, NULL,
                                     state->domain, name, 0);
@@ -361,14 +416,56 @@ static void cleanup_users_delete(struct tevent_req *req)
         return;
     }
     tevent_req_set_callback(subreq, cleanup_users_delete_done, req);
+    return;
+}
+
+static int cleanup_users_logged_in(hash_table_t *table,
+                                   const struct ldb_message *msg)
+{
+    uid_t      uid;
+    hash_key_t key;
+    hash_value_t value;
+    int        ret;
+
+    uid = ldb_msg_find_attr_as_uint64(msg,
+                                      SYSDB_UIDNUM, 0);
+    if (!uid) {
+        DEBUG(2, ("Entry %s has no UID Attribute ?!?\n",
+                  ldb_dn_get_linearized(msg->dn)));
+        return EFAULT;
+    }
+
+    key.type = HASH_KEY_ULONG;
+    key.ul   = (unsigned long) uid;
+
+    ret = hash_lookup(table, &key, &value);
+    if (ret == HASH_SUCCESS) {
+        return EOK;
+    } else if (ret == HASH_ERROR_KEY_NOT_FOUND) {
+        return ENOENT;
+    }
+
+    return EIO;
+}
+
+static void cleanup_users_next(struct tevent_req *req)
+{
+    struct cleanup_users_state *state = tevent_req_data(req,
+                                               struct cleanup_users_state);
+
+    state->cur++;
+    if (state->cur < state->count) {
+        cleanup_users_delete(req);
+        return;
+    }
+
+    tevent_req_done(req);
 }
 
 static void cleanup_users_delete_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
                                                       struct tevent_req);
-    struct cleanup_users_state *state = tevent_req_data(req,
-                                               struct cleanup_users_state);
     int ret;
 
     ret = sysdb_delete_user_recv(subreq);
@@ -380,13 +477,7 @@ static void cleanup_users_delete_done(struct tevent_req *subreq)
         return;
     }
 
-    state->cur++;
-    if (state->cur < state->count) {
-        cleanup_users_delete(req);
-        return;
-    }
-
-    tevent_req_done(req);
+    cleanup_users_next(req);
 }
 
 /* ==Group-Cleanup-Process================================================ */
@@ -404,6 +495,9 @@ struct cleanup_groups_state {
 };
 
 static void cleanup_groups_process(struct tevent_req *subreq);
+static void cleanup_groups_check_users(struct tevent_req *req);
+static void cleanup_groups_check_users_done(struct tevent_req *subreq);
+static void cleanup_groups_next(struct tevent_req *req);
 static void cleanup_groups_delete(struct tevent_req *req);
 static void cleanup_groups_delete_done(struct tevent_req *subreq);
 
@@ -477,7 +571,76 @@ static void cleanup_groups_process(struct tevent_req *subreq)
         tevent_req_done(req);
     }
 
-    cleanup_groups_delete(req);
+    cleanup_groups_check_users(req);
+}
+
+static void cleanup_groups_check_users(struct tevent_req *req)
+{
+    struct cleanup_groups_state *state = tevent_req_data(req,
+                                               struct cleanup_groups_state);
+    struct tevent_req *subreq;
+    const char *subfilter;
+    const char *dn;
+
+    dn = ldb_dn_get_linearized(state->msgs[state->cur]->dn);
+    if (!dn) {
+        tevent_req_error(req, EINVAL);
+        return;
+    }
+
+    subfilter = talloc_asprintf(state, "(%s=%s)",
+                                SYSDB_MEMBEROF, dn);
+    if (!subfilter) {
+        DEBUG(2, ("Failed to build filter\n"));
+        tevent_req_error(req, ENOMEM);
+    }
+
+    subreq = sysdb_search_users_send(state, state->ev,
+                                     state->sysdb, NULL,
+                                     state->domain, subfilter, NULL);
+    if (!subreq) {
+        DEBUG(2, ("Failed to send entry search\n"));
+        tevent_req_error(req, ENOMEM);
+    }
+    tevent_req_set_callback(subreq, cleanup_groups_check_users_done, req);
+}
+
+static void cleanup_groups_check_users_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = tevent_req_callback_data(subreq,
+                                                      struct tevent_req);
+    struct cleanup_groups_state *state = tevent_req_data(req,
+                                               struct cleanup_groups_state);
+    int ret;
+    struct ldb_message **msgs;
+    size_t count;
+
+    ret = sysdb_search_users_recv(subreq, state, &count, &msgs);
+    talloc_zfree(subreq);
+    if (ret != EOK) {
+        if (ret == ENOENT) {
+            cleanup_groups_delete(req);
+            return;
+        }
+        tevent_req_error(req, ret);
+        return;
+    }
+
+    cleanup_groups_next(req);
+}
+
+static void cleanup_groups_next(struct tevent_req *req)
+{
+    struct cleanup_groups_state *state = tevent_req_data(req,
+                                               struct cleanup_groups_state);
+
+    state->cur++;
+    if (state->cur < state->count) {
+        cleanup_groups_check_users(req);
+        return;
+    }
+
+    tevent_req_done(req);
 }
 
 static void cleanup_groups_delete(struct tevent_req *req)
@@ -496,6 +659,7 @@ static void cleanup_groups_delete(struct tevent_req *req)
         return;
     }
 
+    DEBUG(8, ("About to delete group %s\n", name));
     subreq = sysdb_delete_group_send(state, state->ev,
                                      state->sysdb, NULL,
                                      state->domain, name, 0);
@@ -510,8 +674,6 @@ static void cleanup_groups_delete_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
                                                       struct tevent_req);
-    struct cleanup_groups_state *state = tevent_req_data(req,
-                                               struct cleanup_groups_state);
     int ret;
 
     ret = sysdb_delete_group_recv(subreq);
@@ -522,12 +684,6 @@ static void cleanup_groups_delete_done(struct tevent_req *subreq)
         return;
     }
 
-    state->cur++;
-    if (state->cur < state->count) {
-        cleanup_groups_delete(req);
-        return;
-    }
-
-    tevent_req_done(req);
+    cleanup_groups_next(req);
 }
 
diff --git a/src/providers/ldap/sdap.h b/src/providers/ldap/sdap.h
index 16dbb7843..007185fc3 100644
--- a/src/providers/ldap/sdap.h
+++ b/src/providers/ldap/sdap.h
@@ -142,6 +142,7 @@ enum sdap_basic_opt {
     SDAP_KRB5_REALM,
     SDAP_PWD_POLICY,
     SDAP_REFERRALS,
+    SDAP_ACCOUNT_CACHE_EXPIRATION,
 
     SDAP_OPTS_BASIC /* opts counter */
 };