FILES: Add the files provider

Adds a new provider type "files". The provider watches the UNIX password
and group databases for changes using inotify and propagates its
contents to the sysdb.

The files provider is only built on platforms that support the inotify
interface, polling or loading the entries on-deman is not supported.

During initialization, the files are loaded from the environment
variables SSS_FILES_PASSWD and SSS_FILES_GROUP, defaulting to
/etc/passwd and /etc/group respectively. Loading the files from
environment variables is mostly implemented for tests that need to load
nss_wrapped files.

The files provider is a bit different from other provider types in the
sense that it always enumerates full contents of the database.
Therefore, the requests from Data Provider are always just replied to
with success. Enumerating the contents is done in full at the moment,
all users and all groups are removed and added anew. Modifying the
passwd and group databses should be rare enough for this to be
justified and we can optimize the code later.

Since with large databases, the cache update might take a bit of time,
we signal the responders to disable the files domain once we receive the
inotify notification and re-enable the files domain after the update is
finished. The idea is that the NSS configuration would still contain
"files" after "sss" so that if the domain is disabled, libc would fall
back to a direct "files" lookup.

Resolves:
https://fedorahosted.org/sssd/ticket/3262

Reviewed-by: Pavel Březina <pbrezina@redhat.com>

7 files changed, 1179 insertions, 1 deletions

diff --git a/Makefile.am b/Makefile.am
index 09c021d22..e6d3530d0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -339,7 +339,8 @@ sssdlib_LTLIBRARIES = \
     libsss_ldap.la \
     libsss_krb5.la \
     libsss_proxy.la \
-    libsss_simple.la
+    libsss_simple.la \
+    $(NULL)
 
 if BUILD_SAMBA
 sssdlib_LTLIBRARIES += \
@@ -347,6 +348,12 @@ sssdlib_LTLIBRARIES += \
     libsss_ad.la
 endif
 
+if HAVE_INOTIFY
+sssdlib_LTLIBRARIES += \
+    libsss_files.la \
+    $(NULL)
+endif # HAVE_INOTIFY
+
 ldblib_LTLIBRARIES = \
     memberof.la
 
@@ -771,6 +778,7 @@ dist_noinst_HEADERS = \
     src/providers/ad/ad_subdomains.h \
     src/providers/proxy/proxy.h \
     src/providers/proxy/proxy_iface_generated.h \
+    src/providers/files/files_private.h \
     src/tools/tools_util.h \
     src/tools/sss_sync_ops.h \
     src/resolv/async_resolv.h \
@@ -1853,6 +1861,7 @@ if BUILD_SEMANAGE
     FILES_TESTS_LIBS += $(SEMANAGE_LIBS)
 endif
 
+if HAVE_INOTIFY
 files_tests_SOURCES = \
     src/tests/files-tests.c \
     src/util/check_and_open.c \
@@ -1866,6 +1875,7 @@ files_tests_LDADD = \
     $(FILES_TESTS_LIBS) \
     libsss_test_common.la \
     $(SSSD_INTERNAL_LTLIBS)
+endif   # HAVE_INOTIFY
 
 SSSD_RESOLV_TESTS_OBJ = \
     $(SSSD_RESOLV_OBJ)
@@ -3510,6 +3520,23 @@ libsss_proxy_la_LDFLAGS = \
     -avoid-version \
     -module
 
+libsss_files_la_SOURCES = \
+    src/providers/files/files_init.c \
+    src/providers/files/files_id.c \
+    src/providers/files/files_ops.c \
+    src/util/inotify.c \
+    $(NULL)
+libsss_files_la_CFLAGS = \
+    $(AM_CFLAGS) \
+    $(NULL)
+libsss_files_la_LIBADD = \
+    $(PAM_LIBS) \
+    $(NULL)
+libsss_files_la_LDFLAGS = \
+    -avoid-version \
+    -module \
+    $(NULL)
+
 libsss_simple_la_SOURCES = \
     src/providers/simple/simple_access_check.c \
     src/providers/simple/simple_access.c
diff --git a/contrib/sssd.spec.in b/contrib/sssd.spec.in
index c0c9e364b..fe51f9998 100644
--- a/contrib/sssd.spec.in
+++ b/contrib/sssd.spec.in
@@ -848,6 +848,9 @@ done
 %{_sbindir}/sss_cache
 %{_libexecdir}/%{servicename}/sss_signal
 
+# The files provider is intentionally packaged in -common
+%{_libdir}/%{name}/libsss_files.so
+
 %dir %{sssdstatedir}
 %dir %{_localstatedir}/cache/krb5rcache
 %attr(700,sssd,sssd) %dir %{dbpath}
diff --git a/src/providers/files/files_id.c b/src/providers/files/files_id.c
new file mode 100644
index 000000000..41314c66b
--- /dev/null
+++ b/src/providers/files/files_id.c
@@ -0,0 +1,179 @@
+/*
+    SSSD
+
+    files_id.c - Identity operaions on the files provider
+
+    Copyright (C) 2016 Red Hat
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "providers/data_provider/dp.h"
+#include "providers/files/files_private.h"
+
+struct files_account_info_handler_state {
+    struct dp_reply_std reply;
+
+    struct files_id_ctx *id_ctx;
+};
+
+struct tevent_req *
+files_account_info_handler_send(TALLOC_CTX *mem_ctx,
+                                struct files_id_ctx *id_ctx,
+                                struct dp_id_data *data,
+                                struct dp_req_params *params)
+{
+    struct files_account_info_handler_state *state;
+    struct tevent_req *req;
+    struct tevent_req **update_req = NULL;
+    bool needs_update;
+    errno_t ret;
+
+    req = tevent_req_create(mem_ctx, &state,
+                            struct files_account_info_handler_state);
+    if (req == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, "tevent_req_create() failed\n");
+        return NULL;
+    }
+    state->id_ctx = id_ctx;
+
+    switch (data->entry_type & BE_REQ_TYPE_MASK) {
+    case BE_REQ_USER:
+        if (data->filter_type != BE_FILTER_ENUM) {
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Unexpected user filter type: %d\n", data->filter_type);
+            ret = EINVAL;
+            goto immediate;
+        }
+        update_req = &id_ctx->users_req;
+        needs_update = id_ctx->updating_passwd ? true : false;
+        break;
+    case BE_REQ_GROUP:
+        if (data->filter_type != BE_FILTER_ENUM) {
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Unexpected group filter type: %d\n", data->filter_type);
+            ret = EINVAL;
+            goto immediate;
+        }
+        update_req = &id_ctx->groups_req;
+        needs_update = id_ctx->updating_groups ? true : false;
+        break;
+    case BE_REQ_INITGROUPS:
+        if (data->filter_type != BE_FILTER_NAME) {
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Unexpected initgr filter type: %d\n", data->filter_type);
+            ret = EINVAL;
+            goto immediate;
+        }
+        if (strcmp(data->filter_value, DP_REQ_OPT_FILES_INITGR) != 0) {
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Unexpected initgr filter value: %d\n", data->filter_type);
+            ret = EINVAL;
+            goto immediate;
+        }
+        update_req = &id_ctx->initgroups_req;
+        needs_update = id_ctx->updating_groups || id_ctx->updating_passwd \
+                       ? true \
+                       : false;
+        break;
+    default:
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Unexpected entry type: %d\n", data->entry_type & BE_REQ_TYPE_MASK);
+        ret = EINVAL;
+        goto immediate;
+    }
+
+    if (needs_update == false) {
+        DEBUG(SSSDBG_TRACE_LIBS, "The files domain no longer needs an update\n");
+        ret = EOK;
+        goto immediate;
+    }
+
+    if (*update_req != NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, "BUG: Received a concurrent update!\n");
+        ret = EAGAIN;
+        goto immediate;
+    }
+
+    /* id_ctx now must mark the requests as updated when the inotify-induced
+     * update finishes
+     */
+    *update_req = req;
+    return req;
+
+immediate:
+    dp_reply_std_set(&state->reply, DP_ERR_DECIDE, ret, NULL);
+
+    if (ret == EOK) {
+        tevent_req_done(req);
+    } else {
+        tevent_req_error(req, ret);
+    }
+
+    tevent_req_post(req, params->ev);
+    return req;
+}
+
+static void finish_update_req(struct tevent_req **update_req,
+                              errno_t ret)
+{
+    if (*update_req == NULL) {
+        return;
+    }
+
+    if (ret != EOK) {
+        tevent_req_error(*update_req, ret);
+    } else {
+        tevent_req_done(*update_req);
+    }
+    *update_req = NULL;
+}
+
+void files_account_info_finished(struct files_id_ctx *id_ctx,
+                                 int req_type,
+                                 errno_t ret)
+{
+    switch (req_type) {
+    case BE_REQ_USER:
+        finish_update_req(&id_ctx->users_req, ret);
+        if (id_ctx->updating_groups == false) {
+            finish_update_req(&id_ctx->initgroups_req, ret);
+        }
+        break;
+    case BE_REQ_GROUP:
+        finish_update_req(&id_ctx->groups_req, ret);
+        if (id_ctx->updating_passwd == false) {
+            finish_update_req(&id_ctx->initgroups_req, ret);
+        }
+        break;
+    default:
+        DEBUG(SSSDBG_CRIT_FAILURE,
+               "Unexpected req_type %d\n", req_type);
+        return;
+    }
+}
+
+errno_t files_account_info_handler_recv(TALLOC_CTX *mem_ctx,
+                                        struct tevent_req *req,
+                                        struct dp_reply_std *data)
+{
+    struct files_account_info_handler_state *state = NULL;
+
+    state = tevent_req_data(req, struct files_account_info_handler_state);
+
+    TEVENT_REQ_RETURN_ON_ERROR(req);
+
+    *data = state->reply;
+    return EOK;
+}
diff --git a/src/providers/files/files_init.c b/src/providers/files/files_init.c
new file mode 100644
index 000000000..b91dfbac9
--- /dev/null
+++ b/src/providers/files/files_init.c
@@ -0,0 +1,92 @@
+/*
+    SSSD
+
+    files_init.c - Initialization of the files provider
+
+    Copyright (C) 2016 Red Hat
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "providers/data_provider/dp.h"
+#include "providers/files/files_private.h"
+
+int sssm_files_init(TALLOC_CTX *mem_ctx,
+                    struct be_ctx *be_ctx,
+                    struct data_provider *provider,
+                    const char *module_name,
+                    void **_module_data)
+{
+    struct files_id_ctx *ctx;
+    int ret;
+    const char *passwd_file = NULL;
+    const char *group_file = NULL;
+
+    /* So far this is mostly useful for tests */
+    passwd_file = getenv("SSS_FILES_PASSWD");
+    if (passwd_file == NULL) {
+        passwd_file = "/etc/passwd";
+    }
+
+    group_file = getenv("SSS_FILES_GROUP");
+    if (group_file == NULL) {
+        group_file = "/etc/group";
+    }
+
+    ctx = talloc_zero(mem_ctx, struct files_id_ctx);
+    if (ctx == NULL) {
+        return ENOMEM;
+    }
+    ctx->be = be_ctx;
+    ctx->domain = be_ctx->domain;
+    ctx->passwd_file = passwd_file;
+    ctx->group_file = group_file;
+
+    ctx->fctx = sf_init(ctx, be_ctx->ev,
+                        ctx->passwd_file, ctx->group_file,
+                        ctx);
+    if (ctx->fctx == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    *_module_data = ctx;
+    ret = EOK;
+done:
+    if (ret != EOK) {
+        talloc_free(ctx);
+    }
+    return ret;
+}
+
+int sssm_files_id_init(TALLOC_CTX *mem_ctx,
+                       struct be_ctx *be_ctx,
+                       void *module_data,
+                       struct dp_method *dp_methods)
+{
+    struct files_id_ctx *ctx;
+
+    ctx = talloc_get_type(module_data, struct files_id_ctx);
+    if (ctx == NULL) {
+        return EINVAL;
+    }
+
+    dp_set_method(dp_methods, DPM_ACCOUNT_HANDLER,
+                  files_account_info_handler_send,
+                  files_account_info_handler_recv,
+                  ctx, struct files_id_ctx,
+                  struct dp_id_data, struct dp_reply_std);
+
+    return EOK;
+}
diff --git a/src/providers/files/files_ops.c b/src/providers/files/files_ops.c
new file mode 100644
index 000000000..beda47abd
--- /dev/null
+++ b/src/providers/files/files_ops.c
@@ -0,0 +1,801 @@
+/*
+    SSSD
+
+    Files provider operations
+
+    Copyright (C) 2016 Red Hat
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include <dlfcn.h>
+
+#include "config.h"
+
+#include "providers/files/files_private.h"
+#include "db/sysdb.h"
+#include "util/inotify.h"
+#include "util/util.h"
+
+#define FILES_REALLOC_CHUNK 64
+
+#define PWD_MAXSIZE         1024
+#define GRP_MAXSIZE         2048
+
+struct files_ctx {
+    struct snotify_ctx *pwd_watch;
+    struct snotify_ctx *grp_watch;
+
+    struct files_ops_ctx *ops;
+};
+
+static errno_t enum_files_users(TALLOC_CTX *mem_ctx,
+                                struct files_id_ctx *id_ctx,
+                                struct passwd ***_users)
+{
+    errno_t ret, close_ret;
+    struct passwd *pwd_iter = NULL;
+    struct passwd *pwd = NULL;
+    struct passwd **users = NULL;
+    FILE *pwd_handle = NULL;
+    size_t n_users = 0;
+
+    pwd_handle = fopen(id_ctx->passwd_file, "r");
+    if (pwd_handle == NULL) {
+        ret = errno;
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Cannot open passwd file %s [%d]\n",
+              id_ctx->passwd_file, ret);
+        goto done;
+    }
+
+    users = talloc_zero_array(mem_ctx, struct passwd *,
+                              FILES_REALLOC_CHUNK);
+    if (users == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    while ((pwd_iter = fgetpwent(pwd_handle)) != NULL) {
+        /* FIXME - we might want to support paging of sorts to avoid allocating
+         * all users atop a memory context or only return users that differ from
+         * the local storage as a diff to minimize memory spikes
+         */
+        DEBUG(SSSDBG_TRACE_LIBS,
+              "User found (%s, %s, %"SPRIuid", %"SPRIgid", %s, %s, %s)\n",
+              pwd_iter->pw_name, pwd_iter->pw_passwd,
+              pwd_iter->pw_uid, pwd_iter->pw_gid,
+              pwd_iter->pw_gecos, pwd_iter->pw_dir,
+              pwd_iter->pw_shell);
+
+        pwd = talloc_zero(users, struct passwd);
+        if (pwd == NULL) {
+            ret = ENOMEM;
+            goto done;
+        }
+
+        pwd->pw_uid = pwd_iter->pw_uid;
+        pwd->pw_gid = pwd_iter->pw_gid;
+
+        pwd->pw_name = talloc_strdup(pwd, pwd_iter->pw_name);
+        if (pwd->pw_name == NULL) {
+            /* We only check pw_name here on purpose to allow broken
+             * records to be optionally rejected when saving them
+             * or fallback values to be used.
+             */
+            ret = ENOMEM;
+            goto done;
+        }
+
+        pwd->pw_dir = talloc_strdup(pwd, pwd_iter->pw_dir);
+        pwd->pw_gecos = talloc_strdup(pwd, pwd_iter->pw_gecos);
+        pwd->pw_shell = talloc_strdup(pwd, pwd_iter->pw_shell);
+        pwd->pw_passwd = talloc_strdup(pwd, pwd_iter->pw_passwd);
+
+        users[n_users] = pwd;
+        n_users++;
+        if (n_users % FILES_REALLOC_CHUNK == 0) {
+            users = talloc_realloc(mem_ctx,
+                                   users,
+                                   struct passwd *,
+                                   talloc_get_size(users) + FILES_REALLOC_CHUNK);
+            if (users == NULL) {
+                ret = ENOMEM;
+                goto done;
+            }
+        }
+    }
+
+    ret = EOK;
+    *_users = users;
+done:
+    if (ret != EOK) {
+        talloc_free(users);
+    }
+
+    if (pwd_handle) {
+        close_ret = fclose(pwd_handle);
+        if (close_ret != 0) {
+            close_ret = errno;
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Cannot close passwd file %s [%d]\n",
+                  id_ctx->passwd_file, close_ret);
+        }
+    }
+    return ret;
+}
+
+static errno_t enum_files_groups(TALLOC_CTX *mem_ctx,
+                                 struct files_id_ctx *id_ctx,
+                                 struct group ***_groups)
+{
+    errno_t ret, close_ret;
+    struct group *grp_iter = NULL;
+    struct group *grp = NULL;
+    struct group **groups = NULL;
+    size_t n_groups = 0;
+    FILE *grp_handle = NULL;
+
+    grp_handle = fopen(id_ctx->group_file, "r");
+    if (grp_handle == NULL) {
+        ret = errno;
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Cannot open group file %s [%d]\n",
+              id_ctx->group_file, ret);
+        goto done;
+    }
+
+    groups = talloc_zero_array(mem_ctx, struct group *,
+                               FILES_REALLOC_CHUNK);
+    if (groups == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    while ((grp_iter = fgetgrent(grp_handle)) != NULL) {
+        DEBUG(SSSDBG_TRACE_LIBS,
+              "Group found (%s, %"SPRIgid")\n",
+              grp_iter->gr_name, grp_iter->gr_gid);
+
+        grp = talloc_zero(groups, struct group);
+        if (grp == NULL) {
+            ret = ENOMEM;
+            goto done;
+        }
+
+        grp->gr_gid = grp_iter->gr_gid;
+        grp->gr_name = talloc_strdup(grp, grp_iter->gr_name);
+        if (grp->gr_name == NULL) {
+            /* We only check gr_name here on purpose to allow broken
+             * records to be optionally rejected when saving them
+             * or fallback values to be used.
+             */
+            ret = ENOMEM;
+            goto done;
+        }
+        grp->gr_passwd = talloc_strdup(grp, grp_iter->gr_passwd);
+
+        if (grp_iter->gr_mem != NULL && grp_iter->gr_mem[0] != '\0') {
+            size_t nmem;
+
+            for (nmem = 0; grp_iter->gr_mem[nmem] != NULL; nmem++);
+
+            grp->gr_mem = talloc_zero_array(grp, char *, nmem + 1);
+            if (grp->gr_mem == NULL) {
+                ret = ENOMEM;
+                goto done;
+            }
+
+            for (nmem = 0; grp_iter->gr_mem[nmem] != NULL; nmem++) {
+                grp->gr_mem[nmem] = talloc_strdup(grp, grp_iter->gr_mem[nmem]);
+                if (grp->gr_mem[nmem] == NULL) {
+                    ret = ENOMEM;
+                    goto done;
+                }
+            }
+        }
+
+        groups[n_groups] = grp;
+        n_groups++;
+        if (n_groups % FILES_REALLOC_CHUNK == 0) {
+            groups = talloc_realloc(mem_ctx,
+                                    groups,
+                                    struct group *,
+                                    talloc_get_size(groups) + FILES_REALLOC_CHUNK);
+            if (groups == NULL) {
+                ret = ENOMEM;
+                goto done;
+            }
+        }
+    }
+
+    ret = EOK;
+    *_groups = groups;
+done:
+    if (ret != EOK) {
+        talloc_free(groups);
+    }
+
+    if (grp_handle) {
+        close_ret = fclose(grp_handle);
+        if (close_ret != 0) {
+            close_ret = errno;
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Cannot close group file %s [%d]\n",
+                  id_ctx->group_file, close_ret);
+        }
+    }
+    return ret;
+}
+
+static errno_t delete_all_users(struct sss_domain_info *dom)
+{
+    TALLOC_CTX *tmp_ctx;
+    struct ldb_dn *base_dn;
+    errno_t ret;
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        DEBUG(SSSDBG_FATAL_FAILURE, "Out of memory!\n");
+        return ENOMEM;
+    }
+
+    base_dn = sysdb_user_base_dn(tmp_ctx, dom);
+    if (base_dn == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, "Out of memory!\n");
+        ret = ENOMEM;
+        goto done;
+    }
+
+    ret = sysdb_delete_recursive(dom->sysdb, base_dn, true);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_OP_FAILURE, "Unable to delete users subtree [%d]: %s\n",
+              ret, sss_strerror(ret));
+        goto done;
+    }
+
+    ret = EOK;
+
+done:
+    talloc_free(tmp_ctx);
+
+    return ret;
+}
+
+static errno_t save_file_user(struct files_id_ctx *id_ctx,
+                              struct passwd *pw)
+{
+    errno_t ret;
+    char *fqname;
+    TALLOC_CTX *tmp_ctx = NULL;
+    const char *shell;
+    const char *gecos;
+    struct sysdb_attrs *attrs = NULL;
+
+    if (strcmp(pw->pw_name, "root") == 0
+            || pw->pw_uid == 0
+            || pw->pw_gid == 0) {
+        DEBUG(SSSDBG_TRACE_FUNC, "Skipping %s\n", pw->pw_name);
+        return EOK;
+    }
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        return ENOMEM;
+    }
+
+    fqname = sss_create_internal_fqname(tmp_ctx, pw->pw_name,
+                                        id_ctx->domain->name);
+    if (fqname == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    attrs = sysdb_new_attrs(tmp_ctx);
+    if (attrs == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    if (pw->pw_shell && pw->pw_shell[0] != '\0') {
+        shell = pw->pw_shell;
+    } else {
+        shell = NULL;
+    }
+
+    if (pw->pw_gecos && pw->pw_gecos[0] != '\0') {
+        gecos = pw->pw_gecos;
+    } else {
+        gecos = NULL;
+    }
+
+    /* FIXME - optimize later */
+    ret = sysdb_store_user(id_ctx->domain,
+                           fqname,
+                           pw->pw_passwd,
+                           pw->pw_uid,
+                           pw->pw_gid,
+                           gecos,
+                           pw->pw_dir,
+                           shell,
+                           NULL, attrs,
+                           NULL, 0, 0);
+    if (ret != EOK) {
+        goto done;
+    }
+
+    ret = EOK;
+done:
+    talloc_free(tmp_ctx);
+    return ret;
+}
+
+static errno_t sf_enum_groups(struct files_id_ctx *id_ctx);
+
+errno_t sf_enum_users(struct files_id_ctx *id_ctx)
+{
+    errno_t ret;
+    errno_t tret;
+    TALLOC_CTX *tmp_ctx = NULL;
+    struct passwd **users = NULL;
+    bool in_transaction = false;
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        return ENOMEM;
+    }
+
+    ret = enum_files_users(tmp_ctx, id_ctx, &users);
+    if (ret != EOK) {
+        goto done;
+    }
+
+    ret = sysdb_transaction_start(id_ctx->domain->sysdb);
+    if (ret != EOK) {
+        goto done;
+    }
+    in_transaction = true;
+
+    /* remove previous cache contents */
+    /* FIXME - this is terribly inefficient */
+    ret = delete_all_users(id_ctx->domain);
+    if (ret != EOK) {
+        goto done;
+    }
+
+    for (size_t i = 0; users[i]; i++) {
+        ret = save_file_user(id_ctx, users[i]);
+        if (ret != EOK) {
+            DEBUG(SSSDBG_MINOR_FAILURE,
+                  "Cannot save user %s: [%d]: %s\n",
+                  users[i]->pw_name, ret, sss_strerror(ret));
+            continue;
+        }
+    }
+
+    ret = sysdb_transaction_commit(id_ctx->domain->sysdb);
+    if (ret != EOK) {
+        goto done;
+    }
+    in_transaction = false;
+
+    /* Covers the case when someone edits /etc/group, adds a group member and
+     * only then edits passwd and adds the user. The reverse is not needed,
+     * because member/memberof links are established when groups are saved.
+     */
+    ret = sf_enum_groups(id_ctx);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_OP_FAILURE, "Cannot refresh groups\n");
+        goto done;
+    }
+
+    ret = EOK;
+done:
+    if (in_transaction) {
+        tret = sysdb_transaction_cancel(id_ctx->domain->sysdb);
+        if (tret != EOK) {
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Cannot cancel transaction: %d\n", ret);
+        }
+    }
+    talloc_free(tmp_ctx);
+    return ret;
+}
+
+static const char **get_cached_user_names(TALLOC_CTX *mem_ctx,
+                                          struct sss_domain_info *dom)
+{
+    errno_t ret;
+    struct ldb_result *res = NULL;
+    const char **user_names = NULL;
+    unsigned c = 0;
+
+    ret = sysdb_enumpwent(mem_ctx, dom, &res);
+    if (ret != EOK) {
+        goto done;
+    }
+
+    user_names = talloc_zero_array(mem_ctx, const char *, res->count + 1);
+    if (user_names == NULL) {
+        goto done;
+    }
+
+    for (unsigned i = 0; i < res->count; i++) {
+        user_names[c] = ldb_msg_find_attr_as_string(res->msgs[i],
+                                                    SYSDB_NAME,
+                                                    NULL);
+        if (user_names[c] == NULL) {
+            continue;
+        }
+        c++;
+    }
+
+done:
+    /* Don't free res and keep it around to avoid duplicating the names */
+    return user_names;
+}
+
+static errno_t delete_all_groups(struct sss_domain_info *dom)
+{
+    TALLOC_CTX *tmp_ctx;
+    struct ldb_dn *base_dn;
+    errno_t ret;
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        DEBUG(SSSDBG_FATAL_FAILURE, "Out of memory!\n");
+        return ENOMEM;
+    }
+
+    base_dn = sysdb_group_base_dn(tmp_ctx, dom);
+    if (base_dn == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, "Out of memory!\n");
+        ret = ENOMEM;
+        goto done;
+    }
+
+    ret = sysdb_delete_recursive(dom->sysdb, base_dn, true);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_OP_FAILURE, "Unable to delete groups subtree [%d]: %s\n",
+              ret, sss_strerror(ret));
+        goto done;
+    }
+
+    ret = EOK;
+
+done:
+    talloc_free(tmp_ctx);
+
+    return ret;
+}
+
+static errno_t save_file_group(struct files_id_ctx *id_ctx,
+                               struct group *grp,
+                               const char **cached_users)
+{
+    errno_t ret;
+    char *fqname;
+    struct sysdb_attrs *attrs = NULL;
+    TALLOC_CTX *tmp_ctx = NULL;
+    char **fq_gr_files_mem;
+    const char **fq_gr_mem;
+    unsigned mi = 0;
+
+    if (strcmp(grp->gr_name, "root") == 0
+            || grp->gr_gid == 0) {
+        DEBUG(SSSDBG_TRACE_FUNC, "Skipping %s\n", grp->gr_name);
+        return EOK;
+    }
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        return ENOMEM;
+    }
+
+    fqname = sss_create_internal_fqname(tmp_ctx, grp->gr_name,
+                                        id_ctx->domain->name);
+    if (fqname == NULL) {
+        ret = ENOMEM;
+        goto done;
+
+    }
+
+    attrs = sysdb_new_attrs(tmp_ctx);
+    if (attrs == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    if (grp->gr_mem && grp->gr_mem[0]) {
+        fq_gr_files_mem = sss_create_internal_fqname_list(
+                                            tmp_ctx,
+                                            (const char *const*) grp->gr_mem,
+                                            id_ctx->domain->name);
+        if (fq_gr_files_mem == NULL) {
+            ret = ENOMEM;
+            goto done;
+        }
+
+        fq_gr_mem = talloc_zero_array(tmp_ctx, const char *,
+                                      talloc_array_length(fq_gr_files_mem));
+        if (fq_gr_mem == NULL) {
+            ret = ENOMEM;
+            goto done;
+        }
+
+        for (unsigned i=0; fq_gr_files_mem[i] != NULL; i++) {
+            if (string_in_list(fq_gr_files_mem[i],
+                               discard_const(cached_users),
+                               true)) {
+                fq_gr_mem[mi] = fq_gr_files_mem[i];
+                mi++;
+
+                DEBUG(SSSDBG_TRACE_LIBS,
+                      "User %s is cached, will become a member of %s\n",
+                      fq_gr_files_mem[i], grp->gr_name);
+            } else {
+                ret = sysdb_attrs_add_string(attrs,
+                                             SYSDB_GHOST,
+                                             fq_gr_files_mem[i]);
+                if (ret != EOK) {
+                    DEBUG(SSSDBG_MINOR_FAILURE,
+                          "Cannot add ghost %s for group %s\n",
+                          fq_gr_files_mem[i], fqname);
+                    continue;
+                }
+
+                DEBUG(SSSDBG_TRACE_LIBS,
+                      "User %s is not cached, will become a ghost of %s\n",
+                      fq_gr_files_mem[i], grp->gr_name);
+            }
+        }
+
+        if (fq_gr_mem != NULL && fq_gr_mem[0] != NULL) {
+            ret = sysdb_attrs_users_from_str_list(
+                    attrs, SYSDB_MEMBER, id_ctx->domain->name,
+                    (const char *const *) fq_gr_mem);
+            if (ret) {
+                DEBUG(SSSDBG_OP_FAILURE, "Could not add group members\n");
+                goto done;
+            }
+        }
+
+    }
+
+    ret = sysdb_store_group(id_ctx->domain, fqname, grp->gr_gid,
+                            attrs, 0, 0);
+    if (ret) {
+        DEBUG(SSSDBG_OP_FAILURE, "Could not add group to cache\n");
+        goto done;
+    }
+
+    ret = EOK;
+done:
+    talloc_free(tmp_ctx);
+    return ret;
+}
+
+static errno_t sf_enum_groups(struct files_id_ctx *id_ctx)
+{
+    errno_t ret;
+    errno_t tret;
+    TALLOC_CTX *tmp_ctx = NULL;
+    struct group **groups = NULL;
+    bool in_transaction = false;
+    const char **cached_users = NULL;
+
+    tmp_ctx = talloc_new(NULL);
+    if (tmp_ctx == NULL) {
+        return ENOMEM;
+    }
+
+    ret = enum_files_groups(tmp_ctx, id_ctx, &groups);
+    if (ret != EOK) {
+        goto done;
+    }
+
+    cached_users = get_cached_user_names(tmp_ctx, id_ctx->domain);
+    if (cached_users == NULL) {
+        goto done;
+    }
+
+    ret = sysdb_transaction_start(id_ctx->domain->sysdb);
+    if (ret != EOK) {
+        goto done;
+    }
+    in_transaction = true;
+
+    /* remove previous cache contents */
+    ret = delete_all_groups(id_ctx->domain);
+    if (ret != EOK) {
+        goto done;
+    }
+
+    for (size_t i = 0; groups[i]; i++) {
+        ret = save_file_group(id_ctx, groups[i], cached_users);
+        if (ret != EOK) {
+            DEBUG(SSSDBG_MINOR_FAILURE,
+                  "Cannot save group %s\n", groups[i]->gr_name);
+            continue;
+        }
+    }
+
+    ret = sysdb_transaction_commit(id_ctx->domain->sysdb);
+    if (ret != EOK) {
+        goto done;
+    }
+    in_transaction = false;
+
+    ret = EOK;
+done:
+    if (in_transaction) {
+        tret = sysdb_transaction_cancel(id_ctx->domain->sysdb);
+        if (tret != EOK) {
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  "Cannot cancel transaction: %d\n", ret);
+        }
+    }
+    talloc_free(tmp_ctx);
+    return ret;
+}
+
+static void sf_cb_done(struct files_id_ctx *id_ctx)
+{
+    /* Only activate a domain when both callbacks are done */
+    if (id_ctx->updating_passwd == false
+            && id_ctx->updating_groups == false) {
+        dp_sbus_domain_active(id_ctx->be->provider,
+                              id_ctx->domain);
+    }
+}
+
+static int sf_passwd_cb(const char *filename, uint32_t flags, void *pvt)
+{
+    struct files_id_ctx *id_ctx;
+    errno_t ret;
+
+    id_ctx = talloc_get_type(pvt, struct files_id_ctx);
+    if (id_ctx == NULL) {
+        return EINVAL;
+    }
+
+    DEBUG(SSSDBG_TRACE_FUNC, "passwd notification\n");
+
+    if (strcmp(filename, id_ctx->passwd_file) != 0) {
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Wrong file, expected %s, got %s\n",
+              id_ctx->passwd_file, filename);
+        return EINVAL;
+    }
+
+    id_ctx->updating_passwd = true;
+    dp_sbus_domain_inconsistent(id_ctx->be->provider, id_ctx->domain);
+
+    dp_sbus_reset_users_ncache(id_ctx->be->provider, id_ctx->domain);
+    dp_sbus_reset_users_memcache(id_ctx->be->provider);
+    dp_sbus_reset_initgr_memcache(id_ctx->be->provider);
+
+    ret = sf_enum_users(id_ctx);
+
+    id_ctx->updating_passwd = false;
+    sf_cb_done(id_ctx);
+    files_account_info_finished(id_ctx, BE_REQ_USER, ret);
+    return ret;
+}
+
+static int sf_group_cb(const char *filename, uint32_t flags, void *pvt)
+{
+    struct files_id_ctx *id_ctx;
+    errno_t ret;
+
+    id_ctx = talloc_get_type(pvt, struct files_id_ctx);
+    if (id_ctx == NULL) {
+        return EINVAL;
+    }
+
+    DEBUG(SSSDBG_TRACE_FUNC, "group notification\n");
+
+    if (strcmp(filename, id_ctx->group_file) != 0) {
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Wrong file, expected %s, got %s\n",
+              id_ctx->group_file, filename);
+        return EINVAL;
+    }
+
+    id_ctx->updating_groups = true;
+    dp_sbus_domain_inconsistent(id_ctx->be->provider, id_ctx->domain);
+
+    dp_sbus_reset_groups_ncache(id_ctx->be->provider, id_ctx->domain);
+    dp_sbus_reset_groups_memcache(id_ctx->be->provider);
+    dp_sbus_reset_initgr_memcache(id_ctx->be->provider);
+
+    ret = sf_enum_groups(id_ctx);
+
+    id_ctx->updating_groups = false;
+    sf_cb_done(id_ctx);
+    files_account_info_finished(id_ctx, BE_REQ_GROUP, ret);
+    return ret;
+}
+
+static void startup_enum_files(struct tevent_context *ev,
+                               struct tevent_immediate *imm,
+                               void *pvt)
+{
+    struct files_id_ctx *id_ctx = talloc_get_type(pvt, struct files_id_ctx);
+    errno_t ret;
+
+    talloc_zfree(imm);
+
+    ret = sf_enum_users(id_ctx);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Enumerating users failed, data might be inconsistent!\n");
+    }
+
+    ret = sf_enum_groups(id_ctx);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_CRIT_FAILURE,
+              "Enumerating groups failed, data might be inconsistent!\n");
+    }
+}
+
+static struct snotify_ctx *sf_setup_watch(TALLOC_CTX *mem_ctx,
+                                          struct tevent_context *ev,
+                                          const char *filename,
+                                          snotify_cb_fn fn,
+                                          struct files_id_ctx *id_ctx)
+{
+    return snotify_create(mem_ctx, ev, SNOTIFY_WATCH_DIR,
+                          filename, NULL,
+                          IN_DELETE_SELF | IN_CLOSE_WRITE | IN_MOVE_SELF | \
+                          IN_CREATE | IN_MOVED_TO,
+                          fn, id_ctx);
+}
+
+struct files_ctx *sf_init(TALLOC_CTX *mem_ctx,
+                          struct tevent_context *ev,
+                          const char *passwd_file,
+                          const char *group_file,
+                          struct files_id_ctx *id_ctx)
+{
+    struct files_ctx *fctx;
+    struct tevent_immediate *imm;
+
+    fctx = talloc(mem_ctx, struct files_ctx);
+    if (fctx == NULL) {
+        return NULL;
+    }
+
+    fctx->pwd_watch = sf_setup_watch(fctx, ev, passwd_file,
+                                     sf_passwd_cb, id_ctx);
+    fctx->grp_watch = sf_setup_watch(fctx, ev, group_file,
+                                     sf_group_cb, id_ctx);
+    if (fctx->pwd_watch == NULL || fctx->grp_watch == NULL) {
+        talloc_free(fctx);
+        return NULL;
+    }
+
+    /* Enumerate users and groups on startup to process any changes when
+     * sssd was down. We schedule a request here to minimize the time
+     * we spend in the init function
+     */
+    imm = tevent_create_immediate(id_ctx);
+    if (imm == NULL) {
+        DEBUG(SSSDBG_OP_FAILURE, "tevent_create_immediate failed.\n");
+        talloc_free(fctx);
+        return NULL;
+    }
+    tevent_schedule_immediate(imm, ev, startup_enum_files, id_ctx);
+
+    return fctx;
+}
diff --git a/src/providers/files/files_private.h b/src/providers/files/files_private.h
new file mode 100644
index 000000000..a7d195c90
--- /dev/null
+++ b/src/providers/files/files_private.h
@@ -0,0 +1,74 @@
+/*
+    SSSD
+
+    Files provider declarations
+
+    Copyright (C) 2016 Red Hat
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __FILES_PRIVATE_H_
+#define __FILES_PRIVATE_H_
+
+#include "config.h"
+
+#include <talloc.h>
+#include <tevent.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <nss.h>
+#include <pwd.h>
+#include <grp.h>
+
+#include "providers/data_provider/dp.h"
+
+struct files_id_ctx {
+    struct be_ctx *be;
+    struct sss_domain_info *domain;
+    struct files_ctx *fctx;
+
+    const char *passwd_file;
+    const char *group_file;
+
+    bool updating_passwd;
+    bool updating_groups;
+
+    struct tevent_req *users_req;
+    struct tevent_req *groups_req;
+    struct tevent_req *initgroups_req;
+};
+
+/* files_ops.c */
+struct files_ctx *sf_init(TALLOC_CTX *mem_ctx,
+                          struct tevent_context *ev,
+                          const char *passwd_file,
+                          const char *group_file,
+                          struct files_id_ctx *id_ctx);
+
+/* files_id.c */
+struct tevent_req *
+files_account_info_handler_send(TALLOC_CTX *mem_ctx,
+                               struct files_id_ctx *id_ctx,
+                               struct dp_id_data *data,
+                               struct dp_req_params *params);
+
+errno_t files_account_info_handler_recv(TALLOC_CTX *mem_ctx,
+                                        struct tevent_req *req,
+                                        struct dp_reply_std *data);
+
+void files_account_info_finished(struct files_id_ctx *id_ctx,
+                                 int req_type,
+                                 errno_t ret);
+#endif /* __FILES_PRIVATE_H_ */
diff --git a/src/tests/dlopen-tests.c b/src/tests/dlopen-tests.c
index d11ae9651..419857cc7 100644
--- a/src/tests/dlopen-tests.c
+++ b/src/tests/dlopen-tests.c
@@ -80,6 +80,8 @@ struct so {
     { "libsss_util.so", { LIBPFX"libsss_util.so", NULL } },
     { "libsss_simple.so", { LIBPFX"libdlopen_test_providers.so",
                             LIBPFX"libsss_simple.so", NULL } },
+    { "libsss_files.so", { LIBPFX"libdlopen_test_providers.so",
+                           LIBPFX"libsss_files.so", NULL } },
 #ifdef BUILD_SAMBA
     { "libsss_ad.so", { LIBPFX"libdlopen_test_providers.so",
                         LIBPFX"libsss_ad.so", NULL } },