Add handling for indirect memberof other entries.

This creates a new custom attribute, memberofindirect_[plugin].
Using this you can tell the difference between being an actual memberof
another entry and being a memberof as the result if inheritence. This is
particularly useful when trying to remove members of an entry, you can
only remove direct members.

I had to add a couple of short sleep calls to make things work a little
better. The memberof plugin runs as a postop and we have no way of knowing
when it has done its work. If we don't pause we may show some stale
data that memberof hasn't updated yet. .3 seconds is an arbitrary choice.

ticket 966

10 files changed, 265 insertions, 10 deletions

diff --git a/ipalib/plugins/baseldap.py b/ipalib/plugins/baseldap.py
index 2e284274b..3cb72d7b0 100644
--- a/ipalib/plugins/baseldap.py
+++ b/ipalib/plugins/baseldap.py
@@ -50,6 +50,9 @@ global_output_params = (
     Str('member_host?',
         label=_('Member hosts'),
     ),
+    Str('member_hostgroup?',
+        label=_('Member host-groups'),
+    ),
     Str('memberof_hostgroup?',
         label=_('Member of host-groups'),
     ),
@@ -128,6 +131,18 @@ global_output_params = (
     Str('memberindirect_sudocmd?',
         label='Indirect Member SUDO commands',
     ),
+    Str('memberofindirect_group?',
+        label='Indirect Member of group',
+    ),
+    Str('memberofindirect_netgroup?',
+        label='Indirect Member of netgroup',
+    ),
+    Str('memberofindirect_hostgroup?',
+        label='Indirect Member of host-group',
+    ),
+    Str('memberofindirect_role?',
+        label='Indirect Member of role',
+    ),
     Str('externalhost?',
         label=_('External host'),
     ),
@@ -1184,6 +1199,9 @@ class LDAPRemoveMember(LDAPModMember):
                 set(self.obj.default_attributes + member_dns.keys())
             )
 
+        # Give memberOf a chance to update entries
+        time.sleep(.3)
+
         try:
             (dn, entry_attrs) = ldap.get_entry(
                 dn, attrs_list, normalize=self.obj.normalize_dn
diff --git a/ipalib/plugins/group.py b/ipalib/plugins/group.py
index b981731e3..1c0161a9d 100644
--- a/ipalib/plugins/group.py
+++ b/ipalib/plugins/group.py
@@ -85,13 +85,14 @@ class group(LDAPObject):
     search_attributes_config = 'ipagroupsearchfields'
     default_attributes = [
         'cn', 'description', 'gidnumber', 'member', 'memberof',
-        'memberindirect',
+        'memberindirect', 'memberofindirect',
     ]
     uuid_attribute = 'ipauniqueid'
     attribute_members = {
         'member': ['user', 'group'],
         'memberof': ['group', 'netgroup', 'role',],
         'memberindirect': ['user', 'group', 'netgroup', 'role'],
+        'memberofindirect': ['group', 'netgroup', 'role'],
     }
     rdnattr = 'cn'
 
diff --git a/ipalib/plugins/host.py b/ipalib/plugins/host.py
index b819688d0..f5f5157b0 100644
--- a/ipalib/plugins/host.py
+++ b/ipalib/plugins/host.py
@@ -189,13 +189,14 @@ class host(LDAPObject):
     default_attributes = [
         'fqdn', 'description', 'l', 'nshostlocation', 'krbprincipalname',
         'nshardwareplatform', 'nsosversion', 'usercertificate', 'memberof',
-        'krblastpwdchange', 'managedby'
+        'krblastpwdchange', 'managedby', 'memberindirect', 'memberofindirect',
     ]
     uuid_attribute = 'ipauniqueid'
     attribute_members = {
         'enrolledby': ['user'],
         'memberof': ['hostgroup', 'netgroup', 'role'],
         'managedby': ['host'],
+        'memberofindirect': ['hostgroup', 'netgroup', 'role'],
     }
     bindable = True
     relationships = {
diff --git a/ipalib/plugins/hostgroup.py b/ipalib/plugins/hostgroup.py
index 082e4ef00..f661a2ff5 100644
--- a/ipalib/plugins/hostgroup.py
+++ b/ipalib/plugins/hostgroup.py
@@ -60,13 +60,14 @@ class hostgroup(LDAPObject):
     object_name_plural = 'hostgroups'
     object_class = ['ipaobject', 'ipahostgroup']
     default_attributes = ['cn', 'description', 'member', 'memberof',
-        'memberindirect'
+        'memberindirect', 'memberofindirect',
     ]
     uuid_attribute = 'ipauniqueid'
     attribute_members = {
         'member': ['host', 'hostgroup'],
         'memberof': ['hostgroup'],
         'memberindirect': ['host', 'hostgroup'],
+        'memberofindirect': ['host', 'hostgroup'],
     }
 
     label = _('Host Groups')
diff --git a/ipalib/plugins/privilege.py b/ipalib/plugins/privilege.py
index dfc4085ae..0b451635e 100644
--- a/ipalib/plugins/privilege.py
+++ b/ipalib/plugins/privilege.py
@@ -41,11 +41,12 @@ class privilege(LDAPObject):
     object_name_plural = 'privileges'
     object_class = ['nestedgroup', 'groupofnames']
     default_attributes = ['cn', 'description', 'member', 'memberof',
-        'memberindirect'
+        'memberindirect', 'memberofindirect',
     ]
     attribute_members = {
         'member': ['role'],
         'memberof': ['permission'],
+        'memberofindirect': ['permission'],
     }
     reverse_members = {
         'member': ['permission'],
diff --git a/ipalib/plugins/role.py b/ipalib/plugins/role.py
index fd79845ab..3324dba8c 100644
--- a/ipalib/plugins/role.py
+++ b/ipalib/plugins/role.py
@@ -67,12 +67,12 @@ class role(LDAPObject):
     object_name_plural = 'roles'
     object_class = ['groupofnames', 'nestedgroup']
     default_attributes = ['cn', 'description', 'member', 'memberof',
-        'memberindirect'
+        'memberindirect', 'memberofindirect',
     ]
     attribute_members = {
         'member': ['user', 'group', 'host', 'hostgroup'],
-        'memberof': ['privilege'],
-#        'memberindirect': ['user', 'group', 'host', 'hostgroup'],
+        'memberof': ['privilege', 'role'],
+        'memberofindirect': ['role'],
     }
     reverse_members = {
         'member': ['privilege'],
diff --git a/ipalib/plugins/user.py b/ipalib/plugins/user.py
index 0ea3c231f..ae730125d 100644
--- a/ipalib/plugins/user.py
+++ b/ipalib/plugins/user.py
@@ -84,10 +84,12 @@ class user(LDAPObject):
     default_attributes = [
         'uid', 'givenname', 'sn', 'homedirectory', 'loginshell', 'ou',
         'telephonenumber', 'title', 'memberof', 'nsaccountlock',
+        'memberofindirect',
     ]
     uuid_attribute = 'ipauniqueid'
     attribute_members = {
         'memberof': ['group', 'netgroup', 'role'],
+        'memberofindirect': ['group', 'netgroup', 'role'],
     }
     rdnattr = 'uid'
     bindable = True
diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py
index 00d3a4be4..d1e31f5e6 100644
--- a/ipaserver/plugins/ldap2.py
+++ b/ipaserver/plugins/ldap2.py
@@ -32,6 +32,7 @@ import socket
 import string
 import shutil
 import tempfile
+import time
 
 import krbV
 import ldap as _ldap
@@ -583,6 +584,21 @@ class ldap2(CrudBackend, Encoder):
                     time_limit=time_limit, size_limit=size_limit, normalize=normalize)
                 if len(indirect) > 0:
                     r[1]['memberindirect'] = indirect
+        if attrs_list and ('memberofindirect' in attrs_list or '*' in attrs_list):
+            for r in res:
+                if 'memberof' in r[1]:
+                    memberof = r[1]['memberof']
+                    del r[1]['memberof']
+                elif 'memberOf' in r[1]:
+                    memberof = r[1]['memberOf']
+                    del r[1]['memberOf']
+                else:
+                    continue
+                (direct, indirect) = self.get_memberof(r[0], memberof, time_limit=time_limit, size_limit=size_limit, normalize=normalize)
+                if len(direct) > 0:
+                    r[1]['memberof'] = direct
+                if len(indirect) > 0:
+                    r[1]['memberofindirect'] = indirect
 
         return (res, truncated)
 
@@ -745,6 +761,7 @@ class ldap2(CrudBackend, Encoder):
             raise errors.EmptyModlist()
         try:
             self.conn.rename_s(dn, new_rdn, delold=int(del_old))
+            time.sleep(.3) # Give memberOf plugin a chance to work
         except _ldap.LDAPError, e:
             _handle_errors(e, **{})
 
@@ -942,6 +959,60 @@ class ldap2(CrudBackend, Encoder):
 
         return entries
 
+    def get_memberof(self, entry_dn, memberof, time_limit=None, size_limit=None, normalize=True):
+        """
+        Examine the objects that an entry is a member of and determine if they
+        are a direct or indirect member of that group.
+
+        entry_dn: dn of the entry we want the direct/indirect members of
+        memberof: the memberOf attribute for entry_dn
+
+        Returns two memberof lists: (direct, indirect)
+        """
+
+        if not type(memberof) in (list, tuple):
+            return ([], [])
+        if len(memberof) == 0:
+            return ([], [])
+
+        attr_list = ["dn", "memberof"]
+        searchfilter = "(|(member=%s)(memberhost=%s)(memberuser=%s))" % (
+            entry_dn, entry_dn, entry_dn)
+
+        # We have to do three searches because netgroups and pbac are not
+        # within the accounts container.
+        try:
+            (results, truncated) = self.find_entries(searchfilter, attr_list,
+                api.env.container_accounts, time_limit=time_limit,
+                size_limit=size_limit, normalize=normalize)
+        except errors.NotFound:
+            results = []
+        try:
+            (netresults, truncated) = self.find_entries(searchfilter, attr_list,
+                api.env.container_netgroup, time_limit=time_limit,
+                size_limit=size_limit, normalize=normalize)
+        except errors.NotFound:
+            netresults = []
+        results = results + netresults
+        try:
+            (pbacresults, truncated) = self.find_entries(searchfilter,
+                attr_list, 'cn=pbac,%s' % api.env.basedn,
+                time_limit=time_limit, size_limit=size_limit,
+                normalize=normalize)
+        except errors.NotFound:
+            pbacresults = []
+        results = results + pbacresults
+
+        direct = []
+        indirect = []
+        for m in memberof:
+            indirect.append(m.lower())
+        for r in results:
+            direct.append(r[0])
+            indirect.remove(r[0].lower())
+
+        return (direct, indirect)
+
     def set_entry_active(self, dn, active):
         """Mark entry active/inactive."""
         assert isinstance(active, bool)
diff --git a/tests/test_xmlrpc/objectclasses.py b/tests/test_xmlrpc/objectclasses.py
index 0d03b47e3..41350f0bc 100644
--- a/tests/test_xmlrpc/objectclasses.py
+++ b/tests/test_xmlrpc/objectclasses.py
@@ -58,6 +58,7 @@ hostgroup = [
     u'nestedGroup',
     u'groupOfNames',
     u'top',
+    u'mepOriginEntry',
 ]
 
 role = [
diff --git a/tests/test_xmlrpc/test_nesting.py b/tests/test_xmlrpc/test_nesting.py
index 3e692b71c..9ccb136a8 100644
--- a/tests/test_xmlrpc/test_nesting.py
+++ b/tests/test_xmlrpc/test_nesting.py
@@ -30,6 +30,14 @@ group3 = u'testgroup3'
 user1 = u'tuser1'
 user2 = u'tuser2'
 
+hostgroup1 = u'testhostgroup1'
+hgdn1 = u'cn=%s,cn=hostgroups,cn=accounts,%s' % (hostgroup1, api.env.basedn)
+hostgroup2 = u'testhostgroup2'
+hgdn2 = u'cn=%s,cn=hostgroups,cn=accounts,%s' % (hostgroup2, api.env.basedn)
+
+fqdn1 = u'testhost1.%s' % api.env.domain
+host_dn1 = u'fqdn=%s,cn=computers,cn=accounts,%s' % (fqdn1, api.env.basedn)
+
 
 class test_group(Declarative):
     cleanup_commands = [
@@ -38,6 +46,9 @@ class test_group(Declarative):
         ('group_del', [group3], {}),
         ('user_del', [user1], {}),
         ('user_del', [user2], {}),
+        ('host_del', [fqdn1], {}),
+        ('hostgroup_del', [hostgroup1], {}),
+        ('hostgroup_del', [hostgroup2], {}),
     ]
 
     tests = [
@@ -287,7 +298,8 @@ class test_group(Declarative):
                 result={
                         'dn': u'cn=%s,cn=groups,cn=accounts,%s' % (group3, api.env.basedn),
                         'member_user': (u'tuser2',),
-                        'memberof_group': (u'testgroup2', u'testgroup1'),
+                        'memberof_group': [u'testgroup2'],
+                        'memberofindirect_group': [u'testgroup1'],
                         'gidnumber': [fuzzy_digits],
                         'cn': [group3],
                         'description': [u'Test desc 3'],
@@ -306,7 +318,7 @@ class test_group(Declarative):
                     cn=[group1],
                     description=[u'Test desc 1'],
                     gidnumber= [fuzzy_digits],
-                    memberindirect_group = (u'testgroup3',),
+                    memberindirect_group = [u'testgroup3'],
                     member_group = (u'testgroup2',),
                     memberindirect_user = (u'tuser1',u'tuser2',),
                     dn=u'cn=testgroup1,cn=groups,cn=accounts,' + api.env.basedn,
@@ -345,12 +357,159 @@ class test_group(Declarative):
                     cn=[group3],
                     description=[u'Test desc 3'],
                     gidnumber= [fuzzy_digits],
-                    memberof_group = (u'testgroup2', u'testgroup1',),
+                    memberof_group = (u'testgroup2',),
                     member_user = (u'tuser2',),
+                    memberofindirect_group = (u'testgroup1',),
                     dn=u'cn=testgroup3,cn=groups,cn=accounts,' + api.env.basedn,
                 ),
             ),
         ),
 
+        # Now do something similar with hosts and hostgroups
+        dict(
+            desc='Create host %r' % fqdn1,
+            command=('host_add', [fqdn1],
+                dict(
+                    description=u'Test host 1',
+                    l=u'Undisclosed location 1',
+                    force=True,
+                ),
+            ),
+            expected=dict(
+                value=fqdn1,
+                summary=u'Added host "%s"' % fqdn1,
+                result=dict(
+                    dn=host_dn1,
+                    fqdn=[fqdn1],
+                    description=[u'Test host 1'],
+                    l=[u'Undisclosed location 1'],
+                    krbprincipalname=[u'host/%s@%s' % (fqdn1, api.env.realm)],
+                    objectclass=objectclasses.host,
+                    ipauniqueid=[fuzzy_uuid],
+                    managedby_host=[fqdn1],
+                ),
+            ),
+        ),
+
+        dict(
+            desc='Create %r' % hostgroup1,
+            command=('hostgroup_add', [hostgroup1],
+                dict(description=u'Test hostgroup 1')
+            ),
+            expected=dict(
+                value=hostgroup1,
+                summary=u'Added hostgroup "testhostgroup1"',
+                result=dict(
+                    dn=hgdn1,
+                    cn=[hostgroup1],
+                    objectclass=objectclasses.hostgroup,
+                    description=[u'Test hostgroup 1'],
+                    ipauniqueid=[fuzzy_uuid],
+                ),
+            ),
+        ),
+
+
+        dict(
+            desc='Create %r' % hostgroup2,
+            command=('hostgroup_add', [hostgroup2],
+                dict(description=u'Test hostgroup 2')
+            ),
+            expected=dict(
+                value=hostgroup2,
+                summary=u'Added hostgroup "testhostgroup2"',
+                result=dict(
+                    dn=hgdn2,
+                    cn=[hostgroup2],
+                    objectclass=objectclasses.hostgroup,
+                    description=[u'Test hostgroup 2'],
+                    ipauniqueid=[fuzzy_uuid],
+                ),
+            ),
+        ),
+
+
+        dict(
+            desc=u'Add host %r to %r' % (fqdn1, hostgroup2),
+            command=(
+                'hostgroup_add_member', [hostgroup2], dict(host=fqdn1)
+            ),
+            expected=dict(
+                completed=1,
+                failed=dict(
+                    member=dict(
+                        host=tuple(),
+                        hostgroup=tuple(),
+                    ),
+                ),
+                result={
+                    'dn': hgdn2,
+                    'cn': [hostgroup2],
+                    'description': [u'Test hostgroup 2'],
+                    'member_host': [fqdn1],
+                },
+            ),
+        ),
+
+
+        dict(
+            desc=u'Add hostgroup %r to %r' % (hostgroup2, hostgroup1),
+            command=(
+                'hostgroup_add_member', [hostgroup1], dict(hostgroup=hostgroup2)
+            ),
+            expected=dict(
+                completed=1,
+                failed=dict(
+                    member=dict(
+                        host=tuple(),
+                        hostgroup=tuple(),
+                    ),
+                ),
+                result={
+                    'dn': hgdn1,
+                    'cn': [hostgroup1],
+                    'description': [u'Test hostgroup 1'],
+                    'member_hostgroup': [hostgroup2],
+                },
+            ),
+        ),
+
+
+        dict(
+            desc='Retrieve %r' % hostgroup1,
+            command=('hostgroup_show', [hostgroup1], {}),
+            expected=dict(
+                value=hostgroup1,
+                summary=None,
+                result={
+                    'dn': hgdn1,
+                    'memberindirect_host': [u'testhost1.%s' % api.env.domain],
+                    'member_hostgroup': [hostgroup2],
+                    'cn': [hostgroup1],
+                    'description': [u'Test hostgroup 1'],
+                },
+            ),
+        ),
+
+        dict(
+            desc='Retrieve %r' % fqdn1,
+            command=('host_show', [fqdn1], {}),
+            expected=dict(
+                value=fqdn1,
+                summary=None,
+                result=dict(
+                    dn=host_dn1,
+                    fqdn=[fqdn1],
+                    description=[u'Test host 1'],
+                    l=[u'Undisclosed location 1'],
+                    krbprincipalname=[u'host/%s@%s' % (fqdn1, api.env.realm)],
+                    has_keytab=False,
+                    managedby_host=[fqdn1],
+                    memberof_hostgroup = [u'testhostgroup2'],
+                    memberofindirect_hostgroup = [u'testhostgroup1'],
+                    memberofindirect_netgroup = [u'testhostgroup1', u'testhostgroup2'],
+                ),
+            ),
+        ),
 
     ]