Move Managed Entries into their own container in the replicated space.

Repoint cn=Managed Entries,cn=plugins,cn=config in common_setup
Create: cn=Managed Entries,cn=etc,$SUFFIX
Create: cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
Create: cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
Create method for dynamically migrating any and all custom Managed Entries
from the cn=config space into the new container.
Separate the connection creation during update so that a restart can
be performed to initialize changes before performing a delete.
Add wait_for_open_socket() method in installutils

https://fedorahosted.org/freeipa/ticket/1708

13 files changed, 238 insertions, 47 deletions

diff --git a/install/share/Makefile.am b/install/share/Makefile.am
index f2a6a6ca..991f3b47 100644
--- a/install/share/Makefile.am
+++ b/install/share/Makefile.am
@@ -42,6 +42,8 @@ app_DATA =				\
 	schema_compat.uldif		\
 	ldapi.ldif			\
 	wsgi.py				\
+	repoint-managed-entries.ldif	\
+	managed-entries.ldif		\
 	user_private_groups.ldif	\
 	host_nis_groups.ldif		\
 	uuid-ipauniqueid.ldif		\
diff --git a/install/share/host_nis_groups.ldif b/install/share/host_nis_groups.ldif
index bb28c597..096a881f 100644
--- a/install/share/host_nis_groups.ldif
+++ b/install/share/host_nis_groups.ldif
@@ -1,4 +1,4 @@
-dn: cn=NGP HGP Template,cn=etc,$SUFFIX
+dn: cn=NGP HGP Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
 changetype: add
 objectclass: mepTemplateEntry
 cn: NGP HGP Template
@@ -13,11 +13,11 @@ mepMappedAttr: description: ipaNetgroup $$cn
 
 # Changes to this definition need to be reflected in
 # updates/20-host_nis_groups.update
-dn: cn=NGP Definition,cn=Managed Entries,cn=plugins,cn=config
+dn: cn=NGP Definition,cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
 changetype: add
 objectclass: extensibleObject
 cn: NGP Definition
 originScope: cn=hostgroups,cn=accounts,$SUFFIX
 originFilter: objectclass=ipahostgroup
 managedBase: cn=ng,cn=alt,$SUFFIX
-managedTemplate: cn=NGP HGP Template,cn=etc,$SUFFIX
+managedTemplate: cn=NGP HGP Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
diff --git a/install/share/managed-entries.ldif b/install/share/managed-entries.ldif
new file mode 100644
index 00000000..ce65eae5
--- /dev/null
+++ b/install/share/managed-entries.ldif
@@ -0,0 +1,17 @@
+dn: cn=Managed Entries,cn=etc,$SUFFIX
+changetype: add
+objectClass: nsContainer
+objectClass: top
+cn: Managed Entries
+
+dn: cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
+changetype: add
+objectClass: nsContainer
+objectClass: top
+cn: Templates
+
+dn: cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
+changetype: add
+objectClass: nsContainer
+objectClass: top
+cn: Definitions
diff --git a/install/share/repoint-managed-entries.ldif b/install/share/repoint-managed-entries.ldif
new file mode 100644
index 00000000..89666621
--- /dev/null
+++ b/install/share/repoint-managed-entries.ldif
@@ -0,0 +1,5 @@
+# Repoint Managed Entries to the replicated cn=etc space
+dn: cn=Managed Entries,cn=plugins,cn=config
+changetype: modify
+add: nsslapd-pluginConfigArea
+nsslapd-pluginConfigArea: cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
diff --git a/install/share/user_private_groups.ldif b/install/share/user_private_groups.ldif
index 9aed09ba..0d5656d4 100644
--- a/install/share/user_private_groups.ldif
+++ b/install/share/user_private_groups.ldif
@@ -1,4 +1,4 @@
-dn: cn=UPG Template,cn=etc,$SUFFIX
+dn: cn=UPG Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
 changetype: add
 objectclass: mepTemplateEntry
 cn: UPG Template
@@ -12,12 +12,12 @@ mepMappedAttr: description: User private group for $$uid
 
 # Changes to this definition need to be reflected in
 # updates/20-user_private_groups.update
-dn: cn=UPG Definition,cn=Managed Entries,cn=plugins,cn=config
+dn: cn=UPG Definition,cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
 changetype: add
 objectclass: extensibleObject
 cn: UPG Definition
 originScope: cn=users,cn=accounts,$SUFFIX
 originFilter: (&(objectclass=posixAccount)(!(description=__no_upg__)))
 managedBase: cn=groups,cn=accounts,$SUFFIX
-managedTemplate: cn=UPG Template,cn=etc,$SUFFIX
+managedTemplate: cn=UPG Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
 
diff --git a/install/updates/19-managed-entries.update b/install/updates/19-managed-entries.update
new file mode 100644
index 00000000..1d8ebebf
--- /dev/null
+++ b/install/updates/19-managed-entries.update
@@ -0,0 +1,17 @@
+dn: cn=Managed Entries,cn=plugins,cn=config
+only: nsslapd-pluginConfigArea: 'cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX'
+
+dn: cn=Managed Entries,cn=etc,$SUFFIX
+default: objectClass: nsContainer
+default: objectClass: top
+default: cn: Managed Entries
+
+dn: cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
+default: objectClass: nsContainer
+default: objectClass: top
+default: cn: Templates
+
+dn: cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
+default: objectClass: nsContainer
+default: objectClass: top
+default: cn: Definitions
diff --git a/install/updates/20-host_nis_groups.update b/install/updates/20-host_nis_groups.update
index 66298021..c6fe8d8a 100644
--- a/install/updates/20-host_nis_groups.update
+++ b/install/updates/20-host_nis_groups.update
@@ -2,14 +2,22 @@
 # This is required for replication. The template entry will get
 # replicated but the plugin configuration will not.
 
-dn: cn=NGP Definition,cn=Managed Entries,cn=plugins,cn=config
+dn: cn=NGP HGP Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
+default:objectclass: mepTemplateEntry
+default:cn: NGP HGP Template
+default:mepRDNAttr: cn
+default:mepStaticAttr: ipaUniqueId: autogenerate
+default:mepStaticAttr: objectclass: ipanisnetgroup
+default:mepStaticAttr: objectclass: ipaobject
+default:mepStaticAttr: nisDomainName: $DOMAIN
+default:mepMappedAttr: cn: $$cn
+default:mepMappedAttr: memberHost: $$dn
+default:mepMappedAttr: description: ipaNetgroup $$cn
+
+dn: cn=NGP Definition,cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
 default:objectclass: extensibleObject
-default:cn: NGP Definition
+only:cn: NGP Definition
 default:originScope: cn=hostgroups,cn=accounts,$SUFFIX
 default:originFilter: objectclass=ipahostgroup
 default:managedBase: cn=ng,cn=alt,$SUFFIX
-default:managedTemplate: cn=NGP HGP Template,cn=etc,$SUFFIX
-
-# Fix an existing configuration with the wrong cn
-dn: cn=NGP Definition,cn=Managed Entries,cn=plugins,cn=config
-only:cn: NGP Definition
+default:managedTemplate: cn=NGP HGP Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
diff --git a/install/updates/20-user_private_groups.update b/install/updates/20-user_private_groups.update
index 8c7baca4..d54cc02d 100644
--- a/install/updates/20-user_private_groups.update
+++ b/install/updates/20-user_private_groups.update
@@ -2,10 +2,23 @@
 # This is required for replication. The template entry will get
 # replicated but the plugin configuration will not.
 
-dn: cn=UPG Definition,cn=Managed Entries,cn=plugins,cn=config
+dn: cn=UPG Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
+default:objectclass: mepTemplateEntry
+default:cn: UPG Template
+default:mepRDNAttr: cn
+default:mepStaticAttr: objectclass: posixgroup
+default:mepStaticAttr: objectclass: ipaobject
+default:mepStaticAttr: ipaUniqueId: autogenerate
+default:mepMappedAttr: cn: $$uid
+default:mepMappedAttr: gidNumber: $$uidNumber
+default:mepMappedAttr: description: User private group for $$uid
+
+
+dn: cn=UPG Definition,cn=Definitions,cn=Managed Entries,cn=etc,$SUFFIX
 default:objectclass: extensibleObject
+replace:originFilter:objectclass=posixAccount::(&(objectclass=posixAccount)(!(description=__no_upg__)))
 default:cn: UPG Definition
 default:originScope: cn=users,cn=accounts,$SUFFIX
-default:originFilter: (&(objectclass=posixAccount)(!(description=__no_upg__)))
+default:originFilter: objectclass=posixAccount
 default:managedBase: cn=groups,cn=accounts,$SUFFIX
-default:managedTemplate: cn=UPG Template,cn=etc,$SUFFIX
+default:managedTemplate: cn=UPG Template,cn=Templates,cn=Managed Entries,cn=etc,$SUFFIX
diff --git a/install/updates/50-suppress-upg.update b/install/updates/50-suppress-upg.update
deleted file mode 100644
index 57178826..00000000
--- a/install/updates/50-suppress-upg.update
+++ /dev/null
@@ -1,2 +0,0 @@
-dn: cn=UPG Definition,cn=Managed Entries,cn=plugins,cn=config
-replace: originFilter:objectclass=posixAccount::(&(objectclass=posixAccount)(!(description=__no_upg__)))
diff --git a/install/updates/Makefile.am b/install/updates/Makefile.am
index cf29e3f2..bf4d9af9 100644
--- a/install/updates/Makefile.am
+++ b/install/updates/Makefile.am
@@ -7,6 +7,7 @@ app_DATA =				\
 	10-RFC4876.update		\
 	10-config.update		\
 	10-sudo.update			\
+	19-managed-entries.update	\
 	20-aci.update			\
 	20-dna.update			\
 	20-host_nis_groups.update	\
@@ -22,7 +23,6 @@ app_DATA =				\
 	50-lockout-policy.update	\
 	50-groupuuid.update		\
 	50-hbacservice.update		\
-	50-suppress-upg.update		\
 	$(NULL)
 
 EXTRA_DIST =				\
diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py
index 3ef9dda9..790b560b 100644
--- a/ipaserver/install/dsinstance.py
+++ b/ipaserver/install/dsinstance.py
@@ -201,6 +201,7 @@ class DsInstance(service.Service):
         self.step("configuring ssl for ds instance", self.__enable_ssl)
         self.step("configuring certmap.conf", self.__certmap_conf)
         self.step("configure autobind for root", self.__root_autobind)
+        self.step("configure new location for managed entries", self.__repoint_managed_entries)
         self.step("restarting directory server", self.__restart_instance)
 
     def __common_post_setup(self):
@@ -237,6 +238,7 @@ class DsInstance(service.Service):
         self.step("adding default layout", self.__add_default_layout)
         self.step("adding delegation layout", self.__add_delegation_layout)
         self.step("adding replication acis", self.__add_replication_acis)
+        self.step("creating container for managed entries", self.__managed_entries)
         self.step("configuring user private groups", self.__user_private_groups)
         self.step("configuring netgroups from hostgroups", self.__host_nis_groups)
         self.step("creating default Sudo bind user", self.__add_sudo_binduser)
@@ -277,8 +279,6 @@ class DsInstance(service.Service):
         # See LDIFs for automember configuration during replica install
         self.step("setting Auto Member configuration", self.__add_replica_automember_config)
 
-        # Managed Entries configuration is done via update files
-
         self.__common_post_setup()
 
         self.start_creation("Configuring directory server", 60)
@@ -487,6 +487,16 @@ class DsInstance(service.Service):
     def __config_lockout_module(self):
         self._ldap_mod("lockout-conf.ldif")
 
+    def __repoint_managed_entries(self):
+        if not has_managed_entries(self.fqdn, self.dm_password):
+            raise errors.NotFound(reason='Missing Managed Entries Plugin')
+        self._ldap_mod("repoint-managed-entries.ldif", self.sub_dict)
+
+    def __managed_entries(self):
+        if not has_managed_entries(self.fqdn, self.dm_password):
+            raise errors.NotFound(reason='Missing Managed Entries Plugin')
+        self._ldap_mod("managed-entries.ldif", self.sub_dict)
+
     def __user_private_groups(self):
         if not has_managed_entries(self.fqdn, self.dm_password):
             raise errors.NotFound(reason='Missing Managed Entries Plugin')
diff --git a/ipaserver/install/installutils.py b/ipaserver/install/installutils.py
index fd978e71..57601d7a 100644
--- a/ipaserver/install/installutils.py
+++ b/ipaserver/install/installutils.py
@@ -441,6 +441,27 @@ def wait_for_open_ports(host, ports, timeout=0):
                 else:
                     raise e
 
+def wait_for_open_socket(socket_name, timeout=0):
+    """
+    Wait until the specified socket on the local host is open. Timeout
+    in seconds may be specified to limit the wait.
+    """
+    op_timeout = time.time() + timeout
+
+    while True:
+        try:
+            s = socket.socket(socket.AF_UNIX)
+            s.connect(socket_name)
+            s.close()
+            break;
+        except socket.error, e:
+            if e.errno == 111:  # 111: Connection refused
+                if timeout and time.time() > op_timeout: # timeout exceeded
+                    raise e
+                time.sleep(1)
+            else:
+                raise e
+
 def resolve_host(host_name):
     try:
         addrinfos = socket.getaddrinfo(host_name, None,
diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py
index 4805dca9..f2f416b9 100644
--- a/ipaserver/install/ldapupdate.py
+++ b/ipaserver/install/ldapupdate.py
@@ -26,6 +26,7 @@ UPDATES_DIR="/usr/share/ipa/updates/"
 
 import sys
 from ipaserver.install import installutils
+from ipaserver.install import service
 from ipaserver import ipaldap
 from ipapython import entity, ipautil
 from ipalib import util
@@ -48,6 +49,24 @@ class BadSyntax(Exception):
     def __str__(self):
         return repr(self.value)
 
+class IPARestart(service.Service):
+    """
+    Restart the 389 DS service prior to performing deletions.
+    """
+    def __init__(self, live_run=True):
+        """
+        This class is present to provide ldapupdate the means to
+        restart 389 DS to apply updates prior to performing deletes.
+        """
+
+        service.Service.__init__(self, "dirsrv")
+        self.live_run = live_run
+
+    def create_instance(self):
+        self.step("stopping directory server", self.stop)
+        self.step("starting directory server", self.start)
+        self.start_creation("Restarting IPA to initialize updates before performing deletes:")
+
 class LDAPUpdate:
     def __init__(self, dm_password, sub_dict={}, live_run=True,
                  online=True, ldapi=False):
@@ -64,7 +83,6 @@ class LDAPUpdate:
         self.modified = False
         self.online = online
         self.ldapi = ldapi
-
         self.pw_name = pwd.getpwuid(os.geteuid()).pw_name
 
         if sub_dict.get("REALM"):
@@ -418,6 +436,54 @@ class LDAPUpdate:
 
         return self.conn.getList(dn, scope, searchfilter, sattrs)
 
+    def __update_managed_entries(self):
+        """Update and move legacy Managed Entry Plugins."""
+
+        suffix = ipautil.realm_to_suffix(self.realm)
+        searchfilter = '(objectclass=*)'
+        definitions_managed_entries = []
+        old_template_container = 'cn=etc,%s' % suffix
+        old_definition_container = 'cn=Managed Entries,cn=plugins,cn=config'
+        new = 'cn=Managed Entries,cn=etc,%s' % suffix
+        sub = ['cn=Definitions,', 'cn=Templates,']
+        new_managed_entries = []
+        old_templates = []
+        template = None
+        try:
+            definitions_managed_entries = self.conn.getList(old_definition_container, ldap.SCOPE_ONELEVEL, searchfilter,[])
+        except errors.NotFound, e:
+            return new_managed_entries
+        for entry in definitions_managed_entries:
+            new_definition = {}
+            definition_managed_entry_updates = {}
+            definitions_managed_entries
+            old_definition = {'dn': entry.dn, 'deleteentry': ['dn: %s' % entry.dn]}
+            old_template = entry.getValue('managedtemplate')
+            entry.setValues('managedtemplate', entry.getValue('managedtemplate').replace(old_template_container, sub[1] + new))
+            new_definition['dn'] = entry.dn.replace(old_definition_container, sub[0] + new)
+            new_definition['default'] = str(entry).strip().replace(': ', ':').split('\n')[1:]
+            definition_managed_entry_updates[new_definition['dn']] = new_definition
+            definition_managed_entry_updates[old_definition['dn']] = old_definition
+            old_templates.append(old_template)
+            new_managed_entries.append(definition_managed_entry_updates)
+        for old_template in old_templates:
+            try:
+                template = self.conn.getEntry(old_template, ldap.SCOPE_BASE, searchfilter,[])
+                new_template = {}
+                template_managed_entry_updates = {}
+                old_template = {'dn': template.dn, 'deleteentry': ['dn: %s' % template.dn]}
+                new_template['dn'] = template.dn.replace(old_template_container, sub[1] + new)
+                new_template['default'] = str(template).strip().replace(': ', ':').split('\n')[1:]
+                template_managed_entry_updates[new_template['dn']] = new_template
+                template_managed_entry_updates[old_template['dn']] = old_template
+                new_managed_entries.append(template_managed_entry_updates)
+            except errors.NotFound, e:
+                pass
+        if len(new_managed_entries) > 0:
+            new_managed_entries.sort(reverse=True)
+
+        return new_managed_entries
+
     def __apply_updates(self, updates, entry):
         """updates is a list of changes to apply
            entry is the thing to apply them to
@@ -431,7 +497,6 @@ class LDAPUpdate:
         for u in updates:
             # We already do syntax-parsing so this is safe
             (utype, k, values) = u.split(':',2)
-
             values = self.__parse_values(values)
 
             e = entry.getValues(k)
@@ -440,7 +505,6 @@ class LDAPUpdate:
                     e = []
                 else:
                     e = [e]
-
             for v in values:
                 if utype == 'remove':
                     logging.debug("remove: '%s' from %s, current value %s", v, k, e)
@@ -629,6 +693,18 @@ class LDAPUpdate:
         and child in the wrong order.
         """
         dn = updates['dn']
+        deletes = updates.get('deleteentry', [])
+        for d in deletes:
+            try:
+               if self.live_run:
+                   self.conn.deleteEntry(dn)
+               self.modified = True
+            except errors.NotFound, e:
+                logging.info("Deleting non-existent entry %s", e)
+                self.modified = True
+            except errors.DatabaseError, e:
+                logging.error("Delete failed: %s", e)
+
         updates = updates.get('updates', [])
         for u in updates:
             # We already do syntax-parsing so this is safe
@@ -659,6 +735,31 @@ class LDAPUpdate:
         f.sort()
         return f
 
+    def create_connection(self):
+        if self.online:
+            if self.ldapi:
+                self.conn = ipaldap.IPAdmin(ldapi=True, realm=self.realm)
+            else:
+                self.conn = ipaldap.IPAdmin(self.sub_dict['FQDN'],
+                                            ldapi=False,
+                                            realm=self.realm)
+            try:
+                if self.dm_password:
+                    self.conn.do_simple_bind(binddn="cn=directory manager", bindpw=self.dm_password)
+                elif os.getegid() == 0:
+                    try:
+                        # autobind
+                        self.conn.do_external_bind(self.pw_name)
+                    except errors.NotFound:
+                        # Fall back
+                        self.conn.do_sasl_gssapi_bind()
+                else:
+                    self.conn.do_sasl_gssapi_bind()
+            except ldap.LOCAL_ERROR, e:
+                raise RuntimeError('%s' % e.args[0].get('info', '').strip())
+        else:
+            raise RuntimeError("Offline updates are not supported.")
+
     def update(self, files):
         """Execute the update. files is a list of the update files to use.
 
@@ -666,29 +767,7 @@ class LDAPUpdate:
         """
 
         try:
-            if self.online:
-                if self.ldapi:
-                    self.conn = ipaldap.IPAdmin(ldapi=True, realm=self.realm)
-                else:
-                    self.conn = ipaldap.IPAdmin(self.sub_dict['FQDN'],
-                                                ldapi=False,
-                                                realm=self.realm)
-                try:
-                    if self.dm_password:
-                        self.conn.do_simple_bind(binddn="cn=directory manager", bindpw=self.dm_password)
-                    elif os.getegid() == 0:
-                        try:
-                            # autobind
-                            self.conn.do_external_bind(self.pw_name)
-                        except errors.NotFound:
-                            # Fall back
-                            self.conn.do_sasl_gssapi_bind()
-                    else:
-                        self.conn.do_sasl_gssapi_bind()
-                except ldap.LOCAL_ERROR, e:
-                    raise RuntimeError('%s' % e.args[0].get('info', '').strip())
-            else:
-                raise RuntimeError("Offline updates are not supported.")
+            self.create_connection()
             all_updates = {}
             dn_list = {}
             for f in files:
@@ -701,6 +780,20 @@ class LDAPUpdate:
 
                 (all_updates, dn_list) = self.parse_update_file(data, all_updates, dn_list)
 
+            # Process Managed Entry Updates
+            managed_entries = self.__update_managed_entries()
+            if managed_entries:
+                managed_entry_dns = [[m[entry]['dn'] for entry in m] for m in managed_entries]
+                l = len(dn_list.keys())
+
+                # Add Managed Entry DN's to the DN List
+                for dn in managed_entry_dns:
+                    l+=1
+                    dn_list[l] = dn
+                # Add Managed Entry Updates to All Updates List
+                for managed_entry in managed_entries:
+                    all_updates.update(managed_entry)
+
             # For adds and updates we want to apply updates from shortest
             # to greatest length of the DN. For deletes we want the reverse.
             sortedkeys = dn_list.keys()
@@ -709,6 +802,13 @@ class LDAPUpdate:
                 for dn in dn_list[k]:
                     self.__update_record(all_updates[dn])
 
+            # Restart 389 Directory Service
+            socket_name = '/var/run/slapd-%s.socket' % self.realm.replace('.','-')
+            iparestart = IPARestart()
+            iparestart.create_instance()
+            installutils.wait_for_open_socket(socket_name)
+            self.create_connection()
+
             sortedkeys.reverse()
             for k in sortedkeys:
                 for dn in dn_list[k]: