Centralize timeout for waiting for servers to start.

All service start/restart currently go through ipapython/platform so
move the "wait for service to start" code there as well.

A dictionary of known services and ports to wait on is defined in base.py
This is referenced by the platforms by instance name to determine what
to wait for. For the case of dirsrv if we get that as a plain name
(no specific instance) it is assumed to be the main IPA service.

https://fedorahosted.org/freeipa/ticket/2375
https://fedorahosted.org/freeipa/ticket/2610

5 files changed, 133 insertions, 26 deletions

diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index 8884e7be9..e80434cfd 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -41,6 +41,7 @@ import re
 import xmlrpclib
 import datetime
 import netaddr
+import time
 from dns import resolver, rdatatype
 from dns.exception import DNSException
 
@@ -1010,3 +1011,56 @@ def utf8_encode_values(values):
         return map(utf8_encode_value, values)
     else:
         return utf8_encode_value(values)
+
+def wait_for_open_ports(host, ports, timeout=0):
+    """
+    Wait until the specified port(s) on the remote host are open. Timeout
+    in seconds may be specified to limit the wait.
+    """
+    if not isinstance(ports, (tuple, list)):
+        ports = [ports]
+
+    root_logger.debug('wait_for_open_ports: %s %s timeout %d' % (host, ports, timeout))
+    op_timeout = time.time() + timeout
+    ipv6_failover = False
+
+    for port in ports:
+        while True:
+            try:
+                if ipv6_failover:
+                    s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+                else:
+                    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                s.connect((host, port))
+                s.close()
+                break
+            except socket.error, e:
+                if e.errno == 111:  # 111: Connection refused
+                    if timeout and time.time() > op_timeout: # timeout exceeded
+                        raise e
+                    time.sleep(1)
+                elif not ipv6_failover: # fallback to IPv6 connection
+                    ipv6_failover = True
+                else:
+                    raise e
+
+def wait_for_open_socket(socket_name, timeout=0):
+    """
+    Wait until the specified socket on the local host is open. Timeout
+    in seconds may be specified to limit the wait.
+    """
+    op_timeout = time.time() + timeout
+
+    while True:
+        try:
+            s = socket.socket(socket.AF_UNIX)
+            s.connect(socket_name)
+            s.close()
+            break
+        except socket.error, e:
+            if e.errno in (2,111):  # 111: Connection refused, 2: File not found
+                if timeout and time.time() > op_timeout: # timeout exceeded
+                    raise e
+                time.sleep(1)
+            else:
+                raise e
diff --git a/ipapython/platform/base.py b/ipapython/platform/base.py
index bf76b7630..6f9d3867a 100644
--- a/ipapython/platform/base.py
+++ b/ipapython/platform/base.py
@@ -18,29 +18,42 @@
 
 from ipalib.plugable import MagicDict
 
-# Canonical names of services as IPA wants to see them. As we need to have *some* naming,
-# set them as in Red Hat distributions. Actual implementation should make them available
-# through knownservices.<name> and take care of remapping internally, if needed
+# Canonical names of services as IPA wants to see them. As we need to have
+# *some* naming, set them as in Red Hat distributions. Actual implementation
+# should make them available through knownservices.<name> and take care of
+# re-mapping internally, if needed
 wellknownservices = ['certmonger', 'dirsrv', 'httpd', 'ipa', 'krb5kdc',
                      'messagebus', 'nslcd', 'nscd', 'ntpd', 'portmap',
                      'rpcbind', 'kadmin', 'sshd', 'autofs', 'rpcgssd',
                      'rpcidmapd']
 
 
+# The common ports for these services. This is used to wait for the
+# service to become available.
+wellknownports = {
+    'dirsrv@PKI-IPA.service': [7389],
+    'PKI-IPA': [7389],
+    'dirsrv': [389], # this is only used if the incoming instance name is blank
+    'pki-cad': [9180],
+}
+
 class AuthConfig(object):
     """
     AuthConfig class implements system-independent interface to configure
     system authentication resources. In Red Hat systems this is done with
     authconfig(8) utility.
 
-    AuthConfig class is nothing more than a tool to gather configuration options
-    and execute their processing. These options then converted by an actual implementation
-    to series of a system calls to appropriate utilities performing real configuration.
+    AuthConfig class is nothing more than a tool to gather configuration
+    options and execute their processing. These options then converted by
+    an actual implementation to series of a system calls to appropriate
+    utilities performing real configuration.
 
-    IPA *expects* names of AuthConfig's options to follow authconfig(8) naming scheme!
+    IPA *expects* names of AuthConfig's options to follow authconfig(8)
+    naming scheme!
 
-    Actual implementation should be done in ipapython/platform/<platform>.py by inheriting from
-    platform.AuthConfig and redefining __build_args() and execute() methods.
+    Actual implementation should be done in ipapython/platform/<platform>.py
+    by inheriting from platform.AuthConfig and redefining __build_args()
+    and execute() methods.
 
     from ipapython.platform import platform
     class PlatformAuthConfig(platform.AuthConfig):
@@ -53,9 +66,11 @@ class AuthConfig(object):
     authconfig = PlatformAuthConfig
     ....
 
-    See ipapython/platform/redhat.py for a sample implementation that uses authconfig(8) as its backend.
+    See ipapython/platform/redhat.py for a sample implementation that uses
+    authconfig(8) as its backend.
 
-    From IPA code perspective, the authentication configuration should be done with use of ipapython.services.authconfig:
+    From IPA code perspective, the authentication configuration should be
+    done with use of ipapython.services.authconfig:
 
     from ipapython import services as ipaservices
     auth_config = ipaservices.authconfig()
@@ -69,8 +84,8 @@ class AuthConfig(object):
                 add_parameter("nisdomain","foobar")
     auth_config.execute()
 
-    If you need to re-use existing AuthConfig instance for multiple runs, make sure to
-    call 'AuthConfig.reset()' between the runs.
+    If you need to re-use existing AuthConfig instance for multiple runs,
+    make sure to call 'AuthConfig.reset()' between the runs.
     """
 
     def __init__(self):
@@ -106,21 +121,21 @@ class AuthConfig(object):
 
 class PlatformService(object):
     """
-    PlatformService abstracts out external process running on the system which is possible
-    to administer (start, stop, check status, etc).
+    PlatformService abstracts out external process running on the system
+    which is possible to administer (start, stop, check status, etc).
 
     """
 
     def __init__(self, service_name):
         self.service_name = service_name
 
-    def start(self, instance_name="", capture_output=True):
+    def start(self, instance_name="", capture_output=True, wait=True):
         return
 
     def stop(self, instance_name="", capture_output=True):
         return
 
-    def restart(self, instance_name="", capture_output=True):
+    def restart(self, instance_name="", capture_output=True, wait=True):
         return
 
     def is_running(self, instance_name=""):
@@ -149,8 +164,9 @@ class PlatformService(object):
 
 class KnownServices(MagicDict):
     """
-    KnownServices is an abstract class factory that should give out instances of well-known
-    platform services. Actual implementation must create these instances as its own attributes
-    on first access (or instance creation) and cache them.
+    KnownServices is an abstract class factory that should give out instances
+    of well-known platform services. Actual implementation must create these
+    instances as its own attributes on first access (or instance creation)
+    and cache them.
     """
 
diff --git a/ipapython/platform/fedora16.py b/ipapython/platform/fedora16.py
index 985d368f9..8b730e41c 100644
--- a/ipapython/platform/fedora16.py
+++ b/ipapython/platform/fedora16.py
@@ -98,7 +98,7 @@ class Fedora16DirectoryService(Fedora16Service):
             restore_context(dirsrv_systemd)
             ipautil.run(["/bin/systemctl", "--system", "daemon-reload"],raiseonerr=False)
 
-    def restart(self, instance_name="", capture_output=True):
+    def restart(self, instance_name="", capture_output=True, wait=True):
         if len(instance_name) > 0:
             elements = self.service_name.split("@")
             srv_etc = os.path.join(self.SYSTEMD_ETC_PATH, self.service_name)
@@ -109,7 +109,7 @@ class Fedora16DirectoryService(Fedora16Service):
             elif not os.path.samefile(srv_etc, srv_lnk):
                 os.unlink(srv_lnk)
                 os.symlink(srv_etc, srv_lnk)
-        super(Fedora16DirectoryService, self).restart(instance_name, capture_output=capture_output)
+        super(Fedora16DirectoryService, self).restart(instance_name, capture_output=capture_output, wait=wait)
 
 # Enforce restart of IPA services when we do enable it
 # This gets around the fact that after ipa-server-install systemd thinks
diff --git a/ipapython/platform/redhat.py b/ipapython/platform/redhat.py
index 28a43e588..d3c23ab0d 100644
--- a/ipapython/platform/redhat.py
+++ b/ipapython/platform/redhat.py
@@ -26,6 +26,7 @@ import sys
 import socket
 from ipapython import ipautil
 from ipapython.platform import base
+from ipalib import api
 
 # All what we allow exporting directly from this module
 # Everything else is made available through these symbols when they are
@@ -46,14 +47,31 @@ from ipapython.platform import base
 __all__ = ['authconfig', 'service', 'knownservices', 'backup_and_replace_hostname', 'restore_context', 'check_selinux_status']
 
 class RedHatService(base.PlatformService):
+    def __wait_for_open_ports(self, instance_name=""):
+        """
+        If this is a service we need to wait for do so.
+        """
+        ports = None
+        if instance_name in base.wellknownports:
+            ports = base.wellknownports[instance_name]
+        else:
+            if self.service_name in base.wellknownports:
+                ports = base.wellknownports[self.service_name]
+        if ports:
+            ipautil.wait_for_open_ports('localhost', ports, api.env.startup_timeout)
+
     def stop(self, instance_name="", capture_output=True):
         ipautil.run(["/sbin/service", self.service_name, "stop", instance_name], capture_output=capture_output)
 
-    def start(self, instance_name="", capture_output=True):
+    def start(self, instance_name="", capture_output=True, wait=True):
         ipautil.run(["/sbin/service", self.service_name, "start", instance_name], capture_output=capture_output)
+        if wait and self.is_running(instance_name):
+            self.__wait_for_open_ports(instance_name)
 
-    def restart(self, instance_name="", capture_output=True):
+    def restart(self, instance_name="", capture_output=True, wait=True):
         ipautil.run(["/sbin/service", self.service_name, "restart", instance_name], capture_output=capture_output)
+        if wait and self.is_running(instance_name):
+            self.__wait_for_open_ports(instance_name)
 
     def is_running(self, instance_name=""):
         ret = True
diff --git a/ipapython/platform/systemd.py b/ipapython/platform/systemd.py
index ae06c0227..a233e1045 100644
--- a/ipapython/platform/systemd.py
+++ b/ipapython/platform/systemd.py
@@ -20,6 +20,7 @@
 from ipapython import ipautil
 from ipapython.platform import base
 import sys, os, shutil
+from ipalib import api
 
 class SystemdService(base.PlatformService):
     SYSTEMD_ETC_PATH = "/etc/systemd/system/"
@@ -73,16 +74,34 @@ class SystemdService(base.PlatformService):
             return (None,None)
         return dict(map(lambda x: splitter(x, separator=separator), text.split("\n")))
 
+    def __wait_for_open_ports(self, instance_name=""):
+        """
+        If this is a service we need to wait for do so.
+        """
+        ports = None
+        if instance_name in base.wellknownports:
+            ports = base.wellknownports[instance_name]
+        else:
+            elements = self.service_name.split("@")
+            if elements[0] in base.wellknownports:
+                ports = base.wellknownports[elements[0]]
+        if ports:
+            ipautil.wait_for_open_ports('localhost', ports, api.env.startup_timeout)
+
     def stop(self, instance_name="", capture_output=True):
         ipautil.run(["/bin/systemctl", "stop", self.service_instance(instance_name)], capture_output=capture_output)
 
-    def start(self, instance_name="", capture_output=True):
+    def start(self, instance_name="", capture_output=True, wait=True):
         ipautil.run(["/bin/systemctl", "start", self.service_instance(instance_name)], capture_output=capture_output)
+        if wait and self.is_running(instance_name):
+            self.__wait_for_open_ports(self.service_instance(instance_name))
 
-    def restart(self, instance_name="", capture_output=True):
+    def restart(self, instance_name="", capture_output=True, wait=True):
         # Restart command is broken before systemd-36-3.fc16
         # If you have older systemd version, restart of dependent services will hang systemd indefinetly
         ipautil.run(["/bin/systemctl", "restart", self.service_instance(instance_name)], capture_output=capture_output)
+        if wait and self.is_running(instance_name):
+            self.__wait_for_open_ports(self.service_instance(instance_name))
 
     def is_running(self, instance_name=""):
         ret = True