Changes to enable Cobbler to also speak XMLRPC on a Unix domain socket which

will later be usable by cobbler (the command line app) to make it very speedy.  Talking to an open API handle in Cobbler will be much faster when editing, say, 5000 systems at once.

7 files changed, 365 insertions, 48 deletions

diff --git a/cobbler/cobblerd.py b/cobbler/cobblerd.py
index 6edfc8d..4e9e52e 100644
--- a/cobbler/cobblerd.py
+++ b/cobbler/cobblerd.py
@@ -19,6 +19,7 @@ import glob
 from rhpl.translate import _, N_, textdomain, utf8
 import xmlrpclib
 
+from server import xmlrpclib2
 import api as cobbler_api
 import yaml # Howell Clark version
 import utils
@@ -50,12 +51,19 @@ def do_xmlrpc_tasks(bootapi, settings, xmlrpc_port, xmlrpc_port2, logger):
     if str(settings.xmlrpc_rw_enabled) != "0":
         pid2 = os.fork()
         if pid2 == 0:
-            do_xmlrpc(bootapi, settings, xmlrpc_port, logger)
+            do_mandatory_xmlrpc_tasks(bootapi, settings, xmlrpc_port, logger)
         else:
             do_xmlrpc_rw(bootapi, settings, xmlrpc_port2, logger)
     else:
         logger.debug("xmlrpc_rw is disabled in the settings file")
+        do_mandatory_xmlrpc_tasks(bootapi, settings, xmlrpc_port, logger)
+
+def do_mandatory_xmlrpc_tasks(bootapi,settings,xmlrpc_port,logger):
+    pid3 = os.fork()
+    if pid3 == 0:
         do_xmlrpc(bootapi, settings, xmlrpc_port, logger)
+    else:
+        do_xmlrpc_unix(bootapi, settings, logger)
 
 
 def do_other_tasks(bootapi, settings, syslog_port, logger):
@@ -96,7 +104,7 @@ def do_xmlrpc(bootapi, settings, port, logger):
     # This is the simple XMLRPC API we provide to koan and other
     # apps that do not need to manage Cobbler's config
 
-    xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerXMLRPCInterface)
+    xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerXMLRPCInterface,True)
     
     server = remote.CobblerXMLRPCServer(('', port))
     server.logRequests = 0  # don't print stuff
@@ -112,7 +120,7 @@ def do_xmlrpc(bootapi, settings, port, logger):
 
 def do_xmlrpc_rw(bootapi,settings,port,logger):
 
-    xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerReadWriteXMLRPCInterface)
+    xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerReadWriteXMLRPCInterface,False)
     server = remote.CobblerReadWriteXMLRPCServer(('127.0.0.1', port))
     server.logRequests = 0  # don't print stuff
     logger.debug("XMLRPC (read-write variant) running on %s" % port)
@@ -125,6 +133,22 @@ def do_xmlrpc_rw(bootapi,settings,port,logger):
             # interrupted? try to serve again
             time.sleep(0.5)
 
+def do_xmlrpc_unix(bootapi,settings,logger):
+
+    xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerReadWriteXMLRPCInterface,True)
+    SOCKT = "/var/lib/cobbler/sock"
+    server = xmlrpclib2.UnixXMLRPCServer(SOCKT)
+    server.logRequests = 0  # don't print stuff
+    logger.debug("XMLRPC (socket variant) available on %s" % SOCKT)
+    server.register_instance(xinterface)
+
+    while True:
+        try:
+            server.serve_forever()
+        except IOError:
+            # interrupted? try to serve again
+            time.sleep(0.5)
+
 def do_syslog(bootapi, settings, port, logger):
 
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -173,5 +197,5 @@ if __name__ == "__main__":
     xmlrpc_port  = settings.xmlrpc_port
     xmlrpc_port2 = settings.xmlrpc_rw_port
     logger       = bootapi.logger_remote
-    do_xmlrpc_rw(bootapi, settings, xmlrpc_port2, logger)
+    do_xmlrpc_unix(bootapi, settings, logger)
 
diff --git a/cobbler/demo_connect.py b/cobbler/demo_connect.py
index 8a91012..0fa058b 100644
--- a/cobbler/demo_connect.py
+++ b/cobbler/demo_connect.py
@@ -1,15 +1,7 @@
 #!/usr/bin/python
 
 """
-work in progress:  demo connection code for Cobbler read-write API 
-uses SSL+XMLRPC (though just XMLRPC will still work)
-adapted from Virt-Factory's old vf_nodecomm source
-XMLRPCSSL portions based on http://linux.duke.edu/~icon/misc/xmlrpcssl.py
-
 Copyright 2007, Red Hat, Inc
-Michael DeHaan <mdehaan@redhat.com>
-Adrian Likins <alikins@redhat.com>
-Scott Seago <sseago@redhat.com>
 
 This software may be freely redistributed under the terms of the GNU
 general public license.
@@ -19,40 +11,11 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 """
 
-import sys
-import socket
-
-from M2Crypto import SSL
-from M2Crypto.m2xmlrpclib import SSL_Transport, Server
-from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-
-
-# workaround for bz #237902
-class CobblerTransport(SSL_Transport):
-    def __init__(self, ssl_context=None, use_datetime=0):
-        self._use_datetime = use_datetime
-        SSL_Transport.__init__(self,ssl_context=ssl_context)
-
-def demo_connect(username,password,server):
-    my_ctx = SSL.Context('sslv23')
-    # my_ctx.load_client_ca("foo.pem")
-    # my_ctx.load_cert("bar.pem","baz.pem")
-    my_ctx.set_session_id_ctx("xmlrpcssl")
-    my_ctx.set_allow_unknown_ca(True)
-    my_ctx.set_verify(0,-1) # full anonymous (we hope)
-    my_uri = "https://%s:443/cobbler_api_rw" % server
-    print "connecting to: %s" % my_uri
-    my_rserver = Server(my_uri, CobblerTransport(ssl_context = my_ctx))
-    token = my_rserver.login(username,password)
-    print "got a token: %s" % token
-    rc = my_rserver.test(token)
-    print "got test results: %s" % rc
+from server.xmlrpcclient import ServerProxy
 
 if __name__ == "__main__":
-    USERNAME = "mdehaan"
-    PASSWORD = "llamas2007"
-    SERVER   = "mdehaan.rdu.redhat.com"
-    demo_connect(USERNAME,PASSWORD,SERVER)
+    sp = ServerProxy("httpu:///var/lib/cobbler/sock")
+    print sp.login("<system>","")
 
 
 
diff --git a/cobbler/remote.py b/cobbler/remote.py
index 07539e4..898e712 100644
--- a/cobbler/remote.py
+++ b/cobbler/remote.py
@@ -53,9 +53,10 @@ class CobblerXMLRPCInterface:
     interface are intentionally /not/ validated.  It's a public API.
     """
 
-    def __init__(self,api,logger):
+    def __init__(self,api,logger,enable_auth_if_relevant):
         self.api = api
         self.logger = logger
+        self.auth_enabled = enable_auth_if_relevant
 
     def __sorter(self,a,b):
         return cmp(a["name"],b["name"])
@@ -427,9 +428,9 @@ class CobblerXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
 
 class ProxiedXMLRPCInterface:
 
-    def __init__(self,api,logger,proxy_class):
+    def __init__(self,api,logger,proxy_class,enable_auth_if_relevant=True):
         self.logger  = logger
-        self.proxied = proxy_class(api,logger)
+        self.proxied = proxy_class(api,logger,enable_auth_if_relevant)
 
     def _dispatch(self, method, params):
 
@@ -449,8 +450,9 @@ class ProxiedXMLRPCInterface:
 
 class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface):
 
-    def __init__(self,api,logger):
+    def __init__(self,api,logger,enable_auth_if_relevant):
         self.api = api
+        self.auth_enabled = enable_auth_if_relevant
         self.logger = logger
         self.token_cache = TOKEN_CACHE
         self.object_cache = OBJECT_CACHE
@@ -511,9 +513,16 @@ class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface):
         Returns whether this user/pass combo should be given
         access to the cobbler read-write API.
 
+        For the system user, this answer is always "yes", but
+        it is only valid for the socket interface.
+
         FIXME: currently looks for users in /etc/cobbler/auth.conf
         Would be very nice to allow for PAM and/or just Kerberos.
         """
+        if not self.auth_enabled and input_user == "<system>":
+            return True
+        if self.auth_enabled and input_user == "<system>":
+            return False
         return self.api.authenticate(input_user,input_password)
 
     def __validate_token(self,token): 
@@ -527,8 +536,18 @@ class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface):
         """
         self.__invalidate_expired_tokens()
         self.__invalidate_expired_objects()
+
+        if not self.auth_enabled:
+            user = self.get_user_from_token(token)
+            if user == "<system>":
+                self.token_cache[token] = (time.time(), user) # update to prevent timeout
+                return True
+
         if self.token_cache.has_key(token):
             user = self.get_user_from_token(token)
+            if user == "<system>":
+               # system token is only valid over Unix socket
+               return False
             self.token_cache[token] = (time.time(), user) # update to prevent timeout
             return True
         else:
@@ -537,6 +556,8 @@ class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface):
 
     def check_access(self,token,resource,arg1=None,arg2=None):
         validated = self.__validate_token(token)
+        if not self.auth_enabled:
+            return True
         return self.__authorize(token,resource,arg1,arg2)
 
 
diff --git a/cobbler/server/__init__.py b/cobbler/server/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cobbler/server/__init__.py
diff --git a/cobbler/server/xmlrpcclient.py b/cobbler/server/xmlrpcclient.py
new file mode 100644
index 0000000..cc0687c
--- /dev/null
+++ b/cobbler/server/xmlrpcclient.py
@@ -0,0 +1,72 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2006 Anthony Liguori <aliguori@us.ibm.com>
+# Copyright (C) 2007 XenSource Inc.
+# Copyright (C) 2007 Red Hat Inc, Michael DeHaan <mdehaan@redhat.com>
+
+from httplib import FakeSocket, HTTPConnection, HTTP
+import socket
+import string
+import xmlrpclib
+from types import StringTypes
+
+class HTTPUnixConnection(HTTPConnection):
+    def connect(self):
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(self.host)
+
+class HTTPUnix(HTTP):
+    _connection_class = HTTPUnixConnection
+
+class UnixTransport(xmlrpclib.Transport):
+    def request(self, host, handler, request_body, verbose=0):
+        self.__handler = handler
+        return xmlrpclib.Transport.request(self, host, '/RPC2',
+                                           request_body, verbose)
+    def make_connection(self, host):
+        return HTTPUnix(self.__handler)
+
+# See xmlrpclib2.TCPXMLRPCServer._marshalled_dispatch.
+def conv_string(x):
+    if isinstance(x, StringTypes):
+        s = string.replace(x, "'", r"\047")
+        exec "s = '" + s + "'"
+        return s
+    else:
+        return x
+
+
+class ServerProxy(xmlrpclib.ServerProxy):
+    def __init__(self, uri, transport=None, encoding=None, verbose=0,
+                 allow_none=1):
+        if transport == None:
+            (protocol, rest) = uri.split(':', 1)
+            if protocol == 'httpu':
+                uri = 'http:' + rest
+                transport = UnixTransport()
+            else:
+                raise ValueError("only httpu://path/to/socket is supported")
+        xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding,
+                                       verbose, allow_none)
+
+
+    def __request(self, methodname, params):
+        response = xmlrpclib.ServerProxy.__request(self, methodname, params)
+
+        if isinstance(response, tuple):
+            return tuple([conv_string(x) for x in response])
+        else:
+            return conv_string(response)
+
diff --git a/cobbler/server/xmlrpclib2.py b/cobbler/server/xmlrpclib2.py
new file mode 100644
index 0000000..d329241
--- /dev/null
+++ b/cobbler/server/xmlrpclib2.py
@@ -0,0 +1,236 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2006 Anthony Liguori <aliguori@us.ibm.com>
+# Copyright (C) 2006 XenSource Inc.
+# Copyright (C) 2007 Red Hat Inc., Michael DeHaan <mdehaan@redhat.com>
+#============================================================================
+
+"""
+An enhanced XML-RPC client/server interface for Python.
+"""
+
+import re
+import fcntl
+from types import *
+import os    
+import errno
+import traceback
+
+from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+import SocketServer
+import xmlrpclib, socket, os, stat
+
+#import mkdir
+
+#
+# Convert all integers to strings as described in the Xen API
+#
+
+
+def stringify(value):
+    if isinstance(value, long) or \
+       (isinstance(value, int) and not isinstance(value, bool)):
+        return str(value)
+    elif isinstance(value, dict):
+        new_value = {}
+        for k, v in value.items():
+            new_value[stringify(k)] = stringify(v)
+        return new_value
+    elif isinstance(value, (tuple, list)):
+        return [stringify(v) for v in value]
+    else:
+        return value
+
+
+# We're forced to subclass the RequestHandler class so that we can work around
+# some bugs in Keep-Alive handling and also enabled it by default
+class XMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
+    protocol_version = "HTTP/1.1"
+
+    def __init__(self, request, client_address, server):
+        SimpleXMLRPCRequestHandler.__init__(self, request, client_address,
+                                            server)
+
+    # this is inspired by SimpleXMLRPCRequestHandler's do_POST but differs
+    # in a few non-trivial ways
+    # 1) we never generate internal server errors.  We let the exception
+    #    propagate so that it shows up in the Xend debug logs
+    # 2) we don't bother checking for a _dispatch function since we don't
+    #    use one
+    def do_POST(self):
+        addrport = self.client_address
+        #if not connection.hostAllowed(addrport, self.hosts_allowed):
+        #    self.connection.shutdown(1)
+        #    return
+
+        data = self.rfile.read(int(self.headers["content-length"]))
+        rsp = self.server._marshaled_dispatch(data)
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/xml")
+        self.send_header("Content-Length", str(len(rsp)))
+        self.end_headers()
+
+        self.wfile.write(rsp)
+        self.wfile.flush()
+        #if self.close_connection == 1:
+        #    self.connection.shutdown(1)
+
+def parents(dir, perms, enforcePermissions = False):
+    """
+    Ensure that the given directory exists, creating it if necessary, but not
+    complaining if it's already there.
+
+    @param dir The directory name.
+    @param perms One of the stat.S_ constants.
+    @param enforcePermissions Enforce our ownership and the given permissions,
+    even if the directory pre-existed with different ones.
+    """
+    # Catch the exception here, rather than checking for the directory's
+    # existence first, to avoid races.
+    try:
+        os.makedirs(dir, perms)
+    except OSError, exn:
+        if exn.args[0] != errno.EEXIST or not os.path.isdir(dir):
+            raise
+    if enforcePermissions:
+        os.chown(dir, os.geteuid(), os.getegid())
+        os.chmod(dir, stat.S_IRWXU)
+
+
+# This is a base XML-RPC server for TCP.  It sets allow_reuse_address to
+# true, and has an improved marshaller that logs and serializes exceptions.
+
+class TCPXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer):
+    allow_reuse_address = True
+
+    def __init__(self, addr, requestHandler=None,
+                 logRequests = 1):
+        if requestHandler is None:
+            requestHandler = XMLRPCRequestHandler
+        SimpleXMLRPCServer.__init__(self, addr,
+                                    (lambda x, y, z:
+                                     requestHandler(x, y, z)),
+                                    logRequests)
+
+        flags = fcntl.fcntl(self.fileno(), fcntl.F_GETFD)
+        flags |= fcntl.FD_CLOEXEC
+        fcntl.fcntl(self.fileno(), fcntl.F_SETFD, flags)
+
+    def get_request(self):
+        (client, addr) = SimpleXMLRPCServer.get_request(self)
+        flags = fcntl.fcntl(client.fileno(), fcntl.F_GETFD)
+        flags |= fcntl.FD_CLOEXEC
+        fcntl.fcntl(client.fileno(), fcntl.F_SETFD, flags)
+        return (client, addr)
+
+    def _marshaled_dispatch(self, data, dispatch_method = None):
+        params, method = xmlrpclib.loads(data)
+        if False:
+            # Enable this block of code to exit immediately without sending
+            # a response.  This allows you to test client-side crash handling.
+            import sys
+            sys.exit(1)
+        try:
+            if dispatch_method is not None:
+                response = dispatch_method(method, params)
+            else:
+                response = self._dispatch(method, params)
+
+            if (response is None or
+                not isinstance(response, dict) or
+                'Status' not in response):
+                #log.exception('Internal error handling %s: Invalid result %s',
+                #              method, response)
+                response = { "Status": "Failure",
+                             "ErrorDescription":
+                             ['INTERNAL_ERROR',
+                              'Invalid result %s handling %s' %
+                              (response, method)]}
+
+            # With either Unicode or normal strings, we can only transmit
+            # \t, \n, \r, \u0020-\ud7ff, \ue000-\ufffd, and \u10000-\u10ffff
+            # in an XML document.  xmlrpclib does not escape these values
+            # properly, and then breaks when it comes to parse the document.
+            # To hack around this problem, we use repr here and exec above
+            # to transmit the string using Python encoding.
+            # Thanks to David Mertz <mertz@gnosis.cx> for the trick (buried
+            # in xml_pickle.py).
+            if isinstance(response, StringTypes):
+                response = repr(response)[1:-1]
+
+            response = (response,)
+            response = xmlrpclib.dumps(response,
+                                       methodresponse=1,
+                                       allow_none=1)
+        except Exception, exn:
+            try:
+                #if self.xenapi:
+                #    if _is_not_supported(exn):
+                #         errdesc = ['MESSAGE_METHOD_UNKNOWN', method]
+                #    else:
+                #         #log.exception('Internal error handling %s', method)
+                #         errdesc = ['INTERNAL_ERROR', str(exn)]
+                #
+                #    response = xmlrpclib.dumps(
+                #          ({ "Status": "Failure",
+                #             "ErrorDescription": errdesc },),
+                #          methodresponse = 1)
+                #else:
+                #    import xen.xend.XendClient
+                if isinstance(exn, xmlrpclib.Fault):
+                    response = xmlrpclib.dumps(exn)
+                else:
+                    # log.exception('Internal error handling %s', method)
+                    response = xmlrpclib.dumps(
+                            xmlrpclib.Fault(101, str(exn)))
+            except Exception, exn2:
+                # FIXME
+                traceback.print_exc()
+
+        return response
+
+
+notSupportedRE = re.compile(r'method "(.*)" is not supported')
+def _is_not_supported(exn):
+    try:
+        m = notSupportedRE.search(exn[0])
+        return m is not None
+    except:
+        return False
+
+
+# This is a XML-RPC server that sits on a Unix domain socket.
+# It implements proper support for allow_reuse_address by
+# unlink()'ing an existing socket.
+
+class UnixXMLRPCRequestHandler(XMLRPCRequestHandler):
+    def address_string(self):
+        try:
+            return XMLRPCRequestHandler.address_string(self)
+        except ValueError, e:
+            return self.client_address[:2]
+
+class UnixXMLRPCServer(TCPXMLRPCServer):
+    address_family = socket.AF_UNIX
+    allow_address_reuse = True
+
+    def __init__(self, addr, logRequests = 1):
+        parents(os.path.dirname(addr), stat.S_IRWXU, True)
+        if self.allow_reuse_address and os.path.exists(addr):
+            os.unlink(addr)
+
+        TCPXMLRPCServer.__init__(self, addr, 
+                                 UnixXMLRPCRequestHandler, logRequests)
diff --git a/setup.py b/setup.py
index eb2a23d..fddcba3 100644
--- a/setup.py
+++ b/setup.py
@@ -57,6 +57,7 @@ if __name__ == "__main__":
                     "cobbler",
                     "cobbler/yaml",
                     "cobbler/modules", 
+                    "cobbler/server", 
                     "cobbler/webui",
                 ],
                 scripts = ["scripts/cobbler", "scripts/cobblerd", "scripts/cobbler_auth_help"],