From fd84944a95f188b4934b4e18cdbe19344ac6bb5f Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Wed, 19 Dec 2007 17:51:14 -0500 Subject: Changes to enable Cobbler to also speak XMLRPC on a Unix domain socket which will later be usable by cobbler (the command line app) to make it very speedy. Talking to an open API handle in Cobbler will be much faster when editing, say, 5000 systems at once. --- cobbler/cobblerd.py | 32 +++++- cobbler/demo_connect.py | 43 +------- cobbler/remote.py | 29 ++++- cobbler/server/__init__.py | 0 cobbler/server/xmlrpcclient.py | 72 +++++++++++++ cobbler/server/xmlrpclib2.py | 236 +++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 7 files changed, 365 insertions(+), 48 deletions(-) create mode 100644 cobbler/server/__init__.py create mode 100644 cobbler/server/xmlrpcclient.py create mode 100644 cobbler/server/xmlrpclib2.py diff --git a/cobbler/cobblerd.py b/cobbler/cobblerd.py index 6edfc8d..4e9e52e 100644 --- a/cobbler/cobblerd.py +++ b/cobbler/cobblerd.py @@ -19,6 +19,7 @@ import glob from rhpl.translate import _, N_, textdomain, utf8 import xmlrpclib +from server import xmlrpclib2 import api as cobbler_api import yaml # Howell Clark version import utils @@ -50,12 +51,19 @@ def do_xmlrpc_tasks(bootapi, settings, xmlrpc_port, xmlrpc_port2, logger): if str(settings.xmlrpc_rw_enabled) != "0": pid2 = os.fork() if pid2 == 0: - do_xmlrpc(bootapi, settings, xmlrpc_port, logger) + do_mandatory_xmlrpc_tasks(bootapi, settings, xmlrpc_port, logger) else: do_xmlrpc_rw(bootapi, settings, xmlrpc_port2, logger) else: logger.debug("xmlrpc_rw is disabled in the settings file") + do_mandatory_xmlrpc_tasks(bootapi, settings, xmlrpc_port, logger) + +def do_mandatory_xmlrpc_tasks(bootapi,settings,xmlrpc_port,logger): + pid3 = os.fork() + if pid3 == 0: do_xmlrpc(bootapi, settings, xmlrpc_port, logger) + else: + do_xmlrpc_unix(bootapi, settings, logger) def do_other_tasks(bootapi, settings, syslog_port, logger): @@ -96,7 +104,7 @@ def do_xmlrpc(bootapi, settings, port, logger): # This is the simple XMLRPC API we provide to koan and other # apps that do not need to manage Cobbler's config - xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerXMLRPCInterface) + xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerXMLRPCInterface,True) server = remote.CobblerXMLRPCServer(('', port)) server.logRequests = 0 # don't print stuff @@ -112,7 +120,7 @@ def do_xmlrpc(bootapi, settings, port, logger): def do_xmlrpc_rw(bootapi,settings,port,logger): - xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerReadWriteXMLRPCInterface) + xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerReadWriteXMLRPCInterface,False) server = remote.CobblerReadWriteXMLRPCServer(('127.0.0.1', port)) server.logRequests = 0 # don't print stuff logger.debug("XMLRPC (read-write variant) running on %s" % port) @@ -125,6 +133,22 @@ def do_xmlrpc_rw(bootapi,settings,port,logger): # interrupted? try to serve again time.sleep(0.5) +def do_xmlrpc_unix(bootapi,settings,logger): + + xinterface = remote.ProxiedXMLRPCInterface(bootapi,logger,remote.CobblerReadWriteXMLRPCInterface,True) + SOCKT = "/var/lib/cobbler/sock" + server = xmlrpclib2.UnixXMLRPCServer(SOCKT) + server.logRequests = 0 # don't print stuff + logger.debug("XMLRPC (socket variant) available on %s" % SOCKT) + server.register_instance(xinterface) + + while True: + try: + server.serve_forever() + except IOError: + # interrupted? try to serve again + time.sleep(0.5) + def do_syslog(bootapi, settings, port, logger): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -173,5 +197,5 @@ if __name__ == "__main__": xmlrpc_port = settings.xmlrpc_port xmlrpc_port2 = settings.xmlrpc_rw_port logger = bootapi.logger_remote - do_xmlrpc_rw(bootapi, settings, xmlrpc_port2, logger) + do_xmlrpc_unix(bootapi, settings, logger) diff --git a/cobbler/demo_connect.py b/cobbler/demo_connect.py index 8a91012..0fa058b 100644 --- a/cobbler/demo_connect.py +++ b/cobbler/demo_connect.py @@ -1,15 +1,7 @@ #!/usr/bin/python """ -work in progress: demo connection code for Cobbler read-write API -uses SSL+XMLRPC (though just XMLRPC will still work) -adapted from Virt-Factory's old vf_nodecomm source -XMLRPCSSL portions based on http://linux.duke.edu/~icon/misc/xmlrpcssl.py - Copyright 2007, Red Hat, Inc -Michael DeHaan -Adrian Likins -Scott Seago This software may be freely redistributed under the terms of the GNU general public license. @@ -19,40 +11,11 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. """ -import sys -import socket - -from M2Crypto import SSL -from M2Crypto.m2xmlrpclib import SSL_Transport, Server -from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler - - -# workaround for bz #237902 -class CobblerTransport(SSL_Transport): - def __init__(self, ssl_context=None, use_datetime=0): - self._use_datetime = use_datetime - SSL_Transport.__init__(self,ssl_context=ssl_context) - -def demo_connect(username,password,server): - my_ctx = SSL.Context('sslv23') - # my_ctx.load_client_ca("foo.pem") - # my_ctx.load_cert("bar.pem","baz.pem") - my_ctx.set_session_id_ctx("xmlrpcssl") - my_ctx.set_allow_unknown_ca(True) - my_ctx.set_verify(0,-1) # full anonymous (we hope) - my_uri = "https://%s:443/cobbler_api_rw" % server - print "connecting to: %s" % my_uri - my_rserver = Server(my_uri, CobblerTransport(ssl_context = my_ctx)) - token = my_rserver.login(username,password) - print "got a token: %s" % token - rc = my_rserver.test(token) - print "got test results: %s" % rc +from server.xmlrpcclient import ServerProxy if __name__ == "__main__": - USERNAME = "mdehaan" - PASSWORD = "llamas2007" - SERVER = "mdehaan.rdu.redhat.com" - demo_connect(USERNAME,PASSWORD,SERVER) + sp = ServerProxy("httpu:///var/lib/cobbler/sock") + print sp.login("","") diff --git a/cobbler/remote.py b/cobbler/remote.py index 07539e4..898e712 100644 --- a/cobbler/remote.py +++ b/cobbler/remote.py @@ -53,9 +53,10 @@ class CobblerXMLRPCInterface: interface are intentionally /not/ validated. It's a public API. """ - def __init__(self,api,logger): + def __init__(self,api,logger,enable_auth_if_relevant): self.api = api self.logger = logger + self.auth_enabled = enable_auth_if_relevant def __sorter(self,a,b): return cmp(a["name"],b["name"]) @@ -427,9 +428,9 @@ class CobblerXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): class ProxiedXMLRPCInterface: - def __init__(self,api,logger,proxy_class): + def __init__(self,api,logger,proxy_class,enable_auth_if_relevant=True): self.logger = logger - self.proxied = proxy_class(api,logger) + self.proxied = proxy_class(api,logger,enable_auth_if_relevant) def _dispatch(self, method, params): @@ -449,8 +450,9 @@ class ProxiedXMLRPCInterface: class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface): - def __init__(self,api,logger): + def __init__(self,api,logger,enable_auth_if_relevant): self.api = api + self.auth_enabled = enable_auth_if_relevant self.logger = logger self.token_cache = TOKEN_CACHE self.object_cache = OBJECT_CACHE @@ -511,9 +513,16 @@ class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface): Returns whether this user/pass combo should be given access to the cobbler read-write API. + For the system user, this answer is always "yes", but + it is only valid for the socket interface. + FIXME: currently looks for users in /etc/cobbler/auth.conf Would be very nice to allow for PAM and/or just Kerberos. """ + if not self.auth_enabled and input_user == "": + return True + if self.auth_enabled and input_user == "": + return False return self.api.authenticate(input_user,input_password) def __validate_token(self,token): @@ -527,8 +536,18 @@ class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface): """ self.__invalidate_expired_tokens() self.__invalidate_expired_objects() + + if not self.auth_enabled: + user = self.get_user_from_token(token) + if user == "": + self.token_cache[token] = (time.time(), user) # update to prevent timeout + return True + if self.token_cache.has_key(token): user = self.get_user_from_token(token) + if user == "": + # system token is only valid over Unix socket + return False self.token_cache[token] = (time.time(), user) # update to prevent timeout return True else: @@ -537,6 +556,8 @@ class CobblerReadWriteXMLRPCInterface(CobblerXMLRPCInterface): def check_access(self,token,resource,arg1=None,arg2=None): validated = self.__validate_token(token) + if not self.auth_enabled: + return True return self.__authorize(token,resource,arg1,arg2) diff --git a/cobbler/server/__init__.py b/cobbler/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cobbler/server/xmlrpcclient.py b/cobbler/server/xmlrpcclient.py new file mode 100644 index 0000000..cc0687c --- /dev/null +++ b/cobbler/server/xmlrpcclient.py @@ -0,0 +1,72 @@ +#============================================================================ +# This library is free software; you can redistribute it and/or +# modify it under the terms of version 2.1 of the GNU Lesser General Public +# License as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +#============================================================================ +# Copyright (C) 2006 Anthony Liguori +# Copyright (C) 2007 XenSource Inc. +# Copyright (C) 2007 Red Hat Inc, Michael DeHaan + +from httplib import FakeSocket, HTTPConnection, HTTP +import socket +import string +import xmlrpclib +from types import StringTypes + +class HTTPUnixConnection(HTTPConnection): + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(self.host) + +class HTTPUnix(HTTP): + _connection_class = HTTPUnixConnection + +class UnixTransport(xmlrpclib.Transport): + def request(self, host, handler, request_body, verbose=0): + self.__handler = handler + return xmlrpclib.Transport.request(self, host, '/RPC2', + request_body, verbose) + def make_connection(self, host): + return HTTPUnix(self.__handler) + +# See xmlrpclib2.TCPXMLRPCServer._marshalled_dispatch. +def conv_string(x): + if isinstance(x, StringTypes): + s = string.replace(x, "'", r"\047") + exec "s = '" + s + "'" + return s + else: + return x + + +class ServerProxy(xmlrpclib.ServerProxy): + def __init__(self, uri, transport=None, encoding=None, verbose=0, + allow_none=1): + if transport == None: + (protocol, rest) = uri.split(':', 1) + if protocol == 'httpu': + uri = 'http:' + rest + transport = UnixTransport() + else: + raise ValueError("only httpu://path/to/socket is supported") + xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding, + verbose, allow_none) + + + def __request(self, methodname, params): + response = xmlrpclib.ServerProxy.__request(self, methodname, params) + + if isinstance(response, tuple): + return tuple([conv_string(x) for x in response]) + else: + return conv_string(response) + diff --git a/cobbler/server/xmlrpclib2.py b/cobbler/server/xmlrpclib2.py new file mode 100644 index 0000000..d329241 --- /dev/null +++ b/cobbler/server/xmlrpclib2.py @@ -0,0 +1,236 @@ +#============================================================================ +# This library is free software; you can redistribute it and/or +# modify it under the terms of version 2.1 of the GNU Lesser General Public +# License as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +#============================================================================ +# Copyright (C) 2006 Anthony Liguori +# Copyright (C) 2006 XenSource Inc. +# Copyright (C) 2007 Red Hat Inc., Michael DeHaan +#============================================================================ + +""" +An enhanced XML-RPC client/server interface for Python. +""" + +import re +import fcntl +from types import * +import os +import errno +import traceback + +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +import SocketServer +import xmlrpclib, socket, os, stat + +#import mkdir + +# +# Convert all integers to strings as described in the Xen API +# + + +def stringify(value): + if isinstance(value, long) or \ + (isinstance(value, int) and not isinstance(value, bool)): + return str(value) + elif isinstance(value, dict): + new_value = {} + for k, v in value.items(): + new_value[stringify(k)] = stringify(v) + return new_value + elif isinstance(value, (tuple, list)): + return [stringify(v) for v in value] + else: + return value + + +# We're forced to subclass the RequestHandler class so that we can work around +# some bugs in Keep-Alive handling and also enabled it by default +class XMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + protocol_version = "HTTP/1.1" + + def __init__(self, request, client_address, server): + SimpleXMLRPCRequestHandler.__init__(self, request, client_address, + server) + + # this is inspired by SimpleXMLRPCRequestHandler's do_POST but differs + # in a few non-trivial ways + # 1) we never generate internal server errors. We let the exception + # propagate so that it shows up in the Xend debug logs + # 2) we don't bother checking for a _dispatch function since we don't + # use one + def do_POST(self): + addrport = self.client_address + #if not connection.hostAllowed(addrport, self.hosts_allowed): + # self.connection.shutdown(1) + # return + + data = self.rfile.read(int(self.headers["content-length"])) + rsp = self.server._marshaled_dispatch(data) + + self.send_response(200) + self.send_header("Content-Type", "text/xml") + self.send_header("Content-Length", str(len(rsp))) + self.end_headers() + + self.wfile.write(rsp) + self.wfile.flush() + #if self.close_connection == 1: + # self.connection.shutdown(1) + +def parents(dir, perms, enforcePermissions = False): + """ + Ensure that the given directory exists, creating it if necessary, but not + complaining if it's already there. + + @param dir The directory name. + @param perms One of the stat.S_ constants. + @param enforcePermissions Enforce our ownership and the given permissions, + even if the directory pre-existed with different ones. + """ + # Catch the exception here, rather than checking for the directory's + # existence first, to avoid races. + try: + os.makedirs(dir, perms) + except OSError, exn: + if exn.args[0] != errno.EEXIST or not os.path.isdir(dir): + raise + if enforcePermissions: + os.chown(dir, os.geteuid(), os.getegid()) + os.chmod(dir, stat.S_IRWXU) + + +# This is a base XML-RPC server for TCP. It sets allow_reuse_address to +# true, and has an improved marshaller that logs and serializes exceptions. + +class TCPXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer): + allow_reuse_address = True + + def __init__(self, addr, requestHandler=None, + logRequests = 1): + if requestHandler is None: + requestHandler = XMLRPCRequestHandler + SimpleXMLRPCServer.__init__(self, addr, + (lambda x, y, z: + requestHandler(x, y, z)), + logRequests) + + flags = fcntl.fcntl(self.fileno(), fcntl.F_GETFD) + flags |= fcntl.FD_CLOEXEC + fcntl.fcntl(self.fileno(), fcntl.F_SETFD, flags) + + def get_request(self): + (client, addr) = SimpleXMLRPCServer.get_request(self) + flags = fcntl.fcntl(client.fileno(), fcntl.F_GETFD) + flags |= fcntl.FD_CLOEXEC + fcntl.fcntl(client.fileno(), fcntl.F_SETFD, flags) + return (client, addr) + + def _marshaled_dispatch(self, data, dispatch_method = None): + params, method = xmlrpclib.loads(data) + if False: + # Enable this block of code to exit immediately without sending + # a response. This allows you to test client-side crash handling. + import sys + sys.exit(1) + try: + if dispatch_method is not None: + response = dispatch_method(method, params) + else: + response = self._dispatch(method, params) + + if (response is None or + not isinstance(response, dict) or + 'Status' not in response): + #log.exception('Internal error handling %s: Invalid result %s', + # method, response) + response = { "Status": "Failure", + "ErrorDescription": + ['INTERNAL_ERROR', + 'Invalid result %s handling %s' % + (response, method)]} + + # With either Unicode or normal strings, we can only transmit + # \t, \n, \r, \u0020-\ud7ff, \ue000-\ufffd, and \u10000-\u10ffff + # in an XML document. xmlrpclib does not escape these values + # properly, and then breaks when it comes to parse the document. + # To hack around this problem, we use repr here and exec above + # to transmit the string using Python encoding. + # Thanks to David Mertz for the trick (buried + # in xml_pickle.py). + if isinstance(response, StringTypes): + response = repr(response)[1:-1] + + response = (response,) + response = xmlrpclib.dumps(response, + methodresponse=1, + allow_none=1) + except Exception, exn: + try: + #if self.xenapi: + # if _is_not_supported(exn): + # errdesc = ['MESSAGE_METHOD_UNKNOWN', method] + # else: + # #log.exception('Internal error handling %s', method) + # errdesc = ['INTERNAL_ERROR', str(exn)] + # + # response = xmlrpclib.dumps( + # ({ "Status": "Failure", + # "ErrorDescription": errdesc },), + # methodresponse = 1) + #else: + # import xen.xend.XendClient + if isinstance(exn, xmlrpclib.Fault): + response = xmlrpclib.dumps(exn) + else: + # log.exception('Internal error handling %s', method) + response = xmlrpclib.dumps( + xmlrpclib.Fault(101, str(exn))) + except Exception, exn2: + # FIXME + traceback.print_exc() + + return response + + +notSupportedRE = re.compile(r'method "(.*)" is not supported') +def _is_not_supported(exn): + try: + m = notSupportedRE.search(exn[0]) + return m is not None + except: + return False + + +# This is a XML-RPC server that sits on a Unix domain socket. +# It implements proper support for allow_reuse_address by +# unlink()'ing an existing socket. + +class UnixXMLRPCRequestHandler(XMLRPCRequestHandler): + def address_string(self): + try: + return XMLRPCRequestHandler.address_string(self) + except ValueError, e: + return self.client_address[:2] + +class UnixXMLRPCServer(TCPXMLRPCServer): + address_family = socket.AF_UNIX + allow_address_reuse = True + + def __init__(self, addr, logRequests = 1): + parents(os.path.dirname(addr), stat.S_IRWXU, True) + if self.allow_reuse_address and os.path.exists(addr): + os.unlink(addr) + + TCPXMLRPCServer.__init__(self, addr, + UnixXMLRPCRequestHandler, logRequests) diff --git a/setup.py b/setup.py index eb2a23d..fddcba3 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ if __name__ == "__main__": "cobbler", "cobbler/yaml", "cobbler/modules", + "cobbler/server", "cobbler/webui", ], scripts = ["scripts/cobbler", "scripts/cobblerd", "scripts/cobbler_auth_help"], -- cgit