New lite-server implementation

The new development server depends on werkzeug instead of paste. The
werkzeug WSGI server comes with some additional features, most
noticeable multi-processing server. The IPA framework is not compatible
with threaded servers. Werkzeug can serve static files easily and has a
fast auto-reloader.

The new lite-server implementation depends on PR 314 (privilege
separation). For Python 3 support, it additionally depends on PR 393.

Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Martin Basti <mbasti@redhat.com>

5 files changed, 261 insertions, 162 deletions

diff --git a/BUILD.txt b/BUILD.txt
index 620adc31e..10b19430a 100644
--- a/BUILD.txt
+++ b/BUILD.txt
@@ -41,7 +41,7 @@ install the rpms and then configure IPA using ipa-server-install.
 Get a TGT for the admin user with: kinit admin
 
 Next you'll need 2 sessions in the source tree. In the first session run
-python lite-server.py. In the second session copy /etc/ipa/default.conf into
+```make lite-server```. In the second session copy /etc/ipa/default.conf into
 ~/.ipa/default.conf and replace xmlrpc_uri with http://127.0.0.1:8888/ipa/xml.
 Finally run the ./ipa tool and it will make requests to the lite-server
 listening on 127.0.0.1:8888.
diff --git a/Makefile.am b/Makefile.am
index 9bfc899fe..30ad9bb55 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -6,7 +6,6 @@ SUBDIRS = asn1 util client contrib daemons init install $(IPACLIENT_SUBDIRS) ipa
 MOSTLYCLEANFILES = ipasetup.pyc ipasetup.pyo \
 		   ignore_import_errors.pyc ignore_import_errors.pyo \
 		   ipasetup.pyc ipasetup.pyo \
-		   lite-server.pyc lite-server.pyo \
 		   pylint_plugins.pyc pylint_plugins.pyo
 
 # user-facing scripts
@@ -14,7 +13,6 @@ dist_bin_SCRIPTS = ipa
 
 # files required for build but not installed
 dist_noinst_SCRIPTS = ignore_import_errors.py \
-		      lite-server.py \
 		      makeapi \
 		      makeaci \
 		      make-doc \
@@ -119,6 +117,12 @@ _srpms-body: _rpms-prep
 	cp $(RPMBUILD)/SRPMS/*$$(cat $(top_builddir)/.version)*.src.rpm $(top_builddir)/dist/srpms/
 	rm -f rm -f $(top_builddir)/.version
 
+.PHONY: lite-server
+lite-server: $(top_builddir)/ipapython/version.py
+	+$(MAKE) -C $(top_builddir)/install/ui
+	PYTHONPATH=$(top_srcdir) $(PYTHON) -bb \
+	    contrib/lite-server.py $(LITESERVER_ARGS)
+
 .PHONY: lint
 if WITH_POLINT
 POLINT_TARGET = polint
diff --git a/contrib/Makefile.am b/contrib/Makefile.am
index 108a8087d..b28f2e77b 100644
--- a/contrib/Makefile.am
+++ b/contrib/Makefile.am
@@ -1,4 +1,5 @@
 SUBDIRS = completion
 
 EXTRA_DIST = \
-	nssciphersuite
+	nssciphersuite \
+	lite-server.py
diff --git a/contrib/lite-server.py b/contrib/lite-server.py
new file mode 100755
index 000000000..1df5004e2
--- /dev/null
+++ b/contrib/lite-server.py
@@ -0,0 +1,252 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017 FreeIPA Contributors see COPYING for license
+#
+"""In-tree development server
+
+The dev server requires a Kerberos TGT and a file based credential cache:
+
+    $ mkdir -p ~/.ipa
+    $ export KRB5CCNAME=~/.ipa/ccache
+    $ kinit admin
+    $ make lite-server
+
+Optionally you can set KRB5_CONFIG to use a custom Kerberos configuration
+instead of /etc/krb5.conf.
+
+To run the lite-server with another Python interpreter:
+
+    $ make lite-server PYTHON=/path/to/bin/python
+
+To enable profiling:
+
+    $ make lite-server LITESERVER_ARGS='--enable-profiler=-'
+
+By default the dev server supports HTTP only. To switch to HTTPS, you can put
+a PEM file at ~/.ipa/lite.pem. The PEM file must contain a server certificate,
+its unencrypted private key and intermediate chain certs (if applicable).
+
+Prerequisite
+------------
+
+Additionally to build and runtime requirements of FreeIPA, the dev server
+depends on the werkzeug framework and optionally watchdog for auto-reloading.
+You may also have to enable a development COPR.
+
+    $ sudo dnf install -y dnf-plugins-core
+    $ sudo dnf builddep --spec freeipa.spec.in
+    $ sudo dnf install -y python-werkzeug python2-watchdog \
+        python3-werkzeug python3-watchdog
+    $ ./autogen.sh
+
+For more information see
+
+  * http://www.freeipa.org/page/Build
+  * http://www.freeipa.org/page/Testing
+
+"""
+from __future__ import print_function
+
+import os
+import optparse  # pylint: disable=deprecated-module
+import ssl
+import sys
+import warnings
+
+import ipalib
+from ipalib import api
+from ipalib.krb_utils import krb5_parse_ccache
+from ipalib.krb_utils import krb5_unparse_ccache
+
+# pylint: disable=import-error
+from werkzeug.contrib.profiler import ProfilerMiddleware
+from werkzeug.exceptions import NotFound
+from werkzeug.serving import run_simple
+from werkzeug.utils import redirect, append_slash_redirect
+from werkzeug.wsgi import DispatcherMiddleware, SharedDataMiddleware
+# pylint: enable=import-error
+
+
+BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+IMPORTDIR = os.path.dirname(os.path.dirname(os.path.abspath(ipalib.__file__)))
+
+if BASEDIR != IMPORTDIR:
+    warnings.warn(
+        "ipalib was imported from '{}' instead of '{}'!".format(
+            IMPORTDIR, BASEDIR),
+        RuntimeWarning
+    )
+
+STATIC_FILES = {
+    '/ipa/ui': os.path.join(BASEDIR, 'install/ui'),
+    '/ipa/ui/js': os.path.join(BASEDIR, 'install/ui/src'),
+    '/ipa/ui/js/dojo': os.path.join(BASEDIR, 'install/ui/build/dojo'),
+    '/ipa/ui/fonts': '/usr/share/fonts',
+}
+
+
+def get_ccname():
+    """Retrieve and validate Kerberos credential cache
+
+    Only FILE schema is supported.
+    """
+    ccname = os.environ.get('KRB5CCNAME')
+    if ccname is None:
+        raise ValueError("KRB5CCNAME env var is not set.")
+    scheme, location = krb5_parse_ccache(ccname)
+    if scheme != 'FILE':  # MEMORY makes no sense
+        raise ValueError("Unsupported KRB5CCNAME scheme {}".format(scheme))
+    if not os.path.isfile(location):
+        raise ValueError("KRB5CCNAME file '{}' does not exit".format(location))
+    return krb5_unparse_ccache(scheme, location)
+
+
+class KRBCheater(object):
+    """Add KRB5CCNAME to WSGI environ
+    """
+    def __init__(self, app, ccname):
+        self.app = app
+        self.ccname = ccname
+
+    def __call__(self, environ, start_response):
+        environ['KRB5CCNAME'] = self.ccname
+        return self.app(environ, start_response)
+
+
+class StaticFilesMiddleware(SharedDataMiddleware):
+    def get_directory_loader(self, directory):
+        # override directory loader to support index.html
+        def loader(path):
+            if path is not None:
+                path = os.path.join(directory, path)
+            else:
+                path = directory
+            # use index.html for directory views
+            if os.path.isdir(path):
+                path = os.path.join(path, 'index.html')
+            if os.path.isfile(path):
+                return os.path.basename(path), self._opener(path)
+            return None, None
+        return loader
+
+
+def init_api():
+    """Initialize FreeIPA API from command line
+    """
+    parser = optparse.OptionParser()
+
+    parser.add_option(
+        '--dev',
+        help='Run WebUI in development mode',
+        default=True,
+        action='store_false',
+        dest='prod',
+    )
+    parser.add_option(
+        '--host',
+        help='Listen on address HOST (default 127.0.0.1)',
+        default='127.0.0.1',
+    )
+    parser.add_option(
+        '--port',
+        help='Listen on PORT (default 8888)',
+        default=8888,
+        type='int',
+    )
+    parser.add_option(
+        '--enable-profiler',
+        help="Path to WSGI profiler directory or '-' for stderr",
+        default=None,
+        type='str',
+    )
+
+    api.env.in_server = True
+    api.env.startup_traceback = True
+    # workaround for RefererError in rpcserver
+    api.env.in_tree = True
+    # workaround: AttributeError: locked: cannot set ldap2.time_limit to None
+    api.env.mode = 'production'
+
+    # pylint: disable=unused-variable
+    options, args = api.bootstrap_with_global_options(parser, context='lite')
+    api.env._merge(
+        lite_port=options.port,
+        lite_host=options.host,
+        webui_prod=options.prod,
+        lite_profiler=options.enable_profiler,
+        lite_pem=api.env._join('dot_ipa', 'lite.pem'),
+    )
+    api.finalize()
+
+
+def redirect_ui(app):
+    """Redirects for UI
+    """
+    def wsgi(environ, start_response):
+        path_info = environ['PATH_INFO']
+        if path_info in {'/', '/ipa', '/ipa/'}:
+            response = redirect('/ipa/ui/')
+            return response(environ, start_response)
+        # Redirect to append slash to some routes
+        if path_info in {'/ipa/ui', '/ipa/ui/test'}:
+            response = append_slash_redirect(environ)
+            return response(environ, start_response)
+        if path_info == '/favicon.ico':
+            response = redirect('/ipa/ui/favicon.ico')
+            return response(environ, start_response)
+        return app(environ, start_response)
+    return wsgi
+
+
+def main():
+    try:
+        ccname = get_ccname()
+    except ValueError as e:
+        print("ERROR:", e, file=sys.stderr)
+        print("\nliteserver requires a KRB5CCNAME env var and "
+              "a valid Kerberos TGT:\n", file=sys.stderr)
+        print("    export KRB5CCNAME=~/.ipa/ccache", file=sys.stderr)
+        print("    kinit\n", file=sys.stderr)
+        sys.exit(1)
+
+    init_api()
+
+    if os.path.isfile(api.env.lite_pem):
+        ctx = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
+        ctx.load_cert_chain(api.env.lite_pem)
+    else:
+        ctx = None
+
+    app = NotFound()
+    app = DispatcherMiddleware(app, {
+        '/ipa': KRBCheater(api.Backend.wsgi_dispatch, ccname),
+    })
+
+    # only profile api calls
+    if api.env.lite_profiler == '-':
+        print('Profiler enable, stats are written to stderr.')
+        app = ProfilerMiddleware(app, stream=sys.stderr, restrictions=(30,))
+    elif api.env.lite_profiler:
+        profile_dir = os.path.abspath(api.env.lite_profiler)
+        print("Profiler enable, profiles are stored in '{}'.".format(
+            profile_dir
+        ))
+        app = ProfilerMiddleware(app, profile_dir=profile_dir)
+
+    app = StaticFilesMiddleware(app, STATIC_FILES)
+    app = redirect_ui(app)
+
+    run_simple(
+        hostname=api.env.lite_host,
+        port=api.env.lite_port,
+        application=app,
+        processes=5,
+        ssl_context=ctx,
+        use_reloader=True,
+        # debugger doesn't work because framework catches all exceptions
+        # use_debugger=not api.env.webui_prod,
+        # use_evalex=not api.env.webui_prod,
+    )
+
+if __name__ == '__main__':
+    main()
diff --git a/lite-server.py b/lite-server.py
deleted file mode 100755
index cd4f09cbb..000000000
--- a/lite-server.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/usr/bin/python2
-
-# Authors:
-#   Jason Gerard DeRose <jderose@redhat.com>
-#
-# Copyright (C) 2008  Red Hat
-# see file 'COPYING' for use and warranty information
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""
-In-tree paste-based test server.
-
-This uses the *Python Paste* WSGI server.  For more info, see:
-
-    http://pythonpaste.org/
-
-Unfortunately, SSL support is broken under Python 2.6 with paste 1.7.2, see:
-
-    http://trac.pythonpaste.org/pythonpaste/ticket/314
-"""
-
-from os import path, getcwd
-import optparse  # pylint: disable=deprecated-module
-from paste import httpserver
-import paste.gzipper
-from paste.urlmap import URLMap
-from ipalib import api
-from subprocess import check_output, CalledProcessError
-import re
-
-# Ugly hack for test purposes only. GSSAPI has no way to get default ccache
-# name, but we don't need it outside test server
-def get_default_ccache_name():
-    try:
-        out = check_output(['klist'])
-    except CalledProcessError:
-        raise RuntimeError("Default ccache not found. Did you kinit?")
-    match = re.match(r'^Ticket cache:\s*(\S+)', out)
-    if not match:
-        raise RuntimeError("Cannot obtain ccache name")
-    return match.group(1)
-
-
-class KRBCheater(object):
-    def __init__(self, app):
-        self.app = app
-        self.url = app.url
-        self.ccname = get_default_ccache_name()
-
-    def __call__(self, environ, start_response):
-        environ['KRB5CCNAME'] = self.ccname
-        return self.app(environ, start_response)
-
-
-class WebUIApp(object):
-    INDEX_FILE = 'index.html'
-    EXTENSION_TO_MIME_MAP = {
-        'xhtml': 'text/html',
-        'html': 'text/html',
-        'js': 'text/javascript',
-        'inc': 'text/html',
-        'css': 'text/css',
-        'png': 'image/png',
-        'json': 'text/javascript',
-    }
-
-    def __init__(self):
-        self.url = '/ipa/ui'
-
-    def __call__(self, environ, start_response):
-        path_info = environ['PATH_INFO'].lstrip('/')
-        if path_info == '':
-            path_info = self.INDEX_FILE
-        requested_file = path.join(getcwd(), 'install/ui/', path_info)
-        extension = requested_file.rsplit('.', 1)[-1]
-
-        if extension not in self.EXTENSION_TO_MIME_MAP:
-            start_response('404 Not Found', [('Content-Type', 'text/plain')])
-            return ['NOT FOUND']
-        mime_type = self.EXTENSION_TO_MIME_MAP[extension]
-
-        f = None
-        try:
-            f = open(requested_file, 'r')
-            api.log.info('Request file %s' % requested_file)
-            start_response('200 OK', [('Content-Type', mime_type)])
-            return [f.read()]
-        except IOError:
-            start_response('404 Not Found', [('Content-Type', 'text/plain')])
-            return ['NOT FOUND']
-        finally:
-            if f is not None:
-                f.close()
-            api.log.info('Request done')
-
-
-if __name__ == '__main__':
-    parser = optparse.OptionParser()
-
-    parser.add_option('--dev',
-        help='Run WebUI in development mode (requires FireBug)',
-        default=True,
-        action='store_false',
-        dest='prod',
-    )
-    parser.add_option('--host',
-        help='Listen on address HOST (default 127.0.0.1)',
-        default='127.0.0.1',
-    )
-    parser.add_option('--port',
-        help='Listen on PORT (default 8888)',
-        default=8888,
-        type='int',
-    )
-
-    api.env.in_server = True
-    api.env.startup_traceback = True
-    (options, args) = api.bootstrap_with_global_options(parser, context='lite')
-    api.env._merge(
-        lite_port=options.port,
-        lite_host=options.host,
-        webui_prod=options.prod,
-        lite_pem=api.env._join('dot_ipa', 'lite.pem'),
-    )
-    api.finalize()
-
-    urlmap = URLMap()
-    apps = [
-        ('IPA', KRBCheater(api.Backend.wsgi_dispatch)),
-        ('webUI', KRBCheater(WebUIApp())),
-    ]
-    for (name, app) in apps:
-        urlmap[app.url] = app
-        api.log.info('Mounting %s at %s', name, app.url)
-
-    if path.isfile(api.env.lite_pem):
-        pem = api.env.lite_pem
-    else:
-        api.log.info('To enable SSL, place PEM file at %r', api.env.lite_pem)
-        pem = None
-
-    httpserver.serve(paste.gzipper.middleware(urlmap),
-        host=api.env.lite_host,
-        port=api.env.lite_port,
-        ssl_pem=pem,
-    )