Added Python wrapper for pki pkcs12-import.

A Python wrapper module has been added for the pki pkcs12-import
command to provide a mechanism to implement a workaround for JSS
import limitation.

Additional fixes by cheimes have been merged into this patch:

setup.py:
We must track all sub-packages manually.

pylint-build-scan.py:
pylint confuses the 'pki' package with the 'pki' command. The
workaround symlinks the command and analysis the command under its
alternative name.

https://fedorahosted.org/pki/ticket/1742

7 files changed, 501 insertions, 97 deletions

diff --git a/base/common/python/pki/cli.py b/base/common/python/pki/cli/__init__.py
index 2c51056f8..2c51056f8 100644
--- a/base/common/python/pki/cli.py
+++ b/base/common/python/pki/cli/__init__.py
diff --git a/base/common/python/pki/cli/pkcs12.py b/base/common/python/pki/cli/pkcs12.py
new file mode 100644
index 000000000..c0bf9aff0
--- /dev/null
+++ b/base/common/python/pki/cli/pkcs12.py
@@ -0,0 +1,124 @@
+# Authors:
+#     Endi S. Dewata <edewata@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright (C) 2016 Red Hat, Inc.
+# All rights reserved.
+#
+
+from __future__ import absolute_import
+from __future__ import print_function
+import getopt
+import sys
+
+import pki.cli
+
+
+class PKCS12CLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(PKCS12CLI, self).__init__(
+            'pkcs12', 'PKCS #12 utilities')
+
+        self.add_module(PKCS12ImportCLI())
+
+
+class PKCS12ImportCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(PKCS12ImportCLI, self).__init__(
+            'import', 'Import PKCS #12 file into NSS database')
+
+    def print_help(self):
+        print('Usage: pki pkcs12-import [OPTIONS]')
+        print()
+        print('      --pkcs12                    PKCS #12 file containing certificates and keys.')
+        print('      --pkcs12-password           Password for the PKCS #12 file.')
+        print('      --pkcs12-password-file      File containing the PKCS #12 password.')
+        print('      --no-trust-flags            Do not include trust flags')
+        print('  -v, --verbose                   Run in verbose mode.')
+        print('      --debug                     Run in debug mode.')
+        print('      --help                      Show help message.')
+        print()
+
+    def execute(self, args):
+
+        try:
+            opts, _ = getopt.gnu_getopt(args, 'v', [
+                'pkcs12=', 'pkcs12-password=', 'pkcs12-password-file=',
+                'no-trust-flags', 'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        pkcs12_file = None
+        pkcs12_password = None
+        password_file = None
+        no_trust_flags = False
+
+        for o, a in opts:
+            if o == '--pkcs12':
+                pkcs12_file = a
+
+            elif o == '--pkcs12-password':
+                pkcs12_password = a
+
+            elif o == '--pkcs12-password-file':
+                password_file = a
+
+            elif o == '--no-trust-flags':
+                no_trust_flags = True
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        if not pkcs12_file:
+            print('ERROR: Missing PKCS #12 file')
+            self.print_help()
+            sys.exit(1)
+
+        if not pkcs12_password and not password_file:
+            print('ERROR: Missing PKCS #12 password')
+            self.print_help()
+            sys.exit(1)
+
+        main_cli = self.parent.parent
+
+        cmd = ['pkcs12-import']
+
+        if pkcs12_file:
+            cmd.extend(['--pkcs12', pkcs12_file])
+
+        if pkcs12_password:
+            cmd.extend(['--pkcs12-password', pkcs12_password])
+
+        if password_file:
+            cmd.extend(['--pkcs12-password-file', password_file])
+
+        if no_trust_flags:
+            cmd.extend(['--no-trust-flags'])
+
+        main_cli.execute_java(cmd)
diff --git a/base/common/python/setup.py b/base/common/python/setup.py
index 16c1d1760..2ab033784 100644
--- a/base/common/python/setup.py
+++ b/base/common/python/setup.py
@@ -84,7 +84,7 @@ and set up in less than an hour.""",
     license='GPL',
     keywords='pki x509 cert certificate',
     url='http://pki.fedoraproject.org/',
-    packages=['pki'],
+    packages=['pki', 'pki.cli'],
     requirements=['python-nss', 'requests', 'six'],
     classifiers=[
         'Development Status :: 5 - Production/Stable',
diff --git a/base/java-tools/bin/pki b/base/java-tools/bin/pki
index 5e7a7a438..e476cfcfe 100644
--- a/base/java-tools/bin/pki
+++ b/base/java-tools/bin/pki
@@ -19,111 +19,240 @@
 # All rights reserved.
 #
 
+from __future__ import absolute_import
+from __future__ import print_function
 import shlex
 import subprocess
 import sys
+import traceback
 
-def run_java_cli(args):
-
-    # read RESTEasy library path
-    value = subprocess.check_output(
-        '. /usr/share/pki/etc/pki.conf && . /etc/pki/pki.conf && echo $RESTEASY_LIB',
-        shell=True)
-    resteasy_lib = str(value).strip()
-
-    # read logging configuration path
-    value = subprocess.check_output(
-        '. /usr/share/pki/etc/pki.conf && . /etc/pki/pki.conf && echo $LOGGING_CONFIG',
-        shell=True)
-    logging_config = value.decode(sys.getfilesystemencoding()).strip()
-
-    # construct classpath
-    classpath = [
-        '/usr/share/java/commons-cli.jar',
-        '/usr/share/java/commons-codec.jar',
-        '/usr/share/java/commons-httpclient.jar',
-        '/usr/share/java/commons-io.jar',
-        '/usr/share/java/commons-lang.jar',
-        '/usr/share/java/commons-logging.jar',
-        '/usr/share/java/httpcomponents/httpclient.jar',
-        '/usr/share/java/httpcomponents/httpcore.jar',
-        '/usr/share/java/jackson/jackson-core-asl.jar',
-        '/usr/share/java/jackson/jackson-jaxrs.jar',
-        '/usr/share/java/jackson/jackson-mapper-asl.jar',
-        '/usr/share/java/jackson/jackson-mrbean.jar',
-        '/usr/share/java/jackson/jackson-smile.jar',
-        '/usr/share/java/jackson/jackson-xc.jar',
-        '/usr/share/java/jaxb-api.jar',
-        '/usr/share/java/ldapjdk.jar',
-        '/usr/share/java/servlet.jar',
-        resteasy_lib + '/jaxrs-api.jar',
-        resteasy_lib + '/resteasy-atom-provider.jar',
-        resteasy_lib + '/resteasy-client.jar',
-        resteasy_lib + '/resteasy-jaxb-provider.jar',
-        resteasy_lib + '/resteasy-jaxrs.jar',
-        resteasy_lib + '/resteasy-jaxrs-jandex.jar',
-        resteasy_lib + '/resteasy-jackson-provider.jar',
-        '/usr/share/java/pki/pki-nsutil.jar',
-        '/usr/share/java/pki/pki-cmsutil.jar',
-        '/usr/share/java/pki/pki-certsrv.jar',
-        '/usr/share/java/pki/pki-tools.jar',
-        '/usr/lib64/java/jss4.jar',
-        '/usr/lib/java/jss4.jar'
-    ]
-
-    command = [
-        'java',
-        '-cp',
-        ':'.join(classpath),
-        '-Djava.util.logging.config.file=' + logging_config,
-        'com.netscape.cmstools.cli.MainCLI'
-    ]
-
-    command.extend(args)
-
-    rv = subprocess.call(command)
-    exit(rv)
-
-
-# pylint: disable=W0613
-def run_python_cli(args):
-
-    raise Exception('Not implemented')
-
-
-def main(argv):
-
-    # read global options
-    value = subprocess.check_output(
-        '. /etc/pki/pki.conf && echo $PKI_CLI_OPTIONS',
-        shell=True)
-    args = shlex.split(value.strip())
-    args.extend(argv[1:])
-
-    client_type = 'java'
-
-    new_args = []
-
-    # read --client-type parameter and remove it from the argument list
-    i = 0
-    while i < len(args):
-        if args[i] == '--client-type':
-            client_type = args[i + 1]
+import pki.cli
+import pki.cli.pkcs12
+
+
+PYTHON_COMMANDS = ['pkcs12-import']
+
+
+class PKICLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(PKICLI, self).__init__(
+            'pki', 'PKI command-line interface')
+
+        self.database = None
+        self.password = None
+        self.password_file = None
+        self.token = None
+
+        self.add_module(pki.cli.pkcs12.PKCS12CLI())
+
+    def get_full_module_name(self, module_name):
+        return module_name
+
+    def print_help(self):
+        print('Usage: pki [OPTIONS]')
+        print()
+        print('      --client-type <type>     PKI client type (default: java)')
+        print('   -d <path>                   Client security database location ' +
+              '(default: ~/.dogtag/nssdb)')
+        print('   -c <password>               Client security database password ' +
+              '(mutually exclusive to the -C option)')
+        print('   -C <path>                   Client-side password file ' +
+              '(mutually exclusive to the -c option)')
+        print('      --token <name>           Security token name')
+        print()
+        print('  -v, --verbose                Run in verbose mode.')
+        print('      --debug                  Show debug messages.')
+        print('      --help                   Show help message.')
+        print()
+
+        super(PKICLI, self).print_help()
+
+    def execute_java(self, args, stdout=sys.stdout):
+
+        # read RESTEasy library path
+        value = subprocess.check_output(
+            '. /usr/share/pki/etc/pki.conf && . /etc/pki/pki.conf && echo $RESTEASY_LIB',
+            shell=True)
+        resteasy_lib = value.decode(sys.getfilesystemencoding()).strip()
+
+        # read logging configuration path
+        value = subprocess.check_output(
+            '. /usr/share/pki/etc/pki.conf && . /etc/pki/pki.conf && echo $LOGGING_CONFIG',
+            shell=True)
+        logging_config = value.decode(sys.getfilesystemencoding()).strip()
+
+        # construct classpath
+        classpath = [
+            '/usr/share/java/commons-cli.jar',
+            '/usr/share/java/commons-codec.jar',
+            '/usr/share/java/commons-httpclient.jar',
+            '/usr/share/java/commons-io.jar',
+            '/usr/share/java/commons-lang.jar',
+            '/usr/share/java/commons-logging.jar',
+            '/usr/share/java/httpcomponents/httpclient.jar',
+            '/usr/share/java/httpcomponents/httpcore.jar',
+            '/usr/share/java/jackson/jackson-core-asl.jar',
+            '/usr/share/java/jackson/jackson-jaxrs.jar',
+            '/usr/share/java/jackson/jackson-mapper-asl.jar',
+            '/usr/share/java/jackson/jackson-mrbean.jar',
+            '/usr/share/java/jackson/jackson-smile.jar',
+            '/usr/share/java/jackson/jackson-xc.jar',
+            '/usr/share/java/jaxb-api.jar',
+            '/usr/share/java/ldapjdk.jar',
+            '/usr/share/java/servlet.jar',
+            resteasy_lib + '/jaxrs-api.jar',
+            resteasy_lib + '/resteasy-atom-provider.jar',
+            resteasy_lib + '/resteasy-client.jar',
+            resteasy_lib + '/resteasy-jaxb-provider.jar',
+            resteasy_lib + '/resteasy-jaxrs.jar',
+            resteasy_lib + '/resteasy-jaxrs-jandex.jar',
+            resteasy_lib + '/resteasy-jackson-provider.jar',
+            '/usr/share/java/pki/pki-nsutil.jar',
+            '/usr/share/java/pki/pki-cmsutil.jar',
+            '/usr/share/java/pki/pki-certsrv.jar',
+            '/usr/share/java/pki/pki-tools.jar',
+            '/usr/lib64/java/jss4.jar',
+            '/usr/lib/java/jss4.jar'
+        ]
+
+        cmd = [
+            'java',
+            '-cp',
+            ':'.join(classpath),
+            '-Djava.util.logging.config.file=' + logging_config,
+            'com.netscape.cmstools.cli.MainCLI'
+        ]
+
+        # restore options for Java commands
+
+        if self.database:
+            cmd.extend(['-d', self.database])
+
+        if self.password:
+            cmd.extend(['-c', self.password])
+
+        if self.password_file:
+            cmd.extend(['-C', self.password_file])
+
+        if self.token and self.token != 'internal':
+            cmd.extend(['--token', self.token])
+
+        cmd.extend(args)
+
+        if self.verbose:
+            print('Java command: %s' % ' '.join(cmd))
+
+        subprocess.check_call(cmd, stdout=stdout)
+
+    def execute(self, argv):
+
+        # append global options
+        value = subprocess.check_output(
+            '. /usr/share/pki/etc/pki.conf && . /etc/pki/pki.conf && echo $PKI_CLI_OPTIONS',
+            shell=True)
+        value = value.decode(sys.getfilesystemencoding()).strip()
+        args = shlex.split(value)
+        args.extend(argv[1:])
+
+        client_type = 'java'
+
+        pki_options = []
+        command = None
+        cmd_args = []
+
+        # read pki options before the command
+        # remove options for Python module
+
+        i = 0
+        while i < len(args):
+            # if arg is a command, stop
+            if args[i][0] != '-':
+                command = args[i]
+                break
+
+            # get database path
+            if args[i] == '-d':
+                self.database = args[i + 1]
+                pki_options.append(args[i])
+                pki_options.append(args[i + 1])
+                i = i + 2
+
+            # get database password
+            elif args[i] == '-c':
+                self.password = args[i + 1]
+                pki_options.append(args[i])
+                pki_options.append(args[i + 1])
+                i = i + 2
+
+            # get database password file path
+            elif args[i] == '-C':
+                self.password_file = args[i + 1]
+                pki_options.append(args[i])
+                pki_options.append(args[i + 1])
+                i = i + 2
+
+            # get token name
+            elif args[i] == '--token':
+                self.token = args[i + 1]
+                pki_options.append(args[i])
+                pki_options.append(args[i + 1])
+                i = i + 2
+
+            # check verbose option
+            elif args[i] == '-v' or args[i] == '--verbose':
+                self.set_verbose(True)
+                pki_options.append(args[i])
+                i = i + 1
+
+            # check debug option
+            elif args[i] == '--debug':
+                self.set_verbose(True)
+                self.set_debug(True)
+                pki_options.append(args[i])
+                i = i + 1
+
+            # get client type
+            elif args[i] == '--client-type':
+                client_type = args[i + 1]
+                pki_options.append(args[i])
+                pki_options.append(args[i + 1])
+                i = i + 2
+
+            else:  # otherwise, save the arg for the next module
+                cmd_args.append(args[i])
+                i = i + 1
+
+        # save the rest of the args
+        while i < len(args):
+            cmd_args.append(args[i])
             i = i + 1
 
-        else:
-            new_args.append(args[i])
+        if self.verbose:
+            print('PKI options: %s' % ' '.join(pki_options))
+            print('PKI command: %s %s' % (command, ' '.join(cmd_args)))
 
-        i = i + 1
+        if client_type == 'python' or command in PYTHON_COMMANDS:
+            (module, module_args) = self.parse_args(cmd_args)
+            module.execute(module_args)
 
-    if client_type == 'java':
-        run_java_cli(new_args)
+        elif client_type == 'java':
+            self.execute_java(cmd_args)
 
-    elif client_type == 'python':
-        run_python_cli(new_args)
+        else:
+            raise Exception('Unsupported client type: ' + client_type)
 
-    else:
-        raise Exception('Unsupported client type: ' + client_type)
 
 if __name__ == '__main__':
-    main(sys.argv)
+
+    cli = PKICLI()
+
+    try:
+        cli.execute(sys.argv)
+
+    except subprocess.CalledProcessError as e:
+        if cli.verbose:
+            print('ERROR: %s' % e)
+        elif cli.debug:
+            traceback.print_exc()
+        exit(e.returncode)
diff --git a/scripts/pylint-build-scan.py b/scripts/pylint-build-scan.py
new file mode 100755
index 000000000..d4156e87b
--- /dev/null
+++ b/scripts/pylint-build-scan.py
@@ -0,0 +1,146 @@
+#!/usr/bin/python
+
+# Authors:
+#     Christian Heimes <cheimes@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright (C) 2015 Red Hat, Inc.
+# All rights reserved.
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import os
+import pprint
+import re
+import subprocess
+import sys
+
+from distutils.sysconfig import get_python_lib  # pylint: disable=F0401
+
+
+SCRIPTPATH = os.path.dirname(os.path.abspath(__file__))
+PYLINTRC = os.path.join(SCRIPTPATH, 'dogtag.pylintrc')
+FILENAMES = [
+    os.path.abspath(__file__),
+    '{sitepackages}/pki',
+    '{bin}/pki-cmd',  # see HACK
+    '{sbin}/pkispawn',
+    '{sbin}/pkidestroy',
+    '{sbin}/pki-upgrade',
+    '{sbin}/pki-server',
+    '{sbin}/pki-server-upgrade',
+]
+UPGRADE_SCRIPT = re.compile('^[0-9]+-.*')
+
+
+def tox_env(args):
+    """Paths for tox environment"""
+    prefix = args.prefix
+    env = {
+        'bin': os.path.join(prefix, 'bin'),
+        'sbin': os.path.join(prefix, 'bin'),
+        'sharepki': os.path.join(prefix, 'share', 'pki'),
+        'sitepackages': get_python_lib()
+    }
+    return env
+
+
+def rpm_env(args):
+    """Paths for RPM build environment"""
+    prefix = args.prefix
+    relative = get_python_lib().lstrip(os.sep)
+    env = {
+        'bin': os.path.join(prefix, 'usr', 'bin'),
+        'sbin': os.path.join(prefix, 'usr', 'sbin'),
+        'sharepki': os.path.join(prefix, 'usr', 'share', 'pki'),
+        'sitepackages': os.path.join(prefix, relative),
+    }
+    return env
+
+
+def find_upgrades(root):
+    """Find upgrade scripts"""
+    for dirpath, _, filenames in os.walk(root):
+        for filename in filenames:
+            if UPGRADE_SCRIPT.match(filename):
+                yield os.path.join(dirpath, filename)
+
+
+def main():
+    """Dogtag pylint script"""
+    parser = argparse.ArgumentParser(
+        description=main.__doc__,
+        epilog="Additional arguments can be passed to pylint with: "
+               "'-- --arg1 --arg2 ...'",
+    )
+    parser.add_argument('--verbose', action='store_true')
+    subparsers = parser.add_subparsers(dest='command')
+    subparsers.required = True
+
+    toxparser = subparsers.add_parser('tox', help='tox in-tree tests')
+    toxparser.add_argument('--prefix', dest='prefix', default=sys.prefix)
+    toxparser.add_argument('pylint_args', nargs=argparse.REMAINDER)
+    toxparser.set_defaults(get_env=tox_env)
+
+    rpmparser = subparsers.add_parser('rpm', help='RPM source tree tests')
+    rpmparser.add_argument('--prefix', dest='prefix', required=True)
+    rpmparser.add_argument('pylint_args', nargs=argparse.REMAINDER)
+    rpmparser.set_defaults(get_env=rpm_env)
+
+    args = parser.parse_args()
+    env = args.get_env(args)
+    if args.verbose:
+        pprint.pprint(env)
+    # sanity check
+    for key, path in env.items():
+        if not os.path.exists(path):
+            raise RuntimeError('{} ({}) does not exist'.format(key, path))
+
+    if args.pylint_args and args.pylint_args[0] == '--':
+        extra_args = args.pylint_args[1:]
+    else:
+        extra_args = args.pylint_args
+
+    if not os.path.isfile(PYLINTRC):
+        raise IOError('{} not found'.format(PYLINTRC))
+
+    pylint = [
+        'pylint',
+        '--rcfile={}'.format(PYLINTRC)
+    ]
+    pylint.extend(extra_args)
+    pylint.extend(filename.format(**env) for filename in FILENAMES)
+    pylint.extend(find_upgrades('{sharepki}/upgrade'.format(**env)))
+    pylint.extend(find_upgrades('{sharepki}/server/upgrade'.format(**env)))
+    if args.verbose:
+        pprint.pprint(pylint)
+
+    # HACK:
+    # pylint confuses the pki command with the pki package. We create a
+    # symlink from bin/pki to bin/pki-cmd and test bin/pki-cmd instead.
+    pki_bin = '{bin}/pki'.format(**env)
+    pki_cmd = '{bin}/pki-cmd'.format(**env)
+    os.symlink(pki_bin, pki_cmd)
+
+    try:
+        return subprocess.call(pylint, cwd=env['sitepackages'])
+    finally:
+        os.unlink(pki_cmd)
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/setup.py b/setup.py
index 0d760c0ee..e14c7f8d4 100644
--- a/setup.py
+++ b/setup.py
@@ -55,6 +55,7 @@ setup(
     },
     packages=[
         'pki',
+        'pki.cli',
         'pki.server',
         'pki.server.cli',
         'pki.server.deployment',
diff --git a/specs/pki-core.spec b/specs/pki-core.spec
index cf02f67e2..0fb8e8407 100644
--- a/specs/pki-core.spec
+++ b/specs/pki-core.spec
@@ -860,6 +860,10 @@ systemctl daemon-reload
 %{python_sitelib}/pki/*.py
 %{python_sitelib}/pki/*.pyc
 %{python_sitelib}/pki/*.pyo
+%dir %{python_sitelib}/pki/cli
+%{python_sitelib}/pki/cli/*.py
+%{python_sitelib}/pki/cli/*.pyc
+%{python_sitelib}/pki/cli/*.pyo
 %dir %{_localstatedir}/log/pki
 %{_sbindir}/pki-upgrade
 %{_mandir}/man8/pki-upgrade.8.gz