Added TPS VLV management CLI.

A set of pki-server commands has been added to simplify upgrading
TPS VLV indexes.

https://fedorahosted.org/pki/ticket/2354
https://fedorahosted.org/pki/ticket/2263
https://fedorahosted.org/pki/ticket/2269

2 files changed, 506 insertions, 0 deletions

diff --git a/base/server/python/pki/server/__init__.py b/base/server/python/pki/server/__init__.py
index 8347311cf..bf705fd35 100644
--- a/base/server/python/pki/server/__init__.py
+++ b/base/server/python/pki/server/__init__.py
@@ -35,6 +35,7 @@ import tempfile
 
 import pki
 import pki.nssdb
+import pki.util
 
 INSTANCE_BASE_DIR = '/var/lib/pki'
 REGISTRY_DIR = '/etc/sysconfig/pki'
@@ -370,6 +371,15 @@ class PKISubsystem(object):
 
         return connection
 
+    def customize_file(self, input_file, output_file):
+        params = {
+            '{instanceId}': self.instance.name,
+            '{database}': self.config['internaldb.database'],
+            '{rootSuffix}': self.config['internaldb.basedn']
+        }
+
+        pki.util.customize_file(input_file, output_file, params)
+
     def __repr__(self):
         return str(self.instance) + '/' + self.name
 
diff --git a/base/server/python/pki/server/cli/tps.py b/base/server/python/pki/server/cli/tps.py
index f40223ddb..63da3414e 100644
--- a/base/server/python/pki/server/cli/tps.py
+++ b/base/server/python/pki/server/cli/tps.py
@@ -22,14 +22,22 @@ from __future__ import absolute_import
 from __future__ import print_function
 import getopt
 import io
+import ldap
+import ldap.modlist
+import ldif
 import os
 import shutil
 import sys
 import tempfile
+import time
 
 import pki.cli
 
 
+TPS_VLV_PATH = '/usr/share/pki/tps/conf/vlv.ldif'
+TPS_VLV_TASKS_PATH = '/usr/share/pki/tps/conf/vlvtasks.ldif'
+
+
 class TPSCLI(pki.cli.CLI):
 
     def __init__(self):
@@ -37,6 +45,7 @@ class TPSCLI(pki.cli.CLI):
             'tps', 'TPS management commands')
 
         self.add_module(TPSCloneCLI())
+        self.add_module(TPSDBCLI())
 
 
 class TPSCloneCLI(pki.cli.CLI):
@@ -139,3 +148,490 @@ class TPSClonePrepareCLI(pki.cli.CLI):
 
         finally:
             shutil.rmtree(tmpdir)
+
+
+class TPSDBCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(TPSDBCLI, self).__init__(
+            'db', 'TPS database management commands')
+
+        self.add_module(TPSDBVLVCLI())
+
+
+class TPSDBVLVCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(TPSDBVLVCLI, self).__init__(
+            'vlv', 'TPS VLV management commands')
+
+        self.add_module(TPSDBVLVFindCLI())
+        self.add_module(TPSDBVLVAddCLI())
+        self.add_module(TPSDBVLVDeleteCLI())
+        self.add_module(TPSDBVLVReindexCLI())
+
+
+class TPSDBVLVFindCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(TPSDBVLVFindCLI, self).__init__(
+            'find', 'Find TPS VLVs')
+
+    def print_help(self):
+        print('Usage: pki-server tps-db-vlv-find [OPTIONS]')
+        print()
+        print('  -i, --instance <instance ID>       Instance ID (default: pki-tomcat).')
+        print('  -D, --bind-dn <Bind DN>            Connect DN (default: cn=Directory Manager).')
+        print('  -w, --bind-password <password>     Password to connect to database.')
+        print('  -v, --verbose                      Run in verbose mode.')
+        print('      --help                         Show help message.')
+        print()
+
+    def execute(self, args):
+        try:
+            opts, _ = getopt.gnu_getopt(
+                args,
+                'i:D:w:x:g:v',
+                ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=',
+                 'verbose', 'help']
+            )
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        bind_dn = None
+        bind_password = None
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-D', '--bind-dn'):
+                bind_dn = a
+
+            elif o in ('-w', '--bind-password'):
+                bind_password = a
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.PKIInstance(instance_name)
+        instance.load()
+
+        subsystem = instance.get_subsystem('tps')
+
+        if not subsystem:
+            raise Exception('Subsystem not found')
+
+        self.find_vlv(subsystem, bind_dn, bind_password)
+
+    def find_vlv(self, subsystem, bind_dn, bind_password):
+
+        conn = subsystem.open_database(bind_dn=bind_dn,
+                                       bind_password=bind_password)
+
+        try:
+            database = subsystem.config['internaldb.database']
+            base_dn = 'cn=' + database + ',cn=ldbm database, cn=plugins, cn=config'
+
+            if self.verbose:
+                print('Searching %s' % base_dn)
+
+            entries = conn.ldap.search_s(
+                base_dn,
+                ldap.SCOPE_SUBTREE,
+                '(|(objectClass=vlvSearch)(objectClass=vlvIndex))')
+
+            self.print_message('%d entries found' % len(entries))
+
+            if not entries:
+                return
+
+            first = True
+            for entry in entries:
+                dn = entry[0]
+                attrs = entry[1]
+
+                if first:
+                    first = False
+                else:
+                    print()
+
+                print('  dn: %s' % dn)
+                for key, values in attrs.items():
+                    for value in values:
+                        print('  %s: %s' % (key, value))
+
+        finally:
+            conn.close()
+
+
+class TPSDBVLVAddCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(TPSDBVLVAddCLI, self).__init__(
+            'add', 'Add TPS VLVs')
+
+    def print_help(self):
+        print('Usage: pki-server tps-db-vlv-add [OPTIONS]')
+        print()
+        print('  -i, --instance <instance ID>       Instance ID (default: pki-tomcat).')
+        print('  -D, --bind-dn <Bind DN>            Connect DN (default: cn=Directory Manager).')
+        print('  -w, --bind-password <password>     Password to connect to database.')
+        print('  -g, --generate-ldif <outfile>      Generate LDIF of required changes.')
+        print('  -v, --verbose                      Run in verbose mode.')
+        print('      --help                         Show help message.')
+        print()
+
+    def execute(self, args):
+        try:
+            opts, _ = getopt.gnu_getopt(
+                args,
+                'i:D:w:x:g:v',
+                ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=',
+                 'verbose', 'help']
+            )
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        bind_dn = 'cn=Directory Manager'
+        bind_password = None
+        out_file = None
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-D', '--bind-dn'):
+                bind_dn = a
+
+            elif o in ('-w', '--bind-password'):
+                bind_password = a
+
+            elif o in ('-g', '--generate-ldif'):
+                out_file = a
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.PKIInstance(instance_name)
+        instance.load()
+
+        subsystem = instance.get_subsystem('tps')
+
+        if not subsystem:
+            raise Exception('Subsystem not found')
+
+        if out_file:
+            self.generate_ldif(subsystem, out_file)
+            return
+
+        self.add_vlv(subsystem, bind_dn, bind_password)
+
+    def generate_ldif(self, subsystem, out_file):
+        subsystem.customize_file(TPS_VLV_PATH, out_file)
+        self.print_message('Output: %s' % out_file)
+
+    def add_vlv(self, subsystem, bind_dn, bind_password):
+
+        input_file = tempfile.NamedTemporaryFile(delete=False)
+
+        try:
+            subsystem.customize_file(TPS_VLV_PATH, input_file.name)
+
+            conn = subsystem.open_database(bind_dn=bind_dn,
+                                           bind_password=bind_password)
+
+            try:
+                parser = ldif.LDIFRecordList(open(input_file.name, 'rb'))
+                parser.parse()
+
+                for dn, entry in parser.all_records:
+
+                    if self.verbose:
+                        print('Adding %s' % dn)
+
+                    add_modlist = ldap.modlist.addModlist(entry)
+                    conn.ldap.add_s(dn, add_modlist)
+
+            finally:
+                conn.close()
+
+        finally:
+            os.unlink(input_file.name)
+
+        self.print_message('VLVs added')
+
+
+class TPSDBVLVDeleteCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(TPSDBVLVDeleteCLI, self).__init__(
+            'del', 'Delete TPS VLVs')
+
+    def print_help(self):
+        print('Usage: pki-server tps-db-vlv-del [OPTIONS]')
+        print()
+        print('  -i, --instance <instance ID>       Instance ID (default: pki-tomcat).')
+        print('  -D, --bind-dn <Bind DN>            Connect DN (default: cn=Directory Manager).')
+        print('  -w, --bind-password <password>     Password to connect to DB.')
+        print('  -g, --generate-ldif <outfile>      Generate LDIF of required changes.')
+        print('  -v, --verbose                      Run in verbose mode.')
+        print('      --help                         Show help message.')
+        print()
+
+    def execute(self, args):
+        try:
+            opts, _ = getopt.gnu_getopt(
+                args,
+                'i:D:w:x:g:v',
+                ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=',
+                 'verbose', 'help']
+            )
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        bind_dn = None
+        bind_password = None
+        out_file = None
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-D', '--bind-dn'):
+                bind_dn = a
+
+            elif o in ('-w', '--bind-password'):
+                bind_password = a
+
+            elif o in ('-g', '--generate-ldif'):
+                out_file = a
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.PKIInstance(instance_name)
+        instance.load()
+
+        subsystem = instance.get_subsystem('tps')
+
+        if not subsystem:
+            raise Exception('Subsystem not found')
+
+        if out_file:
+            self.generate_ldif(subsystem, out_file)
+            return
+
+        self.delete_vlv(subsystem, bind_dn, bind_password)
+
+    def generate_ldif(self, subsystem, out_file):
+
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+
+        try:
+            subsystem.customize_file(TPS_VLV_PATH, tmp_file.name)
+
+            parser = ldif.LDIFRecordList(open(tmp_file.name, 'rb'))
+            parser.parse()
+
+            with open(out_file, 'w') as outfile:
+
+                writer = ldif.LDIFWriter(outfile)
+
+                for dn, _ in reversed(parser.all_records):
+                    entry = {'changetype': ['delete']}
+                    writer.unparse(dn, entry)
+
+            self.print_message('Output: %s' % out_file)
+
+        finally:
+            os.unlink(tmp_file.name)
+
+    def delete_vlv(self, subsystem, bind_dn, bind_password):
+
+        conn = subsystem.open_database(bind_dn=bind_dn,
+                                       bind_password=bind_password)
+        try:
+            database = subsystem.config['internaldb.database']
+            base_dn = 'cn=' + database + ',cn=ldbm database, cn=plugins, cn=config'
+
+            if self.verbose:
+                print('Searching %s' % base_dn)
+
+            entries = conn.ldap.search_s(
+                base_dn,
+                ldap.SCOPE_SUBTREE,
+                '(|(objectClass=vlvSearch)(objectClass=vlvIndex))')
+
+            if not entries:
+                self.print_message('VLVs not found')
+                return
+
+            for entry in reversed(entries):
+                dn = entry[0]
+
+                if self.verbose:
+                    print('Deleting %s' % dn)
+
+                conn.ldap.delete_s(dn)
+
+        finally:
+            conn.close()
+
+        self.print_message('VLVs deleted')
+
+
+class TPSDBVLVReindexCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(TPSDBVLVReindexCLI, self).__init__(
+            'reindex', 'Re-index TPS VLVs')
+
+    def print_help(self):
+        print('Usage: pki-server tps-db-vlv-reindex [OPTIONS]')
+        print()
+        print('  -i, --instance <instance ID>       Instance ID (default: pki-tomcat).')
+        print('  -D, --bind-dn <Bind DN>            Connect DN (default: cn=Directory Manager).')
+        print('  -w, --bind-password <password>     Password to connect to database.')
+        print('  -g, --generate-ldif <outfile>      Generate LDIF of required changes.')
+        print('  -v, --verbose                      Run in verbose mode.')
+        print('      --help                         Show help message.')
+        print()
+
+    def execute(self, args):
+        try:
+            opts, _ = getopt.gnu_getopt(
+                args,
+                'i:D:w:x:g:v',
+                ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=',
+                 'verbose', 'help']
+            )
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        bind_dn = 'cn=Directory Manager'
+        bind_password = None
+        out_file = None
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-D', '--bind-dn'):
+                bind_dn = a
+
+            elif o in ('-w', '--bind-password'):
+                bind_password = a
+
+            elif o in ('-g', '--generate-ldif'):
+                out_file = a
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.PKIInstance(instance_name)
+        instance.load()
+
+        subsystem = instance.get_subsystem('tps')
+
+        if not subsystem:
+            raise Exception('Subsystem not found')
+
+        if out_file:
+            self.generate_ldif(subsystem, out_file)
+            return
+
+        self.reindex_vlv(subsystem, bind_dn, bind_password)
+
+    def generate_ldif(self, subsystem, out_file):
+        subsystem.customize_file(TPS_VLV_TASKS_PATH, out_file)
+        self.print_message('Output: %s' % out_file)
+
+    def reindex_vlv(self, subsystem, bind_dn, bind_password):
+
+        input_file = tempfile.NamedTemporaryFile(delete=False)
+        subsystem.customize_file(TPS_VLV_TASKS_PATH, input_file.name)
+
+        conn = subsystem.open_database(bind_dn=bind_dn,
+                                       bind_password=bind_password)
+
+        try:
+            parser = ldif.LDIFRecordList(open(input_file.name, 'rb'))
+            parser.parse()
+
+            for dn, entry in parser.all_records:
+
+                if self.verbose:
+                    print('Adding %s' % dn)
+
+                add_modlist = ldap.modlist.addModlist(entry)
+                conn.ldap.add_s(dn, add_modlist)
+
+                while True:
+                    time.sleep(1)
+
+                    try:
+                        if self.verbose:
+                            print('Checking %s' % dn)
+
+                        conn.ldap.search_s(dn, ldap.SCOPE_BASE)
+                    except ldap.NO_SUCH_OBJECT:
+                        break
+
+        finally:
+            os.unlink(input_file.name)
+            conn.close()
+
+        self.print_message('Reindex complete')