Refactored upgrade framework into base and server upgrade.

The upgrade framework has been split into base and server upgrade
frameworks since they will be run automatically by different RPM
packages during upgrade. The base upgrade framework will upgrade
the system configuration. The server upgrade framework will upgrade
the instances and subsystems.

Ticket #544

7 files changed, 435 insertions, 14 deletions

diff --git a/base/server/CMakeLists.txt b/base/server/CMakeLists.txt
index a2ba5183e..8beedbb67 100644
--- a/base/server/CMakeLists.txt
+++ b/base/server/CMakeLists.txt
@@ -44,9 +44,17 @@ install(
         WORLD_EXECUTE WORLD_READ
 )
 
+# install Python libraries
+install(
+    DIRECTORY
+        python/
+    DESTINATION
+        ${PYTHON_SITE_PACKAGES}
+)
+
+# install executables
 install(
     FILES
-        src/pki-upgrade
         src/pkispawn
         src/pkidestroy
     DESTINATION
@@ -58,6 +66,17 @@ install(
 )
 
 install(
+    FILES
+        sbin/pki-server-upgrade
+    DESTINATION
+        ${SBIN_INSTALL_DIR}
+    PERMISSIONS
+        OWNER_EXECUTE OWNER_WRITE OWNER_READ
+        GROUP_EXECUTE GROUP_READ
+        WORLD_EXECUTE WORLD_READ
+)
+
+install(
     DIRECTORY
         upgrade
     DESTINATION
diff --git a/base/server/python/pki/server/__init__.py b/base/server/python/pki/server/__init__.py
new file mode 100644
index 000000000..62dee2feb
--- /dev/null
+++ b/base/server/python/pki/server/__init__.py
@@ -0,0 +1,93 @@
+#!/usr/bin/python
+# Authors:
+#     Endi S. Dewata <edewata@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright (C) 2013 Red Hat, Inc.
+# All rights reserved.
+#
+
+import re
+import os
+
+import pki
+
+INSTANCE_BASE_DIR = '/var/lib/pki'
+REGISTRY_DIR      = '/etc/sysconfig/pki'
+SUBSYSTEM_TYPES   = ['ca', 'kra', 'ocsp', 'tks']
+
+
+class PKISubsystem(object):
+
+    def __init__(self, instance, subsystemName):
+        self.instance = instance
+        self.name = subsystemName
+        self.type = instance.type
+        if self.type >= 10:
+            self.conf_dir = os.path.join(INSTANCE_BASE_DIR, \
+                instance.name, 'conf', subsystemName)
+            self.base_dir = os.path.join(INSTANCE_BASE_DIR, \
+                instance.name, subsystemName)
+        else:
+            self.conf_dir = os.path.join(BASE_DIR, instance.name, 'conf')
+            self.base_dir = os.path.join(BASE_DIR, instance.name)
+
+        self.validate()
+
+    def validate(self):
+        if not os.path.exists(self.conf_dir):
+            raise PKIException(
+                'Invalid subsystem: ' + self.__repr__(),
+                 None, self.instance)
+
+
+    def __repr__(self):
+        return str(self.instance) + '/' + self.name
+
+
+class PKIInstance(object):
+
+    def __init__(self, name, type=10):
+        self.name = name
+        self.type = type
+        if self.type >= 10:
+            self.conf_dir = os.path.join(INSTANCE_BASE_DIR, name, 'conf')
+            self.base_dir = os.path.join(INSTANCE_BASE_DIR, name)
+        else:
+            self.conf_dir = os.path.join(BASE_DIR, name, 'conf')
+            self.base_dir = os.path.join(BASE_DIR, name)
+
+        self.validate()
+
+    def validate(self):
+        if not os.path.exists(self.conf_dir):
+            raise PKIException(
+                'Invalid instance: ' + self.__repr__(), None)
+
+    def __repr__(self):
+        if self.type == 9:
+            return "Dogtag 9 " + self.name
+        return self.name
+
+
+class PKIServerException(pki.PKIException):
+
+    def __init__(self, message, exception=None,\
+                 instance=None, subsystem=None):
+
+        pki.upgrade.PKIException.__init__(self, message, exception)
+
+        self.instance = instance
+        self.subsystem = subsystem
diff --git a/base/server/python/pki/server/upgrade.py b/base/server/python/pki/server/upgrade.py
new file mode 100644
index 000000000..45ed9c13b
--- /dev/null
+++ b/base/server/python/pki/server/upgrade.py
@@ -0,0 +1,307 @@
+#!/usr/bin/python
+# Authors:
+#     Endi S. Dewata <edewata@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright (C) 2013 Red Hat, Inc.
+# All rights reserved.
+#
+
+import os
+import traceback
+
+import pki
+import pki.upgrade
+import pki.server
+
+from pki.upgrade import verbose
+
+UPGRADE_DIR       = pki.SHARE_DIR + '/server/upgrade'
+
+INSTANCE_TRACKER  = '%s/tomcat.conf'
+SUBSYSTEM_TRACKER = '%s/CS.cfg'
+
+
+class PKIServerUpgradeScriptlet(pki.upgrade.PKIUpgradeScriptlet):
+
+    def __init__(self):
+
+        super(PKIServerUpgradeScriptlet, self).__init__()
+
+    def can_upgrade(self, instance, subsystem=None):
+
+        # A scriptlet can run if the version matches the tracker and
+        # the index is the next to be executed.
+
+        tracker = self.upgrader.get_tracker(instance, subsystem)
+
+        return self.version == tracker.get_version() and \
+            self.index == tracker.get_index() + 1
+
+    def update_tracker(self, instance, subsystem=None):
+
+        # Increment the index in the tracker. If it's the last scriptlet
+        # in this version, update the tracker version.
+
+        tracker = self.upgrader.get_tracker(instance, subsystem)
+
+        if not self.last:
+            tracker.set_index(self.index)
+
+        else:
+            tracker.remove_index()
+            tracker.set_version(self.version.next)
+
+        tracker.write()
+
+    def upgrade(self):
+
+        for instance in self.upgrader.instances():
+
+            self.upgrade_subsystems(instance)
+
+            # If upgrading a specific subsystem don't upgrade the instance.
+            if self.upgrader.subsystemName:
+                continue
+
+            if not self.can_upgrade(instance):
+                if verbose: print 'Skipping ' + str(instance) + ' instance.'
+                continue
+
+            try:
+                if verbose: print 'Upgrading ' + str(instance) + ' instance.'
+                self.upgrade_instance(instance)
+                self.update_tracker(instance)
+
+            except Exception as e:
+
+                if verbose: traceback.print_exc()
+                else: print 'ERROR: ' + e.message
+
+                message = 'Failed upgrading ' + str(instance) + ' instance.'
+                if self.upgrader.silent:
+                    print message
+                else:
+                    result = pki.read_text(message + ' Continue (Yes/No)',
+                        options=['Y', 'N'], default='Y', delimiter='?', caseSensitive=False).lower()
+                    if result == 'y': continue
+
+                raise pki.server.PKIServerException(
+                    'Upgrade failed in ' + str(instance) + ': ' + e.message,
+                    e, instance)
+
+    def upgrade_subsystems(self, instance):
+
+        for subsystem in self.upgrader.subsystems(instance):
+
+            if not self.can_upgrade(instance, subsystem):
+                if verbose: print 'Skipping ' + str(subsystem) + ' subsystem.'
+                continue
+
+            try:
+                if verbose: print 'Upgrading ' + str(subsystem) + ' subsystem.'
+                self.upgrade_subsystem(instance, subsystem)
+                self.update_tracker(instance, subsystem)
+
+            except Exception as e:
+
+                if verbose: traceback.print_exc()
+                else: print 'ERROR: ' + e.message
+
+                message = 'Failed upgrading ' + str(subsystem) + ' subsystem.'
+                if self.upgrader.silent:
+                    print message
+                else:
+                    result = pki.read_text(message + ' Continue (Yes/No)',
+                        options=['Y', 'N'], default='Y', delimiter='?', caseSensitive=False).lower()
+                    if result == 'y': continue
+
+                raise pki.server.PKIServerException(
+                    'Upgrade failed in ' + str(subsystem) + ': ' + e.message,
+                    e, instance, subsystem)
+
+    def upgrade_subsystem(self, instance, subsystem):
+        # Callback method to upgrade a subsystem.
+        pass
+
+    def upgrade_instance(self, instance):
+        # Callback method to upgrade an instance.
+        pass
+
+
+class PKIServerUpgrader(pki.upgrade.PKIUpgrader):
+
+    def __init__(self, instanceName=None, instanceType=None, subsystemName=None, \
+        upgrade_dir=UPGRADE_DIR, version=None, index=None, silent=False):
+
+        super(PKIServerUpgrader, self).__init__(upgrade_dir, version, index, silent)
+
+        if subsystemName and not instanceName:
+            raise pki.PKIException(
+                'Invalid subsystem: ' + subsystemName + ', Instance not defined')
+
+        self.instanceName = instanceName
+        self.instanceType = instanceType
+        self.subsystemName = subsystemName
+
+        self.instance_trackers = {}
+        self.subsystem_trackers = {}
+
+    def instances(self):
+
+        if self.instanceName and self.instanceType:
+            return [pki.server.PKIInstance(self.instanceName, self.instanceType)]
+
+        list = []
+
+        if not self.instanceType or self.instanceType >=10:
+            if os.path.exists(os.path.join(pki.server.REGISTRY_DIR,'tomcat')):
+                for instanceName in os.listdir(pki.server.INSTANCE_BASE_DIR):
+                    if not self.instanceName or \
+                        self.instanceName == instanceName:
+                        list.append(pki.server.PKIInstance(instanceName))
+
+        if not self.instanceType or self.instanceType == 9:
+            for s in pki.server.SUBSYSTEM_TYPES:
+                if os.path.exists(os.path.join(pki.server.REGISTRY_DIR, s)):
+                    for instanceName in \
+                        os.listdir(os.path.join(pki.server.REGISTRY_DIR, s)):
+                        if not self.instanceName or \
+                            self.instanceName == instanceName:
+                            list.append(pki.server.PKIInstance(instanceName, 9))
+
+        list.sort()
+
+        return list
+
+
+    def subsystems(self, instance):
+
+        if self.subsystemName:
+            return [pki.server.PKISubsystem(instance, self.subsystemName)]
+
+        list = []
+
+        if instance.type >= 10:
+            registry_dir = os.path.join(pki.server.REGISTRY_DIR, 'tomcat',
+                instance.name)
+            for subsystemName in os.listdir(registry_dir):
+                if subsystemName in pki.server.SUBSYSTEM_TYPES:
+                    list.append(pki.server.PKISubsystem(instance, subsystemName))
+        else:
+            for subsystemName in pki.server.SUBSYSTEM_TYPES:
+                registry_dir = os.path.join(
+                    pki.server.REGISTRY_DIR,
+                    subsystemName,
+                    instance.name)
+                if os.path.exists(registry_dir):
+                    list.append(pki.server.PKISubsystem(instance, subsystemName))
+
+        list.sort()
+
+        return list
+
+    def get_tracker(self, instance, subsystem=None):
+
+        if subsystem:
+            name = str(subsystem)
+            try:
+                tracker = self.subsystem_trackers[instance]
+            except KeyError:
+                tracker = pki.upgrade.PKIUpgradeTracker(name + ' subsystem',
+                    SUBSYSTEM_TRACKER % subsystem.conf_dir,
+                    version_key='cms.product.version',
+                    index_key='cms.upgrade.index')
+                self.subsystem_trackers[name] = tracker
+
+        else:
+            try:
+                tracker = self.instance_trackers[str(instance)]
+            except KeyError:
+                tracker = pki.upgrade.PKIUpgradeTracker(str(instance) + ' instance',
+                    INSTANCE_TRACKER % instance.conf_dir,
+                    version_key='PKI_VERSION',
+                    index_key='PKI_UPGRADE_INDEX')
+                self.instance_trackers[str(instance)] = tracker
+
+        return tracker
+
+    def get_current_version(self):
+
+        current_version = None
+
+        for instance in self.instances():
+
+            # if upgrading the entire instance, check the instance version
+            if not self.subsystemName:
+                tracker = self.get_tracker(instance)
+                version = tracker.get_version()
+
+                # if instance version is older, use instance version
+                if not current_version or version < current_version:
+                    current_version = version
+
+            for subsystem in self.subsystems(instance):
+
+                # subsystem is always upgraded, check the subsystem version
+                tracker = self.get_tracker(instance, subsystem)
+                version = tracker.get_version()
+
+                # if subsystem version is older, use subsystem version
+                if not current_version or version < current_version:
+                    current_version = version
+
+        return current_version
+
+    def show_tracker(self):
+
+        for instance in self.instances():
+
+            if not self.subsystemName:
+                tracker = self.get_tracker(instance)
+                tracker.show()
+
+            for subsystem in self.subsystems(instance):
+
+                tracker = self.get_tracker(instance, subsystem)
+                tracker.show()
+
+    def set_tracker(self, version):
+
+        for instance in self.instances():
+
+            if not self.subsystemName:
+                tracker = self.get_tracker(instance)
+                tracker.set(version)
+
+            for subsystem in self.subsystems(instance):
+
+                tracker = self.get_tracker(instance, subsystem)
+                tracker.set(version)
+
+    def remove_tracker(self):
+
+        for instance in self.instances():
+
+            if not self.subsystemName:
+                tracker = self.get_tracker(instance)
+                tracker.remove()
+
+            for subsystem in self.subsystems(instance):
+
+                tracker = self.get_tracker(instance, subsystem)
+                tracker.remove()
+
+        print 'Tracker has been removed.'
diff --git a/base/server/src/pki-upgrade b/base/server/sbin/pki-server-upgrade
index b6f4acb12..58bc9ccb4 100755
--- a/base/server/src/pki-upgrade
+++ b/base/server/sbin/pki-server-upgrade
@@ -20,15 +20,11 @@
 #
 
 import getopt
-import os
-import re
 import signal
-import subprocess
 import sys
-import traceback
 
-import pki.upgrade
 import pki
+import pki.server.upgrade
 
 
 def interrupt_handler(signal, frame):
@@ -39,15 +35,19 @@ def interrupt_handler(signal, frame):
 
 
 def usage():
-    print 'Usage: pki-upgrade [OPTIONS]'
+    print 'Usage: pki-server-upgrade [OPTIONS]'
+    print
     print '  -i, --instance <instance>      Upgrade a specific instance only.'
     print '  -s, --subsystem <subsystem>    Upgrade a specific subsystem in an instance only.'
     print '  -t, --instance-type <type>     Specify 9 for upgraded Dogtag 9 instances only,'
     print '                                 10 for Dogtag 10 instances only.'
+    print
     print '  --scriptlet-version <version>  Run scriptlets for a specific version only.'
     print '  --scriptlet-index <index>      Run a specific scriptlet only.'
+    print
     print '  --silent                       Upgrade in silent mode. Ignore any failures.'
     print '  --status                       Show upgrade status only. Do not perform upgrade.'
+    print
     print '  -X                             Show advanced usage.'
     print '  -v, --verbose                  Run in verbose mode.'
     print '  -h, --help                     Show this help message.'
@@ -80,8 +80,10 @@ def main(argv):
     instanceName = None
     subsystemName = None
     instanceType = None
+
     version = None
     index = None
+
     silent = False
     status = False
     remove_tracker = False
@@ -142,7 +144,7 @@ def main(argv):
         sys.exit(1)
 
     try:
-        upgrader = pki.upgrade.PKIUpgrader(
+        upgrader = pki.server.upgrade.PKIServerUpgrader(
             instanceName = instanceName,
             subsystemName = subsystemName,
             instanceType = instanceType,
diff --git a/base/server/upgrade/10.0.1/01-ReplaceRandomNumberGenerator b/base/server/upgrade/10.0.1/01-ReplaceRandomNumberGenerator
index 1731edfb5..e73ede236 100755
--- a/base/server/upgrade/10.0.1/01-ReplaceRandomNumberGenerator
+++ b/base/server/upgrade/10.0.1/01-ReplaceRandomNumberGenerator
@@ -28,10 +28,10 @@ import sys
 from lxml import etree
 
 import pki
-import pki.upgrade
+import pki.server.upgrade
 
 
-class ReplaceRandomNumberGenerator(pki.upgrade.PKIUpgradeScriptlet):
+class ReplaceRandomNumberGenerator(pki.server.upgrade.PKIServerUpgradeScriptlet):
 
     def __init__(self):
 
diff --git a/base/server/upgrade/10.0.1/02-CloningInterfaceChanges b/base/server/upgrade/10.0.1/02-CloningInterfaceChanges
index 2ecc1f8ce..d97cb4258 100755
--- a/base/server/upgrade/10.0.1/02-CloningInterfaceChanges
+++ b/base/server/upgrade/10.0.1/02-CloningInterfaceChanges
@@ -24,9 +24,9 @@ import sys
 from lxml import etree as ET
 
 import pki
-import pki.upgrade
+import pki.server.upgrade
 
-class CloningInterfaceChanges(pki.upgrade.PKIUpgradeScriptlet):
+class CloningInterfaceChanges(pki.server.upgrade.PKIServerUpgradeScriptlet):
 
     updateDomainServletData = """
         <servlet>
diff --git a/base/server/upgrade/10.0.1/03-AddRestServlet b/base/server/upgrade/10.0.1/03-AddRestServlet
index 36af3eebc..dd656e972 100755
--- a/base/server/upgrade/10.0.1/03-AddRestServlet
+++ b/base/server/upgrade/10.0.1/03-AddRestServlet
@@ -24,9 +24,9 @@ import sys
 from lxml import etree as ET
 
 import pki
-import pki.upgrade
+import pki.server.upgrade
 
-class AddRestServlet(pki.upgrade.PKIUpgradeScriptlet):
+class AddRestServlet(pki.server.upgrade.PKIServerUpgradeScriptlet):
 
     restServicesServletData = """
         <servlet>