Refactored upgrade framework into base and server upgrade.

The upgrade framework has been split into base and server upgrade
frameworks since they will be run automatically by different RPM
packages during upgrade. The base upgrade framework will upgrade
the system configuration. The server upgrade framework will upgrade
the instances and subsystems.

Ticket #544

2 files changed, 45 insertions, 366 deletions

diff --git a/base/common/python/pki/__init__.py b/base/common/python/pki/__init__.py
index 4d29d0fe1..f9822dde9 100644
--- a/base/common/python/pki/__init__.py
+++ b/base/common/python/pki/__init__.py
@@ -26,9 +26,6 @@ import os
 CONF_DIR          = '/etc/pki'
 SHARE_DIR         = '/usr/share/pki'
 BASE_DIR          = '/var/lib'
-INSTANCE_BASE_DIR = '/var/lib/pki'
-REGISTRY_DIR      = '/etc/sysconfig/pki'
-SUBSYSTEM_TYPES   = ['ca', 'kra', 'ocsp', 'tks']
 
 PACKAGE_VERSION   = SHARE_DIR + '/VERSION'
 
@@ -92,66 +89,11 @@ def implementation_version():
 
     raise Exception('Missing implementation version.')
 
-class PKISubsystem(object):
-
-    def __init__(self, instance, subsystemName):
-        self.instance = instance
-        self.name = subsystemName
-        self.type = instance.type
-        if self.type >= 10:
-            self.conf_dir = os.path.join(INSTANCE_BASE_DIR, \
-                instance.name, 'conf', subsystemName)
-            self.base_dir = os.path.join(INSTANCE_BASE_DIR, \
-                instance.name, subsystemName)
-        else:
-            self.conf_dir = os.path.join(BASE_DIR, instance.name, 'conf')
-            self.base_dir = os.path.join(BASE_DIR, instance.name)
-
-        self.validate()
-
-    def validate(self):
-        if not os.path.exists(self.conf_dir):
-            raise PKIException(
-                'Invalid subsystem: ' + self.__repr__(),
-                 None, self.instance)
-
-
-    def __repr__(self):
-        return str(self.instance) + '/' + self.name
-
-
-class PKIInstance(object):
-
-    def __init__(self, name, type=10):
-        self.name = name
-        self.type = type
-        if self.type >= 10:
-            self.conf_dir = os.path.join(INSTANCE_BASE_DIR, name, 'conf')
-            self.base_dir = os.path.join(INSTANCE_BASE_DIR, name)
-        else:
-            self.conf_dir = os.path.join(BASE_DIR, name, 'conf')
-            self.base_dir = os.path.join(BASE_DIR, name)
-
-        self.validate()
-
-    def validate(self):
-        if not os.path.exists(self.conf_dir):
-            raise PKIException(
-                'Invalid instance: ' + self.__repr__(), None)
-
-
-    def __repr__(self):
-        if self.type == 9:
-            return "Dogtag 9 " + self.name
-        return self.name
 
 class PKIException(Exception):
 
-    def __init__(self, message, exception=None,\
-                 instance=None, subsystem=None):
+    def __init__(self, message, exception=None):
 
         Exception.__init__(self, message)
 
         self.exception = exception
-        self.instance = instance
-        self.subsystem = subsystem
diff --git a/base/common/python/pki/upgrade.py b/base/common/python/pki/upgrade.py
index d87a7ef29..66f07ff66 100644
--- a/base/common/python/pki/upgrade.py
+++ b/base/common/python/pki/upgrade.py
@@ -21,26 +21,18 @@
 
 import functools
 import os
-import pwd
 import re
-import shutil
-import sys
 import traceback
 
 import pki
 
 
-DEFAULT_VERSION   = '10.0.0'
+DEFAULT_VERSION    = '10.0.0'
 
-UPGRADE_DIR       = pki.SHARE_DIR + '/server/upgrade'
-VERSION_DIR       = UPGRADE_DIR + '/%s'
-SCRIPTLET_FILE    = VERSION_DIR + '/%s'
+UPGRADE_DIR        = pki.SHARE_DIR + '/upgrade'
+SYSTEM_TRACKER     = pki.CONF_DIR + '/pki.conf'
 
-SYSTEM_TRACKER    = pki.CONF_DIR + '/pki.conf'
-INSTANCE_TRACKER  = '%s/tomcat.conf'
-SUBSYSTEM_TRACKER = '%s/CS.cfg'
-
-verbose           = False
+verbose            = False
 
 
 @functools.total_ordering
@@ -107,6 +99,7 @@ class Version(object):
     def __repr__(self):
         return self.version
 
+
 class PKIUpgradeTracker(object):
 
     def __init__(self, name, filename,
@@ -148,16 +141,16 @@ class PKIUpgradeTracker(object):
 
     def remove(self):
 
-        print 'Removing ' + self.name + ' tracker.'
+        if verbose: print 'Removing ' + self.name + ' tracker.'
 
         self.remove_version()
         self.remove_index()
         self.write()
 
 
-    def reset(self, version):
+    def set(self, version):
 
-        print 'Resetting ' + self.name + ' tracker.'
+        if verbose: print 'Setting ' + self.name + ' tracker to version ' + str(version) + '.'
 
         self.set_version(version)
         self.remove_index()
@@ -292,24 +285,22 @@ class PKIUpgradeScriptlet(object):
         self.message = None
         self.upgrader = None
 
-
-    def can_upgrade(self, instance=None, subsystem=None):
+    def can_upgrade(self):
 
         # A scriptlet can run if the version matches the tracker and
         # the index is the next to be executed.
 
-        tracker = self.upgrader.get_tracker(instance, subsystem)
+        tracker = self.upgrader.get_tracker()
 
         return self.version == tracker.get_version() and \
             self.index == tracker.get_index() + 1
 
-
-    def update_tracker(self, instance=None, subsystem=None):
+    def update_tracker(self):
 
         # Increment the index in the tracker. If it's the last scriptlet
         # in this version, update the tracker version.
 
-        tracker = self.upgrader.get_tracker(instance, subsystem)
+        tracker = self.upgrader.get_tracker()
 
         if not self.last:
             tracker.set_index(self.index)
@@ -320,98 +311,12 @@ class PKIUpgradeScriptlet(object):
 
         tracker.write()
 
-
-    def upgrade_subsystem(self, instance, subsystem):
-        # Callback method to upgrade a subsystem.
-        pass
-
-
-    def upgrade_instance(self, instance):
-        # Callback method to upgrade an instance.
-        pass
-
-
     def upgrade_system(self):
         # Callback method to upgrade the system.
         pass
 
-
-    def upgrade_subsystems(self, instance):
-
-        for subsystem in self.upgrader.subsystems(instance):
-
-            if not self.can_upgrade(instance, subsystem):
-                if verbose: print 'Skipping ' + str(subsystem) + ' subsystem.'
-                continue
-
-            try:
-                if verbose: print 'Upgrading ' + str(subsystem) + ' subsystem.'
-                self.upgrade_subsystem(instance, subsystem)
-                self.update_tracker(instance, subsystem)
-
-            except Exception as e:
-
-                if verbose: traceback.print_exc()
-                else: print 'ERROR: ' + e.message
-
-                message = 'Failed upgrading ' + str(subsystem) + ' subsystem.'
-                if self.upgrader.silent:
-                    print message
-                else:
-                    result = pki.read_text(message + ' Continue (Yes/No)',
-                        options=['Y', 'N'], default='Y', delimiter='?', caseSensitive=False).lower()
-                    if result == 'y': continue
-
-                raise pki.PKIException(
-                    'Upgrade failed in ' + str(subsystem) + ': ' + e.message,
-                    e, instance, subsystem)
-
-
-    def upgrade_instances(self):
-
-        for instance in self.upgrader.instances():
-
-            self.upgrade_subsystems(instance)
-
-            # If upgrading a specific subsystem don't upgrade the instance.
-            if self.upgrader.subsystemName:
-                continue
-
-            if not self.can_upgrade(instance):
-                if verbose: print 'Skipping ' + str(instance) + ' instance.'
-                continue
-
-            try:
-                if verbose: print 'Upgrading ' + str(instance) + ' instance.'
-                self.upgrade_instance(instance)
-                self.update_tracker(instance)
-
-            except Exception as e:
-
-                if verbose: traceback.print_exc()
-                else: print 'ERROR: ' + e.message
-
-                message = 'Failed upgrading ' + str(instance) + ' instance.'
-                if self.upgrader.silent:
-                    print message
-                else:
-                    result = pki.read_text(message + ' Continue (Yes/No)',
-                        options=['Y', 'N'], default='Y', delimiter='?', caseSensitive=False).lower()
-                    if result == 'y': continue
-
-                raise pki.PKIException(
-                    'Upgrade failed in ' + str(instance) + ': ' + e.message,
-                    e, instance)
-
-
     def upgrade(self):
 
-        self.upgrade_instances()
-
-        # If upgrading a specific instance don't upgrade the system.
-        if self.upgrader.instanceName:
-            return
-
         try:
             if not self.can_upgrade():
                 if verbose: print 'Skipping system.'
@@ -436,11 +341,9 @@ class PKIUpgradeScriptlet(object):
 
             raise pki.PKIException('Upgrade failed: ' + e.message, e)
 
-
     def __eq__(self, other):
         return self.version == other.version and self.index == other.index
 
-
     def __lt__(self, other):
         if self.version < other.version:
             return True
@@ -448,33 +351,25 @@ class PKIUpgradeScriptlet(object):
         return self.version == other.version and self.index < other.index
 
 
-class PKIUpgrader():
+class PKIUpgrader(object):
 
-    def __init__(self, instanceName=None, instanceType=None, \
-        subsystemName=None, version=None, index=None, silent=False):
+    def __init__(self, upgrade_dir=UPGRADE_DIR, version=None, index=None, silent=False):
 
-        self.instanceName = instanceName
-        self.subsystemName = subsystemName
+        self.upgrade_dir = upgrade_dir
         self.version = version
         self.index = index
-        self.silent = silent
-        self.instanceType = instanceType
 
-        if version and not os.path.exists(VERSION_DIR % str(version)):
-            raise pki.PKIException(
-                'Invalid version: ' + str(version),
-                None)
+        self.silent = silent
 
-        if subsystemName and not instanceName:
+        if version and not os.path.exists(self.version_dir(version)):
             raise pki.PKIException(
-                'Invalid subsystem: ' + subsystemName +\
-                ', Instance not defined',
-                None)
+                'Invalid scriptlet version: ' + str(version))
 
         self.system_tracker = None
-        self.instance_trackers = {}
-        self.subsystem_trackers = {}
 
+    def version_dir(self, version):
+
+        return os.path.join(self.upgrade_dir, str(version))
 
     def versions(self):
 
@@ -483,8 +378,8 @@ class PKIUpgrader():
 
         all_versions = []
 
-        if os.path.exists(UPGRADE_DIR):
-            for version in os.listdir(UPGRADE_DIR):
+        if os.path.exists(self.upgrade_dir):
+            for version in os.listdir(self.upgrade_dir):
                 version = Version(version)
 
                 # skip old versions
@@ -513,7 +408,8 @@ class PKIUpgrader():
 
     def scriptlets(self, version):
 
-        filenames = os.listdir(VERSION_DIR % str(version))
+        version_dir = self.version_dir(version)
+        filenames = os.listdir(version_dir)
         scriptlets = []
 
         for filename in filenames:
@@ -532,7 +428,7 @@ class PKIUpgrader():
 
             # load scriptlet class
             vars = {}
-            execfile(SCRIPTLET_FILE % (str(version), filename), vars)
+            execfile(os.path.join(version_dir, filename), vars)
 
             # create scriptlet object
             scriptlet = vars[classname]()
@@ -550,132 +446,28 @@ class PKIUpgrader():
         return scriptlets
 
 
-    def instances(self):
-
-        if self.instanceName and self.instanceType:
-            return [pki.PKIInstance(self.instanceName, self.instanceType)]
-
-        list = []
-        if not self.instanceType or self.instanceType >=10:
-            if os.path.exists(os.path.join(pki.REGISTRY_DIR,'tomcat')):
-                for instanceName in os.listdir(pki.INSTANCE_BASE_DIR):
-                    if not self.instanceName or \
-                        self.instanceName == instanceName:
-                        list.append(pki.PKIInstance(instanceName))
-
-        if not self.instanceType or self.instanceType == 9:
-            for s in pki.SUBSYSTEM_TYPES:
-                if os.path.exists(os.path.join(pki.REGISTRY_DIR, s)):
-                    for instanceName in \
-                        os.listdir(os.path.join(pki.REGISTRY_DIR, s)):
-                        if not self.instanceName or \
-                            self.instanceName == instanceName:
-                            list.append(pki.PKIInstance(instanceName, 9))
-
-        list.sort()
-        return list
-
-
-    def subsystems(self, instance):
-
-        if self.subsystemName:
-            return [pki.PKISubsystem(instance, self.subsystemName)]
-
-        list = []
-
-        if instance.type >= 10:
-            registry_dir = os.path.join(pki.REGISTRY_DIR, 'tomcat',
-                instance.name)
-            for subsystemName in os.listdir(registry_dir):
-                if subsystemName in pki.SUBSYSTEM_TYPES:
-                    list.append(pki.PKISubsystem(instance, subsystemName))
-        else:
-            for subsystemName in pki.SUBSYSTEM_TYPES:
-                registry_dir = os.path.join(
-                    pki.REGISTRY_DIR,
-                    subsystemName,
-                    instance.name)
-                if os.path.exists(registry_dir):
-                    list.append(pki.PKISubsystem(instance, subsystemName))
-
-        list.sort()
-
-        return list
-
+    def get_tracker(self):
 
-    def get_tracker(self, instance=None, subsystem=None):
-
-        if subsystem:
-            name = str(subsystem)
-            try:
-                tracker = self.subsystem_trackers[instance]
-            except KeyError:
-                tracker = PKIUpgradeTracker(name + ' subsystem',
-                    SUBSYSTEM_TRACKER % subsystem.conf_dir,
-                    version_key='cms.product.version',
-                    index_key='cms.upgrade.index')
-                self.subsystem_trackers[name] = tracker
-
-        elif instance:
-            try:
-                tracker = self.instance_trackers[str(instance)]
-            except KeyError:
-                tracker = PKIUpgradeTracker(str(instance) + ' instance',
-                    INSTANCE_TRACKER % instance.conf_dir,
-                    version_key='PKI_VERSION',
-                    index_key='PKI_UPGRADE_INDEX')
-                self.instance_trackers[str(instance)] = tracker
+        if self.system_tracker:
+            tracker = self.system_tracker
 
         else:
-            if self.system_tracker:
-                tracker = self.system_tracker
-            else:
-                tracker = PKIUpgradeTracker('system', SYSTEM_TRACKER,
-                    version_key='PKI_VERSION',
-                    index_key='PKI_UPGRADE_INDEX')
-                self.system_tracker = tracker
+            tracker = PKIUpgradeTracker('system', SYSTEM_TRACKER,
+                version_key='PKI_VERSION',
+                index_key='PKI_UPGRADE_INDEX')
+            self.system_tracker = tracker
 
         return tracker
 
-    # return the oldest version of the components being upgraded
     def get_current_version(self):
 
-        current_version = None
-
-        # if upgrading the entire system, get the system version
-        if not self.instanceName:
-            tracker = self.get_tracker()
-            current_version = tracker.get_version()
-
-        for instance in self.instances():
-
-            # if upgrading the entire instance, check the instance version
-            if not self.subsystemName:
-                tracker = self.get_tracker(instance)
-                version = tracker.get_version()
-
-                # if instance version is older, use instance version
-                if not current_version or version < current_version:
-                    current_version = version
-
-            for subsystem in self.subsystems(instance):
-
-                # subsystem is always upgraded, check the subsystem version
-                tracker = self.get_tracker(instance, subsystem)
-                version = tracker.get_version()
-
-                # if subsystem version is older, use subsystem version
-                if not current_version or version < current_version:
-                    current_version = version
-
-        return current_version
-
+        tracker = self.get_tracker()
+        return tracker.get_version()
 
     def get_target_version(self):
 
         return Version(pki.implementation_version())
 
-
     def is_complete(self):
 
         current_version = self.get_current_version()
@@ -683,7 +475,6 @@ class PKIUpgrader():
 
         return current_version == target_version
 
-
     def upgrade_version(self, version):
 
         print 'Upgrading from version ' + str(version) + ' to ' + str(version.next) + ':'
@@ -694,27 +485,7 @@ class PKIUpgrader():
 
             print 'No upgrade scriptlets.'
 
-            for instance in self.instances():
-                for subsystem in self.subsystems(instance):
-
-                    # update subsystem tracker
-                    tracker = self.get_tracker(instance, subsystem)
-                    tracker.remove_index()
-                    tracker.set_version(version.next)
-                    tracker.write()
-
-                # update instance tracker
-                tracker = self.get_tracker(instance)
-                tracker.remove_index()
-                tracker.set_version(version.next)
-                tracker.write()
-
-            # update system tracker
-            tracker = self.get_tracker()
-            tracker.remove_index()
-            tracker.set_version(version.next)
-            tracker.write()
-
+            self.set_tracker(version.next)
             return
 
         # execute scriptlets
@@ -777,21 +548,8 @@ class PKIUpgrader():
 
     def show_tracker(self):
 
-        if not self.instanceName:
-            tracker = self.get_tracker()
-            tracker.show()
-
-        for instance in self.instances():
-
-            if not self.subsystemName:
-                tracker = self.get_tracker(instance)
-                tracker.show()
-
-            for subsystem in self.subsystems(instance):
-
-                tracker = self.get_tracker(instance, subsystem)
-                tracker.show()
-
+        tracker = self.get_tracker()
+        tracker.show()
 
     def status(self):
 
@@ -805,41 +563,20 @@ class PKIUpgrader():
 
 
     def set_tracker(self, version):
-        if not self.instanceName:
-            tracker = self.get_tracker()
-            tracker.reset(version)
-
-        for instance in self.instances():
-
-            if not self.subsystemName:
-                tracker = self.get_tracker(instance)
-                tracker.reset(version)
-
-            for subsystem in self.subsystems(instance):
-
-                tracker = self.get_tracker(instance, subsystem)
-                tracker.reset(version)
 
+        tracker = self.get_tracker()
+        tracker.set(version)
 
     def reset_tracker(self):
 
         target_version = self.get_target_version()
         self.set_tracker(target_version)
 
+        print 'Tracker has been set to version ' + str(target_version) + '.'
 
     def remove_tracker(self):
 
-        if not self.instanceName:
-            tracker = self.get_tracker()
-            tracker.remove()
-
-        for instance in self.instances():
-
-            if not self.subsystemName:
-                tracker = self.get_tracker(instance)
-                tracker.remove()
-
-            for subsystem in self.subsystems(instance):
+        tracker = self.get_tracker()
+        tracker.remove()
 
-                tracker = self.get_tracker(instance, subsystem)
-                tracker.remove()
+        print 'Tracker has been removed.'