Work on an shelve-based external storage, for performance testing. Sqlite

is just as likely at this point.

7 files changed, 159 insertions, 5 deletions

diff --git a/cobbler/api.py b/cobbler/api.py
index c1063b5..955c41d 100644
--- a/cobbler/api.py
+++ b/cobbler/api.py
@@ -226,6 +226,9 @@ class BootAPI:
         """
         return self._config.deserialize()
 
+    ## FIXME: would be nice to have functions to just deserialize
+    ## certain collections for efficiency in WUI calls.
+
 if __name__ == "__main__":
     api = BootAPI()
     print api.version()
diff --git a/cobbler/cobbler.py b/cobbler/cobbler.py
index 59f1783..879943d 100755
--- a/cobbler/cobbler.py
+++ b/cobbler/cobbler.py
@@ -112,7 +112,8 @@ class BootCLI:
         """
         Run the command line and return system exit code
         """
-        self.api.deserialize()
+        # deserialization is implicit with API construction
+        # self.api.deserialize()
         self.relay_args(self.args[1:], self.commands['toplevel'])
 
     def usage(self,args):
@@ -295,7 +296,9 @@ class BootCLI:
         self.__generic_copy(args,collection_fn,control_fn,exc_msg)
         if objname != objname2:
             collection_fn().remove(objname, with_delete=True)
-        self.api.serialize()
+        
+        # new cobbler does not require explicit serialize calls
+        # self.api.serialize()
 
     def __generic_remove(self,args,alias1,alias2,collection_fn):
         commands = {
@@ -605,7 +608,9 @@ class BootCLI:
             else:
                 raise CX(_("this command doesn't take an option called '%(argument)s'") % { "argument" : key })
         on_ok()
-        self.api.serialize()
+
+        # new cobbler does not require explicit serialize calls
+        # self.api.serialize()
 
     def relay_args(self, args, commands):
         """
diff --git a/cobbler/collection.py b/cobbler/collection.py
index 59eebe8..de7ae72 100644
--- a/cobbler/collection.py
+++ b/cobbler/collection.py
@@ -135,7 +135,9 @@ class Collection(serializable.Serializable):
             self._run_triggers(ref,"/var/lib/cobbler/triggers/add/%s/pre/*" % self.collection_type())
             self.listing[ref.name.lower()] = ref
 
-            self.config.api.serialize()
+            # save just this item if possible, if not, save
+            # the whole collection
+            self.config.serialize_item(self, ref)
 
             lite_sync = action_litesync.BootLiteSync(self.config)
             if isinstance(ref, item_system.System):
diff --git a/cobbler/config.py b/cobbler/config.py
index ee41ea5..658c8ce 100644
--- a/cobbler/config.py
+++ b/cobbler/config.py
@@ -162,6 +162,20 @@ class Config:
               return False
        return True
 
+   def serialize_item(self,collection,item):
+       """
+       Save item in the collection, resaving the whole collection if needed,
+       but ideally just saving the item.
+       """
+       return serializer.serialize_item(collection,item)
+      
+
+   def serialize_delete(self,collection,item):
+       """
+       Erase item from a storage file, if neccessary rewritting the file.
+       """
+       return serializer.serialize_delete(collection,item) 
+
    def deserialize(self):
        """
        Load the object hierachy from disk, using the filenames referenced in each object.
diff --git a/cobbler/item.py b/cobbler/item.py
index b2fc61d..5ec77a2 100644
--- a/cobbler/item.py
+++ b/cobbler/item.py
@@ -51,7 +51,6 @@ class Item(serializable.Serializable):
         self.clear(is_subobject)      # reset behavior differs for inheritance cases
         self.parent = ''              # all objects by default are not subobjects
         self.children = {}            # caching for performance reasons, not serialized
-
  
         
 
diff --git a/cobbler/modules/serializer_shelve.py b/cobbler/modules/serializer_shelve.py
new file mode 100644
index 0000000..174bb0e
--- /dev/null
+++ b/cobbler/modules/serializer_shelve.py
@@ -0,0 +1,109 @@
+"""
+Serializer code for cobbler
+
+Copyright 2007, Red Hat, Inc
+Michael DeHaan <mdehaan@redhat.com>
+
+NOTE: as it stands, the performance of this serializer is not great
+      nor has it been throughly tested.  It is, however, about 4x faster 
+      than the YAML version.  It could be optimized further.
+
+This software may be freely redistributed under the terms of the GNU
+general public license.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+"""
+
+import distutils.sysconfig
+import os
+import sys
+import glob
+import traceback
+
+plib = distutils.sysconfig.get_python_lib()
+mod_path="%s/cobbler" % plib
+sys.path.insert(0, mod_path)
+
+
+from rhpl.translate import _, N_, textdomain, utf8
+from cexceptions import *
+import os
+import shelve
+
+def register():
+    """
+    The mandatory cobbler module registration hook.
+    """
+    return True 
+
+def serialize(obj):
+    """
+    Save an object to disk.  Object must "implement" Serializable.
+    Will create intermediate paths if it can.  Returns True on Success,
+    False on permission errors.
+    """
+    fd = shelve.open(obj.filename() + ".shelve","c")
+
+    # FIXME: this needs to understand deletes
+    # FIXME: create partial serializer and don't use this
+
+    for entry in obj:
+        fd[entry.name] = entry.to_datastruct()
+    fd.sync()
+    return True
+
+def serialize_item(obj, item):
+    fd = shelve.open(obj.filename() + ".shelve","w")
+    fd[item.name] = item.to_datastruct()
+    fd.sync()
+    return True
+
+# NOTE: not heavily tested
+def serialize_item(obj, item):
+    fd = shelve.open(obj.filename() + ".shelve","w")
+    del fd[item.name]
+    fd.sync()
+    return True
+
+def deserialize(obj,topological=False):
+    """
+    Populate an existing object with the contents of datastruct.
+    Object must "implement" Serializable.  Returns True assuming
+    files could be read and contained decent YAML.  Otherwise returns
+    False.
+    """
+    filename = obj.filename() + ".shelve"
+    try: 
+        fd = shelve.open(filename, "r")
+    except:
+        if not os.path.exists(filename):
+            return True
+        else:
+            traceback.print_exc()
+            raise CX(_("Can't access storage file"))
+
+    datastruct = []
+    for (key,value) in fd.iteritems():
+       datastruct.append(value)
+
+    fd.close()
+
+    if topological and type(datastruct) == list:
+       # in order to build the graph links from the flat list, sort by the
+       # depth of items in the graph.  If an object doesn't have a depth, sort it as
+       # if the depth were 0.  It will be assigned a proper depth at serialization
+       # time.  This is a bit cleaner implementation wise than a topological sort,
+       # though that would make a shiny upgrade.
+       datastruct.sort(__depth_cmp)
+    obj.from_datastruct(datastruct)
+    return True
+
+def __depth_cmp(item1, item2):
+    if not item1.has_key("depth"):
+       return 1
+    if not item2.has_key("depth"):
+       return -1
+    return cmp(item1["depth"],item2["depth"])
+
diff --git a/cobbler/serializer.py b/cobbler/serializer.py
index 871aad9..2aa5d87 100644
--- a/cobbler/serializer.py
+++ b/cobbler/serializer.py
@@ -33,10 +33,32 @@ def serialize(obj):
     """
     Save a collection to disk or other storage.  
     """
+    
     storage_module = __get_storage_module(obj.collection_type())
     storage_module.serialize(obj)
     return True
 
+def serialize_item(collection, item):
+    storage_module = __get_storage_module(collection.collection_type())
+    save_fn = getattr(storage_module, "serialize_item", None)
+    if save_fn is None:
+        # print "DEBUG: full serializer"
+        return storage_module.serialize(collection)
+    else:
+        # print "DEBUG: partial serializer"
+        return save_fn(collection,item)
+
+def serialize_delete(collection, item):
+    storage_module = __get_storage_module(collection.collection_type())
+    delete_fn = getattr(storage_module, "serialize_delete", None)
+    if delete_fn is None:
+        # print "DEBUG: full delete"
+        return storage_module.serialize(collection)
+    else:
+        # print "DEBUG: partial delete"
+        return delete_fn(collection,item)
+    
+
 def deserialize(obj,topological=False):
     """
     Fill in an empty collection from disk or other storage