From 096b9e109e2f8a954af25b8f5241d5f7fd089755 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Mon, 15 Oct 2007 11:01:19 -0400 Subject: Work on an shelve-based external storage, for performance testing. Sqlite is just as likely at this point. --- cobbler/api.py | 3 + cobbler/cobbler.py | 11 +++- cobbler/collection.py | 4 +- cobbler/config.py | 14 +++++ cobbler/item.py | 1 - cobbler/modules/serializer_shelve.py | 109 +++++++++++++++++++++++++++++++++++ cobbler/serializer.py | 22 +++++++ 7 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 cobbler/modules/serializer_shelve.py (limited to 'cobbler') diff --git a/cobbler/api.py b/cobbler/api.py index c1063b5..955c41d 100644 --- a/cobbler/api.py +++ b/cobbler/api.py @@ -226,6 +226,9 @@ class BootAPI: """ return self._config.deserialize() + ## FIXME: would be nice to have functions to just deserialize + ## certain collections for efficiency in WUI calls. + if __name__ == "__main__": api = BootAPI() print api.version() diff --git a/cobbler/cobbler.py b/cobbler/cobbler.py index 59f1783..879943d 100755 --- a/cobbler/cobbler.py +++ b/cobbler/cobbler.py @@ -112,7 +112,8 @@ class BootCLI: """ Run the command line and return system exit code """ - self.api.deserialize() + # deserialization is implicit with API construction + # self.api.deserialize() self.relay_args(self.args[1:], self.commands['toplevel']) def usage(self,args): @@ -295,7 +296,9 @@ class BootCLI: self.__generic_copy(args,collection_fn,control_fn,exc_msg) if objname != objname2: collection_fn().remove(objname, with_delete=True) - self.api.serialize() + + # new cobbler does not require explicit serialize calls + # self.api.serialize() def __generic_remove(self,args,alias1,alias2,collection_fn): commands = { @@ -605,7 +608,9 @@ class BootCLI: else: raise CX(_("this command doesn't take an option called '%(argument)s'") % { "argument" : key }) on_ok() - self.api.serialize() + + # new cobbler does not require explicit serialize calls + # self.api.serialize() def relay_args(self, args, commands): """ diff --git a/cobbler/collection.py b/cobbler/collection.py index 59eebe8..de7ae72 100644 --- a/cobbler/collection.py +++ b/cobbler/collection.py @@ -135,7 +135,9 @@ class Collection(serializable.Serializable): self._run_triggers(ref,"/var/lib/cobbler/triggers/add/%s/pre/*" % self.collection_type()) self.listing[ref.name.lower()] = ref - self.config.api.serialize() + # save just this item if possible, if not, save + # the whole collection + self.config.serialize_item(self, ref) lite_sync = action_litesync.BootLiteSync(self.config) if isinstance(ref, item_system.System): diff --git a/cobbler/config.py b/cobbler/config.py index ee41ea5..658c8ce 100644 --- a/cobbler/config.py +++ b/cobbler/config.py @@ -162,6 +162,20 @@ class Config: return False return True + def serialize_item(self,collection,item): + """ + Save item in the collection, resaving the whole collection if needed, + but ideally just saving the item. + """ + return serializer.serialize_item(collection,item) + + + def serialize_delete(self,collection,item): + """ + Erase item from a storage file, if neccessary rewritting the file. + """ + return serializer.serialize_delete(collection,item) + def deserialize(self): """ Load the object hierachy from disk, using the filenames referenced in each object. diff --git a/cobbler/item.py b/cobbler/item.py index b2fc61d..5ec77a2 100644 --- a/cobbler/item.py +++ b/cobbler/item.py @@ -51,7 +51,6 @@ class Item(serializable.Serializable): self.clear(is_subobject) # reset behavior differs for inheritance cases self.parent = '' # all objects by default are not subobjects self.children = {} # caching for performance reasons, not serialized - diff --git a/cobbler/modules/serializer_shelve.py b/cobbler/modules/serializer_shelve.py new file mode 100644 index 0000000..174bb0e --- /dev/null +++ b/cobbler/modules/serializer_shelve.py @@ -0,0 +1,109 @@ +""" +Serializer code for cobbler + +Copyright 2007, Red Hat, Inc +Michael DeHaan + +NOTE: as it stands, the performance of this serializer is not great + nor has it been throughly tested. It is, however, about 4x faster + than the YAML version. It could be optimized further. + +This software may be freely redistributed under the terms of the GNU +general public license. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +""" + +import distutils.sysconfig +import os +import sys +import glob +import traceback + +plib = distutils.sysconfig.get_python_lib() +mod_path="%s/cobbler" % plib +sys.path.insert(0, mod_path) + + +from rhpl.translate import _, N_, textdomain, utf8 +from cexceptions import * +import os +import shelve + +def register(): + """ + The mandatory cobbler module registration hook. + """ + return True + +def serialize(obj): + """ + Save an object to disk. Object must "implement" Serializable. + Will create intermediate paths if it can. Returns True on Success, + False on permission errors. + """ + fd = shelve.open(obj.filename() + ".shelve","c") + + # FIXME: this needs to understand deletes + # FIXME: create partial serializer and don't use this + + for entry in obj: + fd[entry.name] = entry.to_datastruct() + fd.sync() + return True + +def serialize_item(obj, item): + fd = shelve.open(obj.filename() + ".shelve","w") + fd[item.name] = item.to_datastruct() + fd.sync() + return True + +# NOTE: not heavily tested +def serialize_item(obj, item): + fd = shelve.open(obj.filename() + ".shelve","w") + del fd[item.name] + fd.sync() + return True + +def deserialize(obj,topological=False): + """ + Populate an existing object with the contents of datastruct. + Object must "implement" Serializable. Returns True assuming + files could be read and contained decent YAML. Otherwise returns + False. + """ + filename = obj.filename() + ".shelve" + try: + fd = shelve.open(filename, "r") + except: + if not os.path.exists(filename): + return True + else: + traceback.print_exc() + raise CX(_("Can't access storage file")) + + datastruct = [] + for (key,value) in fd.iteritems(): + datastruct.append(value) + + fd.close() + + if topological and type(datastruct) == list: + # in order to build the graph links from the flat list, sort by the + # depth of items in the graph. If an object doesn't have a depth, sort it as + # if the depth were 0. It will be assigned a proper depth at serialization + # time. This is a bit cleaner implementation wise than a topological sort, + # though that would make a shiny upgrade. + datastruct.sort(__depth_cmp) + obj.from_datastruct(datastruct) + return True + +def __depth_cmp(item1, item2): + if not item1.has_key("depth"): + return 1 + if not item2.has_key("depth"): + return -1 + return cmp(item1["depth"],item2["depth"]) + diff --git a/cobbler/serializer.py b/cobbler/serializer.py index 871aad9..2aa5d87 100644 --- a/cobbler/serializer.py +++ b/cobbler/serializer.py @@ -33,10 +33,32 @@ def serialize(obj): """ Save a collection to disk or other storage. """ + storage_module = __get_storage_module(obj.collection_type()) storage_module.serialize(obj) return True +def serialize_item(collection, item): + storage_module = __get_storage_module(collection.collection_type()) + save_fn = getattr(storage_module, "serialize_item", None) + if save_fn is None: + # print "DEBUG: full serializer" + return storage_module.serialize(collection) + else: + # print "DEBUG: partial serializer" + return save_fn(collection,item) + +def serialize_delete(collection, item): + storage_module = __get_storage_module(collection.collection_type()) + delete_fn = getattr(storage_module, "serialize_delete", None) + if delete_fn is None: + # print "DEBUG: full delete" + return storage_module.serialize(collection) + else: + # print "DEBUG: partial delete" + return delete_fn(collection,item) + + def deserialize(obj,topological=False): """ Fill in an empty collection from disk or other storage -- cgit