summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichael DeHaan <mdehaan@redhat.com>2007-10-15 11:01:19 -0400
committerMichael DeHaan <mdehaan@redhat.com>2007-10-15 11:01:19 -0400
commit096b9e109e2f8a954af25b8f5241d5f7fd089755 (patch)
tree57b70179eed0b5b3b65a8f78d3fb5ad8093496d5
parent2fd4775c68160a8617fff5f7015542bf8c1501e5 (diff)
downloadthird_party-cobbler-096b9e109e2f8a954af25b8f5241d5f7fd089755.zip
third_party-cobbler-096b9e109e2f8a954af25b8f5241d5f7fd089755.tar.gz
third_party-cobbler-096b9e109e2f8a954af25b8f5241d5f7fd089755.tar.xz
Work on an shelve-based external storage, for performance testing. Sqlite
is just as likely at this point.
-rw-r--r--Makefile13
-rw-r--r--cobbler/api.py3
-rwxr-xr-xcobbler/cobbler.py11
-rw-r--r--cobbler/collection.py4
-rw-r--r--cobbler/config.py14
-rw-r--r--cobbler/item.py1
-rw-r--r--cobbler/modules/serializer_shelve.py109
-rw-r--r--cobbler/serializer.py22
-rw-r--r--docs/wui.html2
9 files changed, 172 insertions, 7 deletions
diff --git a/Makefile b/Makefile
index cb2661a..11dc5ce 100644
--- a/Makefile
+++ b/Makefile
@@ -16,9 +16,13 @@ manpage:
pod2man --center="cobbler" --release="" ./docs/cobbler.pod | gzip -c > ./docs/cobbler.1.gz
pod2html ./docs/cobbler.pod > ./docs/cobbler.html
-test: install
+test:
python tests/tests.py
-rm -rf /tmp/_cobbler-*
+
+test2:
+ python tests/multi.py
+
build: clean updatewui messages
python setup.py build -f
@@ -28,9 +32,11 @@ install: clean manpage
devinstall:
cp /var/lib/cobbler/settings /tmp/cobbler_settings
cp /etc/cobbler/auth.conf /tmp/cobbler_auth.conf
+ cp /etc/cobbler/modules.conf /tmp/cobbler_modules.conf
make install
cp /tmp/cobbler_settings /var/lib/cobbler/settings
cp /tmp/cobbler_auth.conf /etc/cobbler/auth.conf
+ cp /tmp/cobbler_modules.conf /etc/cobbler/modules.conf
sdist: clean messages updatewui
python setup.py sdist
@@ -69,3 +75,8 @@ updatewui:
-(rm ./webui_templates/*.bak)
mv ./webui_templates/master.py ./cobbler/webui
+eraseconfig:
+ -rm /var/lib/cobbler/distros*
+ -rm /var/lib/cobbler/profiles*
+ -rm /var/lib/cobbler/systems*
+ -rm /var/lib/cobbler/repos*
diff --git a/cobbler/api.py b/cobbler/api.py
index c1063b5..955c41d 100644
--- a/cobbler/api.py
+++ b/cobbler/api.py
@@ -226,6 +226,9 @@ class BootAPI:
"""
return self._config.deserialize()
+ ## FIXME: would be nice to have functions to just deserialize
+ ## certain collections for efficiency in WUI calls.
+
if __name__ == "__main__":
api = BootAPI()
print api.version()
diff --git a/cobbler/cobbler.py b/cobbler/cobbler.py
index 59f1783..879943d 100755
--- a/cobbler/cobbler.py
+++ b/cobbler/cobbler.py
@@ -112,7 +112,8 @@ class BootCLI:
"""
Run the command line and return system exit code
"""
- self.api.deserialize()
+ # deserialization is implicit with API construction
+ # self.api.deserialize()
self.relay_args(self.args[1:], self.commands['toplevel'])
def usage(self,args):
@@ -295,7 +296,9 @@ class BootCLI:
self.__generic_copy(args,collection_fn,control_fn,exc_msg)
if objname != objname2:
collection_fn().remove(objname, with_delete=True)
- self.api.serialize()
+
+ # new cobbler does not require explicit serialize calls
+ # self.api.serialize()
def __generic_remove(self,args,alias1,alias2,collection_fn):
commands = {
@@ -605,7 +608,9 @@ class BootCLI:
else:
raise CX(_("this command doesn't take an option called '%(argument)s'") % { "argument" : key })
on_ok()
- self.api.serialize()
+
+ # new cobbler does not require explicit serialize calls
+ # self.api.serialize()
def relay_args(self, args, commands):
"""
diff --git a/cobbler/collection.py b/cobbler/collection.py
index 59eebe8..de7ae72 100644
--- a/cobbler/collection.py
+++ b/cobbler/collection.py
@@ -135,7 +135,9 @@ class Collection(serializable.Serializable):
self._run_triggers(ref,"/var/lib/cobbler/triggers/add/%s/pre/*" % self.collection_type())
self.listing[ref.name.lower()] = ref
- self.config.api.serialize()
+ # save just this item if possible, if not, save
+ # the whole collection
+ self.config.serialize_item(self, ref)
lite_sync = action_litesync.BootLiteSync(self.config)
if isinstance(ref, item_system.System):
diff --git a/cobbler/config.py b/cobbler/config.py
index ee41ea5..658c8ce 100644
--- a/cobbler/config.py
+++ b/cobbler/config.py
@@ -162,6 +162,20 @@ class Config:
return False
return True
+ def serialize_item(self,collection,item):
+ """
+ Save item in the collection, resaving the whole collection if needed,
+ but ideally just saving the item.
+ """
+ return serializer.serialize_item(collection,item)
+
+
+ def serialize_delete(self,collection,item):
+ """
+ Erase item from a storage file, if neccessary rewritting the file.
+ """
+ return serializer.serialize_delete(collection,item)
+
def deserialize(self):
"""
Load the object hierachy from disk, using the filenames referenced in each object.
diff --git a/cobbler/item.py b/cobbler/item.py
index b2fc61d..5ec77a2 100644
--- a/cobbler/item.py
+++ b/cobbler/item.py
@@ -51,7 +51,6 @@ class Item(serializable.Serializable):
self.clear(is_subobject) # reset behavior differs for inheritance cases
self.parent = '' # all objects by default are not subobjects
self.children = {} # caching for performance reasons, not serialized
-
diff --git a/cobbler/modules/serializer_shelve.py b/cobbler/modules/serializer_shelve.py
new file mode 100644
index 0000000..174bb0e
--- /dev/null
+++ b/cobbler/modules/serializer_shelve.py
@@ -0,0 +1,109 @@
+"""
+Serializer code for cobbler
+
+Copyright 2007, Red Hat, Inc
+Michael DeHaan <mdehaan@redhat.com>
+
+NOTE: as it stands, the performance of this serializer is not great
+ nor has it been throughly tested. It is, however, about 4x faster
+ than the YAML version. It could be optimized further.
+
+This software may be freely redistributed under the terms of the GNU
+general public license.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+"""
+
+import distutils.sysconfig
+import os
+import sys
+import glob
+import traceback
+
+plib = distutils.sysconfig.get_python_lib()
+mod_path="%s/cobbler" % plib
+sys.path.insert(0, mod_path)
+
+
+from rhpl.translate import _, N_, textdomain, utf8
+from cexceptions import *
+import os
+import shelve
+
+def register():
+ """
+ The mandatory cobbler module registration hook.
+ """
+ return True
+
+def serialize(obj):
+ """
+ Save an object to disk. Object must "implement" Serializable.
+ Will create intermediate paths if it can. Returns True on Success,
+ False on permission errors.
+ """
+ fd = shelve.open(obj.filename() + ".shelve","c")
+
+ # FIXME: this needs to understand deletes
+ # FIXME: create partial serializer and don't use this
+
+ for entry in obj:
+ fd[entry.name] = entry.to_datastruct()
+ fd.sync()
+ return True
+
+def serialize_item(obj, item):
+ fd = shelve.open(obj.filename() + ".shelve","w")
+ fd[item.name] = item.to_datastruct()
+ fd.sync()
+ return True
+
+# NOTE: not heavily tested
+def serialize_item(obj, item):
+ fd = shelve.open(obj.filename() + ".shelve","w")
+ del fd[item.name]
+ fd.sync()
+ return True
+
+def deserialize(obj,topological=False):
+ """
+ Populate an existing object with the contents of datastruct.
+ Object must "implement" Serializable. Returns True assuming
+ files could be read and contained decent YAML. Otherwise returns
+ False.
+ """
+ filename = obj.filename() + ".shelve"
+ try:
+ fd = shelve.open(filename, "r")
+ except:
+ if not os.path.exists(filename):
+ return True
+ else:
+ traceback.print_exc()
+ raise CX(_("Can't access storage file"))
+
+ datastruct = []
+ for (key,value) in fd.iteritems():
+ datastruct.append(value)
+
+ fd.close()
+
+ if topological and type(datastruct) == list:
+ # in order to build the graph links from the flat list, sort by the
+ # depth of items in the graph. If an object doesn't have a depth, sort it as
+ # if the depth were 0. It will be assigned a proper depth at serialization
+ # time. This is a bit cleaner implementation wise than a topological sort,
+ # though that would make a shiny upgrade.
+ datastruct.sort(__depth_cmp)
+ obj.from_datastruct(datastruct)
+ return True
+
+def __depth_cmp(item1, item2):
+ if not item1.has_key("depth"):
+ return 1
+ if not item2.has_key("depth"):
+ return -1
+ return cmp(item1["depth"],item2["depth"])
+
diff --git a/cobbler/serializer.py b/cobbler/serializer.py
index 871aad9..2aa5d87 100644
--- a/cobbler/serializer.py
+++ b/cobbler/serializer.py
@@ -33,10 +33,32 @@ def serialize(obj):
"""
Save a collection to disk or other storage.
"""
+
storage_module = __get_storage_module(obj.collection_type())
storage_module.serialize(obj)
return True
+def serialize_item(collection, item):
+ storage_module = __get_storage_module(collection.collection_type())
+ save_fn = getattr(storage_module, "serialize_item", None)
+ if save_fn is None:
+ # print "DEBUG: full serializer"
+ return storage_module.serialize(collection)
+ else:
+ # print "DEBUG: partial serializer"
+ return save_fn(collection,item)
+
+def serialize_delete(collection, item):
+ storage_module = __get_storage_module(collection.collection_type())
+ delete_fn = getattr(storage_module, "serialize_delete", None)
+ if delete_fn is None:
+ # print "DEBUG: full delete"
+ return storage_module.serialize(collection)
+ else:
+ # print "DEBUG: partial delete"
+ return delete_fn(collection,item)
+
+
def deserialize(obj,topological=False):
"""
Fill in an empty collection from disk or other storage
diff --git a/docs/wui.html b/docs/wui.html
index 66bfdc4..38a1db7 100644
--- a/docs/wui.html
+++ b/docs/wui.html
@@ -40,7 +40,7 @@ IRC channel.
</p>
<p>
-It is expected that you have read the <A HREF="<A HREF="/cobbler/webui/cobbler.html">Cobbler manpage</A>, which for the most part focuses on cobbler as run from the command line. You will need to use the command line some, so please do read the docs. For starters, you should have started your cobbler install with running "cobbler check" locally. If not,
+It is expected that you have read the <A HREF="/cobbler/webui/cobbler.html">Cobbler manpage</A>, which for the most part focuses on cobbler as run from the command line. You will need to use the command line some, so please do read the docs. For starters, you should have started your cobbler install with running "cobbler check" locally. If not,
please do so now before continuing. This will make sure your installation is configured and ready to go. The rest
of this document will mainly be detailing the differences between the CLI (as described in the manpage) and the Web interface.
</p>