Ongoing work on a better DVD import...

2 files changed, 157 insertions, 121 deletions

diff --git a/cobbler/action_import.py b/cobbler/action_import.py
index 47e68e3..770d031 100644
--- a/cobbler/action_import.py
+++ b/cobbler/action_import.py
@@ -19,47 +19,17 @@ import os
 import os.path
 import traceback
 import sub_process
+import glob
 
 import api
 
 WGET_CMD = "wget --mirror --no-parent --no-host-directories --directory-prefix %s/%s %s"
 RSYNC_CMD =  "rsync -a %s %s %s/ks_mirror/%s --exclude-from=/etc/cobbler/rsync.exclude --delete --delete-excluded --progress"
 
-# MATCH_LIST uses path segments of mirror URLs to assign kickstart
-# files.  It's not all that intelligent.
-# patches welcome!
-
-MATCH_LIST = (
-   ( "FC-5/"    , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "FC-6/"    , "/etc/cobbler/kickstart_fc6.ks" ),
-   ( "RHEL-4/"  , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "RHEL-5/"  , "/etc/cobbler/kickstart_fc6.ks" ),
-   ( "Centos/4" , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "Centos/5" , "/etc/cobbler/kickstart_fc6.ks" ),
-   ( "1/"       , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "2/"       , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "3/"       , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "4/"       , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "5/"       , "/etc/cobbler/kickstart_fc5.ks" ),
-   ( "6/"       , "/etc/cobbler/kickstart_fc6.ks" ),
-)
-
-# the following is a filter to reduce import scan times,
-# particularly over NFS.  these indicate directory segments
-# that we do not need to recurse into.  In the case where
-# these path segments are important to a certain distro import,
-# it's a bug, and this list needs to be edited.  please submit
-# patches or reports in this case.
-
-DIRECTORY_SIEVE = [
-   "debuginfo", "ppc", "s390x", "s390", "variant-src",
-   "ftp-isos", "compat-layer", "compat-layer-tree",
-   "SRPMS", "headers", "dosutils", "Publishers",
-   "LIVE", "RedHat", "image-template", "logs",
-   "EMEA", "APAC", "isolinux",
-   "debug", "repodata", "repoview", "Fedora",
-   "stylesheet-images", "buildinstall", "partner", "noarch",
-   "src-isos", "dvd-isos", "docs", "misc"
+TRY_LIST = [
+   "Fedora", "RedHat", "Client", "Server", "Centos",
+   "Fedora/RPMS", "RedHat/RPMS", "Client/RPMS", "Server/RPMS", "Centos/RPMS",
+   "RPMS"
 ]
 
 class Importer:
@@ -73,7 +43,8 @@ class Importer:
        self.profiles = config.profiles()
        self.systems  = config.systems()
        self.settings = config.settings()
-       self.serialize_counter = 0
+
+   # ----------------------------------------------------------------------
 
    def run(self):
        if self.mirror is None:
@@ -104,11 +75,14 @@ class Importer:
            self.run_this(RSYNC_CMD, (spacer, self.mirror, self.settings.webdir, self.mirror_name))
 
 
-       processed_repos = {}
-       os.path.walk(self.path, self.walker, processed_repos)
+       self.processed_repos = {}
+
+       os.path.walk(self.path, self.walker, None)
+
        self.guess_kickstarts()
        return True
 
+   # ----------------------------------------------------------------------
 
    def mkdir(self, dir):
        try:
@@ -116,10 +90,16 @@ class Importer:
        except:
            print "- didn't create %s" % dir
 
+   # ----------------------------------------------------------------------
+
    def run_this(self, cmd, args):
        my_cmd = cmd % args
        print "- %s" % my_cmd
-       sub_process.call(my_cmd,shell=True)
+       rc = sub_process.call(my_cmd,shell=True)
+       if rc != 0:
+          raise cexceptions.CobblerException("Command failed.")
+
+   # ----------------------------------------------------------------------
 
    def guess_kickstarts(self):
 
@@ -128,112 +108,167 @@ class Importer:
        at the kernel path, from that, see if we can guess the distro,
        and if we can, assign a kickstart if one is available for it.
        """
-       # FIXME: refactor this and make it more intelligent
-       # FIXME: if no distro can be found from the path, find through alternative means.
 
        for profile in self.profiles:
            distro = self.distros.find(profile.distro)
            if distro is None:
                raise cexceptions.CobblerException("orphan_distro2",profile.name,profile.distro)
-           kpath = distro.kernel
-           if not kpath.startswith("%s/ks_mirror/" % self.settings.webdir):
+           if not distro.kernel.startswith("%s/ks_mirror/" % self.settings.webdir):
+               # this isn't a mirrored profile, so we won't touch it
+               print "- skipping %s since profile isn't mirrored" % profile.name
+               continue
+           if distro.ks_meta.has_key("tree") or profile.ks_meta.has_key("tree"):
+               # this distro has already been imported, do not proceed
+               print "- skipping %s since existing tree attributes were found" % profile.name
                continue
-           for entry in MATCH_LIST:
-               (part, kickstart) = entry
-               if kpath.find(part) != -1:
-                   if os.path.exists(kickstart):
-                       print "*** ASSIGNING kickstart: %s" % kickstart
-                       profile.set_kickstart(kickstart)
-                       # from the kernel path, the tree path is always two up.
-                       dirname = os.path.dirname(kpath)
-                       print "dirname = %s" % dirname
-                       tokens = dirname.split("/")
-                       tokens = tokens[:-2]
-                       base = "/".join(tokens) 
-                       dest_link = os.path.join(self.settings.webdir, "links", distro.name)
-                       print "base=%s -> %s to %s" % (base, dest_link)
-                       if not os.path.exists(dest_link):
-                           os.symlink(base, dest_link)                       
-                       base = base.replace(self.settings.webdir,"")
-                       tree = "tree=http://%s/cblr/links/%s" % distro.name
-                       print "*** KICKSTART TREE = %s" % tree
-                       distro.set_ksmeta(tree)
-                       self.serialize_counter = self.serialize_counter + 1
-                       if (self.serialize_counter % 5) == 0:
+ 
+           kdir = os.path.dirname(distro.kernel)   
+           base_dir = "/".join(kdir.split("/")[0:-2])
+      
+           for try_entry in TRY_LIST:
+               for dnames in [ "fedora", "centos", "redhat" ]:
+                   try_dir = os.path.join(base_dir, try_entry)
+                   if os.path.exists(try_dir):
+                       rpms = glob.glob(os.path.join(try_dir, "*release-*"))
+                       for rpm in rpms:
+                           if rpm.find("notes") != -1:
+                               continue
+                           results = self.scan_rpm_filename(rpm)
+                           if results is None:
+                               continue
+                           (flavor, major, minor) = results
+                           print "- determining best kickstart for %s %s.%s" % (flavor, major, minor)          
+                           kickstart = self.set_kickstart(profile, flavor, major, minor)
+                           print "- kickstart=%s" % kickstart
+                           self.configure_tree_location(distro)
+                           self.distros.add(distro) # re-save
                            self.api.serialize()
 
-   def walker(self,processed_repos,dirname,fnames):
+   # --------------------------------------------------------------------
+
+   def configure_tree_location(self, distro):
+       # find the tree location
+       dirname = os.path.dirname(distro.kernel)
+       tokens = dirname.split("/")
+       tokens = tokens[:-2]
+       base = "/".join(tokens)
+       dest_link = os.path.join(self.settings.webdir, "links", distro.name)
+       if not os.path.exists(dest_link):
+           os.symlink(base, dest_link)
+       base = base.replace(self.settings.webdir,"")
+       tree = "tree=http://%s/cblr/links/%s" % (self.settings.server, distro.name)
+       print "- %s" % tree
+       distro.set_ksmeta(tree)
+
+   # ---------------------------------------------------------------------
+
+   def set_kickstart(self, profile, flavor, major, minor):
+       if flavor == "fedora":
+           if major >= 6:
+                return profile.set_kickstart("/etc/cobbler/kickstart_fc6.ks")
+       if flavor == "redhat" or flavor == "centos":
+           if major >= 5:
+                return profile.set_kickstart("/etc/cobbler/kickstart_fc6.ks")
+       print "- using default kickstart file choice"
+       return profile.set_kickstart("/etc/cobbler/kickstart_fc5.ks")
+
+   # ---------------------------------------------------------------------
+
+   def scan_rpm_filename(self, rpm):
+       rpm = os.path.basename(rpm)
+       (first, rest) = rpm.split("-release-")
+       flavor = first.lower()
+       (major, rest) = rest.split("-",1)
+       (minor, rest) = rest.split(".",1)
+       major = int(major)
+       minor = int(minor)
+       return (flavor, major, minor)
+
+   # ----------------------------------------------------------------------
+
+   def walker(self,foo,dirname,fnames):
+       
        initrd = None
        kernel = None
-       for tentative in fnames:
-           for filter_out in DIRECTORY_SIEVE:
-               if tentative == filter_out:
-                   fnames.remove(tentative)
-       print "%s" % dirname
-       if not self.is_pxe_or_virt_dir(dirname):
+       if not self.is_relevant_dir(dirname):
            return
-       # keep track of where we've run create repo
+
        for x in fnames:
            if x.startswith("initrd"):
                initrd = os.path.join(dirname,x)
            if x.startswith("vmlinuz"):
                kernel = os.path.join(dirname,x)
            if initrd is not None and kernel is not None:
-               self.add_entry(dirname,kernel,initrd)
-               # repo is up two locations and down in repodata
+               self.last_distro = self.add_entry(dirname,kernel,initrd)
                path_parts = kernel.split("/")[:-3]
                comps_path = "/".join(path_parts)
-               print "- looking for comps in %s" % comps_path
-               comps_file = os.path.join(comps_path, "repodata", "comps.xml")
-               if not os.path.exists(comps_file):
-                   print "- no comps file found: %s" % comps_file
-               try:
-                   # don't run creatrepo twice -- this can happen easily for Xen and PXE, when
-                   # they'll share same repo files.
-                   if not processed_repos.has_key(comps_path):
-                      cmd = "createrepo --basedir / --groupfile %s %s" % (comps_file, comps_path)
-                      print "- %s" % cmd
-                      sub_process.call(cmd,shell=True)
-                      print "- repository updated"
-                      processed_repos[comps_path] = 1
-               except:
-                   print "- error launching createrepo, ignoring for now..."
-                   traceback.print_exc()
+               print "- running repo update on %s" % comps_path
+               self.process_comps_file(comps_path)
+   # ----------------------------------------------------------------------
+               
+
+   def process_comps_file(self, comps_path):
+
+
+       comps_file = os.path.join(comps_path, "repodata", "comps.xml")
+       if not os.path.exists(comps_file):
+           print "- no comps file found: %s" % comps_file
+           return
+       try:
+           # don't run creatrepo twice -- this can happen easily for Xen and PXE, when
+           # they'll share same repo files.
+           if not self.processed_repos.has_key(comps_path):
+               cmd = "createrepo --basedir / --groupfile %s %s" % (comps_file, comps_path)
+               print "- %s" % cmd
+               sub_process.call(cmd,shell=True)
+               self.processed_repos[comps_path] = 1
+       except:
+           print "- error launching createrepo, ignoring..."
+           traceback.print_exc()
+        
 
    def add_entry(self,dirname,kernel,initrd):
        pxe_arch = self.get_pxe_arch(dirname)
        name = self.get_proposed_name(dirname)
-       if self.distros.find(name) is not None:
-           print "already registered: %s" % name
+
+       existing_distro = self.distros.find(name)
+
+       if existing_distro is not None:
+           print "- modifying existing distro: %s" % name
+           distro = existing_distro
        else:
+           print "- creating new distro: %s" % name
            distro = self.config.new_distro()
-           distro.set_name(name)
-           distro.set_kernel(kernel)
-           distro.set_initrd(initrd)
-           distro.set_arch(pxe_arch)
-           self.distros.add(distro)
-           print "(distro added)"
-           if self.profiles.find(name) is None:
-               profile = self.config.new_profile()
-               profile.set_name(name)
-               profile.set_distro(name)
-               self.profiles.add(profile)
-               print "(profile added)"
-           self.serialize_counter = self.serialize_counter + 1
-           if (self.serialize_counter % 5) == 0:
-               self.api.serialize()
+           
+       distro.set_name(name)
+       distro.set_kernel(kernel)
+       distro.set_initrd(initrd)
+       distro.set_arch(pxe_arch)
+       self.distros.add(distro)
+       
+       existing_profile = self.profiles.find(name) 
+
+       if existing_profile is None:
+           print "- creating new profile: %s" % name 
+           profile = self.config.new_profile()
+       else:
+           print "- modifying existing profile: %s" % name
+           profile = existing_profile
+
+       profile.set_name(name)
+       profile.set_distro(name)
+
+       self.profiles.add(profile)
+       self.api.serialize()
+
+       return distro
 
    def get_proposed_name(self,dirname):
        name = "-".join(dirname.split("/"))
        if name.startswith("-"):
           name = name[1:]
-       # some of this filtering is a bit excessive though we want to compensate for
-       # finding "tree" vs "image" in the path and so on, and being a little more
-       # aggressive in filtering will reduce path name lengths in all circumstances.
-       # long paths are bad because they are hard to type, look weird, and run up
-       # against the 255 char kernel options limit too quickly.
        name = name.replace("var-www-cobbler-", "")
-       name = name.replace("ks-mirror-","")
+       name = name.replace("ks_mirror-","")
        name = name.replace("os-images-","")
        name = name.replace("tree-images-","")
        name = name.replace("images-","")
@@ -242,7 +277,7 @@ class Importer:
        return name
 
    def get_pxe_arch(self,dirname):
-       t = dirname
+       t = dirname.lower()
        if t.find("x86_64") != -1:
           return "x86_64"
        if t.find("ia64") != -1:
@@ -251,8 +286,9 @@ class Importer:
           return "x86"
        return "x86"
 
-   def is_pxe_or_virt_dir(self,dirname):
-       if dirname.find("pxe") != -1 or dirname.find("xen") != -1 or dirname.find("virt") != -1:
-           return True
+   def is_relevant_dir(self,dirname):
+       for x in [ "pxe", "xen", "virt" ]:
+           if dirname.find(x) != -1:
+               return True
        return False
 
diff --git a/cobbler/action_sync.py b/cobbler/action_sync.py
index cee56e6..decadd1 100644
--- a/cobbler/action_sync.py
+++ b/cobbler/action_sync.py
@@ -272,7 +272,7 @@ class BootSync:
                 if not x.endswith(".py"):
                     self.rmfile(path)
             if os.path.isdir(path):
-                if not x in ["localmirror","repo_mirror","ks_mirror","kickstarts","kickstarts_sys","distros","images","systems","profiles"] :
+                if not x in ["localmirror","repo_mirror","ks_mirror","kickstarts","kickstarts_sys","distros","images","systems","profiles","links"] :
                     # delete directories that shouldn't exist
                     self.rmtree(path)
                 if x in ["kickstarts","kickstarts_sys","images","systems","distros","profiles"]:
@@ -668,7 +668,7 @@ class BootSync:
                 pxe_fn = self.get_pxe_filename(system.name)
                 kickstart_path = "http://%s/cblr/kickstarts_sys/%s/ks.cfg" % (self.settings.server, pxe_fn)
             elif kickstart_path.startswith("/") or kickstart_path.find("/cobbler/kickstarts/") != -1:
-                kickstart_path = "http://%s/cobbler_track/kickstarts/%s/ks.cfg" % (self.settings.server, profile.name)
+                kickstart_path = "http://%s/cblr/kickstarts/%s/ks.cfg" % (self.settings.server, profile.name)
 
             if distro.breed is None or distro.breed == "redhat":
                 append_line = "%s ks=%s" % (append_line, kickstart_path)
@@ -772,7 +772,7 @@ class BootSync:
 
         # make URLs for koan if the kickstart files are locally managed (which is preferred)
         if clone.kickstart and clone.kickstart.startswith("/"):
-            clone.kickstart = "http://%s/cobbler_track/kickstarts/%s/ks.cfg" % (self.settings.server, clone.name)
+            clone.kickstart = "http://%s/cblr/kickstarts/%s/ks.cfg" % (self.settings.server, clone.name)
         self.tee(fd,yaml.dump(clone.to_datastruct()))
         self.close_file(fd)