Added feature where a network install location can be given to do cobbler imports without

mirroring.  This is explained in the manpage and online docs.

5 files changed, 153 insertions, 51 deletions

diff --git a/cobbler/action_import.py b/cobbler/action_import.py
index c3e7b1a..bf42dbc 100644
--- a/cobbler/action_import.py
+++ b/cobbler/action_import.py
@@ -26,7 +26,7 @@ import shutil
 from rhpl.translate import _, N_, textdomain, utf8
 
 WGET_CMD = "wget --mirror --no-parent --no-host-directories --directory-prefix %s/%s %s"
-RSYNC_CMD =  "rsync -a %s %s %s/ks_mirror/%s --exclude-from=/etc/cobbler/rsync.exclude --delete --delete-excluded --progress"
+RSYNC_CMD =  "rsync -a %s %s %s/ks_mirror/%s --exclude-from=/etc/cobbler/rsync.exclude --progress"
 
 TRY_LIST = [
    "Fedora", "RedHat", "Client", "Server", "Centos", "CentOS",
@@ -36,11 +36,22 @@ TRY_LIST = [
 
 class Importer:
 
-   def __init__(self,api,config,mirror,mirror_name):
+   def __init__(self,api,config,mirror,mirror_name,network_root=None):
+       """
+       Performs an import of a install tree (or trees) from the given
+       mirror address.  The prefix of the distro is to be specified
+       by mirror name.  For instance, if FC-6 is given, FC-6-xen-i386
+       would be a potential distro that could be created.  For content
+       available on external servers via a known nfs:// or ftp:// or
+       http:// path, we can import without doing rsync mirorring to 
+       cobbler's http directory.  This is explained in more detail 
+       in the manpage.  Leave network_root to None if want mirroring.
+       """
        self.api = api
        self.config = config
        self.mirror = mirror
        self.mirror_name = mirror_name
+       self.network_root = network_root 
        self.distros  = config.distros()
        self.profiles = config.profiles()
        self.systems  = config.systems()
@@ -58,33 +69,53 @@ class Importer:
        if self.mirror_name is None:
            raise CX(_("import failed.  no --name specified"))
        
-       # make the output path
-       self.path = "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name)
-       self.mkdir(self.path)
-
-       # prevent rsync from creating the directory name twice
-       if not self.mirror.endswith("/"):
-           self.mirror = "%s/" % self.mirror 
-
-       if self.mirror.startswith("http://"):
-           # http mirrors are kind of primative.  rsync is better.
-           self.run_this(WGET_CMD, (self.settings.webdir, self.mirror_name, self.mirror))
-       else:
-           # use rsync.. no SSH for public mirrors and local files.
-           # presence of user@host syntax means use SSH
-           spacer = ""
-           if not self.mirror.startswith("rsync://") and not self.mirror.startswith("/"):
-               spacer = ' -e "ssh" '
-           self.run_this(RSYNC_CMD, (spacer, self.mirror, self.settings.webdir, self.mirror_name))
-
+       # make the output path and mirror content but only if not specifying that a network
+       # accessible support location already exists
+
+       if self.network_root is None:
+           self.path = "%s/ks_mirror/%s" % (self.settings.webdir, self.mirror_name)
+           self.mkdir(self.path)
+
+           # prevent rsync from creating the directory name twice
+           if not self.mirror.endswith("/"):
+               self.mirror = "%s/" % self.mirror 
+
+           if self.mirror.startswith("http://"):
+               # http mirrors are kind of primative.  rsync is better.
+               # that's why this isn't documented in the manpage.
+               # TODO: how about adding recursive FTP as an option?
+               self.run_this(WGET_CMD, (self.settings.webdir, self.mirror_name, self.mirror))
+           else:
+               # use rsync.. no SSH for public mirrors and local files.
+               # presence of user@host syntax means use SSH
+               spacer = ""
+               if not self.mirror.startswith("rsync://") and not self.mirror.startswith("/"):
+                   spacer = ' -e "ssh" '
+               self.run_this(RSYNC_CMD, (spacer, self.mirror, self.settings.webdir, self.mirror_name))
+
+       # see that the root given is valid
+
+       if self.network_root is not None:
+           if not self.network_root.endswith("/"):
+               self.network_root = self.network_root + "/"
+           self.path = self.mirror
+           found_root = False
+           valid_roots = [ "nfs://", "ftp://", "http://" ]
+           for valid_root in valid_roots:
+               if self.network_root.startswith(valid_root):
+                   found_root = True
+           if not found_root:
+               raise CX(_("Network root given to --available-as must be nfs://, ftp://, or http://"))
 
        self.processed_repos = {}
 
        print _("---------------- (adding distros)")
        os.path.walk(self.path, self.distro_adder, {})
 
-       print _("---------------- (associating repos)")
-       self.repo_finder()
+       if self.network_root is None:
+           print _("---------------- (associating repos)")
+           # FIXME: this automagic is not possible (yet) without mirroring 
+           self.repo_finder()
 
        print _("---------------- (associating kickstarts)")
        self.kickstart_finder() 
@@ -114,7 +145,6 @@ class Importer:
    # ----------------------------------------------------------------------
 
    def kickstart_finder(self):
-
        """
        For all of the profiles in the config w/o a kickstart, look
        at the kernel path, from that, see if we can guess the distro,
@@ -126,10 +156,13 @@ class Importer:
            if distro is None or not (distro in self.distros_added):
                print _("- skipping distro %s since it wasn't imported this time") % profile.distro
                continue
-           if not distro.kernel.startswith("%s/ks_mirror/" % self.settings.webdir):
-               # this isn't a mirrored profile, so we won't touch it
-               print _("- skipping %s since profile isn't mirrored") % profile.name
-               continue
+
+           # THIS IS OBSOLETE:
+           #
+           #if not distro.kernel.startswith("%s/ks_mirror/" % self.settings.webdir):
+           #    # this isn't a mirrored profile, so we won't touch it
+           #    print _("- skipping %s since profile isn't mirrored") % profile.name
+           #    continue
  
            kdir = os.path.dirname(distro.kernel)   
            base_dir = "/".join(kdir.split("/")[0:-2])
@@ -145,9 +178,8 @@ class Importer:
                        if results is None:
                            continue
                        (flavor, major, minor) = results
-                       print _("- determining best kickstart for %(flavor)s %(major)s") % { "flavor" : flavor, "major" : major }
+                       print _("- finding default kickstart template for %(flavor)s %(major)s") % { "flavor" : flavor, "major" : major }
                        kickstart = self.set_kickstart(profile, flavor, major, minor)
-                       print _("- kickstart=%s") % kickstart
                        self.configure_tree_location(distro)
                        self.distros.add(distro) # re-save
                        self.api.serialize()
@@ -161,21 +193,55 @@ class Importer:
        tokens = tokens[:-2]
        base = "/".join(tokens)
        dest_link = os.path.join(self.settings.webdir, "links", distro.name)
-       if not os.path.exists(dest_link):
-           try:
-               os.symlink(base, dest_link)
-           except:
-               # this shouldn't happen but I've seen it ... debug ...
-               print _("- symlink creation failed: %(base)s, %(dest)s") % { "base" : base, "dest" : dest_link }
-       base = base.replace(self.settings.webdir,"")
+
+       # create the links directory only if we are mirroring because with
+       # SELinux Apache can't symlink to NFS (without some doing)
+
+       if self.network_root is None:
+           if not os.path.exists(dest_link):
+               try:
+                   os.symlink(base, dest_link)
+               except:
+                   # this shouldn't happen but I've seen it ... debug ...
+                   print _("- symlink creation failed: %(base)s, %(dest)s") % { "base" : base, "dest" : dest_link }
+
+           # FIXME: looks like "base" isn't used later.  remove?
+           base = base.replace(self.settings.webdir,"")
        
        meta = distro.ks_meta
-       meta["tree"] = "http://%s/cblr/links/%s" % (self.settings.server, distro.name)
+
+       # how we set the tree depends on whether an explicit network_root was specified
+       if self.network_root is None:
+           meta["tree"] = "http://%s/cblr/links/%s" % (self.settings.server, distro.name)
+       else:
+           # where we assign the kickstart source is relative to our current directory
+           # and the input start directory in the crawl.  We find the path segments
+           # between and tack them on the network source path to find the explicit
+           # network path to the distro that Anaconda can digest.  
+           tail = self.path_tail(self.mirror, base)
+           meta["tree"] = self.network_root
+           if meta["tree"].endswith("/"):
+              meta["tree"] = self.network_root[:-1]
+           meta["tree"] = meta["tree"] + tail.rstrip()
+
        print _("- tree: %s") % meta["tree"]
        distro.set_ksmeta(meta)
 
    # ---------------------------------------------------------------------
 
+   def path_tail(self, apath, bpath):
+       """ 
+       Given two paths (B is longer than A), find the part in B not in A
+       """
+       position = bpath.find(apath)
+       if position != 0:
+           print "%s, %s, %s" % (apath, bpath, position)
+           raise CX(_("Error: possible symlink traversal?: %s") % bpath)
+       rposition = position + len(self.mirror)
+       return bpath[rposition:]
+
+   # ---------------------------------------------------------------------
+
    def set_kickstart(self, profile, flavor, major, minor):
        if flavor == "fedora":
            if major >= 6:
@@ -395,9 +461,15 @@ class Importer:
        archname = pxe_arch
        if archname == "x86":
           archname = "i386"
-       name = "-".join(dirname.split("/"))
+       # FIXME: this is new, needs testing ...
+       if self.network_root is not None:
+          name = "-".join(self.path_tail(self.path,dirname).split("/"))
+       else:
+          # remove the part that says /var/www/cobbler/ks_mirror/name
+          name = "-".join(dirname.split("/")[6:])
        if name.startswith("-"):
           name = name[1:]
+       name = self.mirror_name + "-" + name
        name = name.replace("-os","")
        name = name.replace("-images","")
        name = name.replace("-tree","")
diff --git a/cobbler/api.py b/cobbler/api.py
index bd597b2..6268283 100644
--- a/cobbler/api.py
+++ b/cobbler/api.py
@@ -177,12 +177,17 @@ class BootAPI:
         statusifier = action_status.BootStatusReport(self._config, mode)
         return statusifier.run()
 
-    def import_tree(self,mirror_url,mirror_name):
+    def import_tree(self,mirror_url,mirror_name,network_root=None):
         """
         Automatically import a directory tree full of distribution files.
-        mirror_url can be a string that represents a path, a user@host syntax for SSH, or an rsync:// address
-        """
-        importer = action_import.Importer(self, self._config, mirror_url, mirror_name)
+        mirror_url can be a string that represents a path, a user@host 
+        syntax for SSH, or an rsync:// address.  If mirror_url is a 
+        filesystem path and mirroring is not desired, set network_root 
+        to something like "nfs://path/to/mirror_url/root" 
+        """
+        importer = action_import.Importer(
+            self, self._config, mirror_url, mirror_name, network_root
+        )
         return importer.run()
 
     def serialize(self):
diff --git a/cobbler/cobbler.py b/cobbler/cobbler.py
index 0a8ae33..13daabd 100755
--- a/cobbler/cobbler.py
+++ b/cobbler/cobbler.py
@@ -671,19 +671,25 @@ class BootCLI:
         """
         self.temp_mirror = None
         self.temp_mirror_name = None
+        self.temp_network_root = None
         def set_mirror_name(a):
             self.temp_mirror_name = a
         def set_mirror(a):
             self.temp_mirror = a
+        def set_network_root(a):
+            self.temp_network_root = a
         def go_import():
             return self.api.import_tree(
                 self.temp_mirror,
-                self.temp_mirror_name)
+                self.temp_mirror_name,
+                network_root=self.temp_network_root
+            )
         commands = {
             '--path'         : lambda(a): set_mirror(a),
             '--mirror'       : lambda(a): set_mirror(a),
             '--mirror-name'  : lambda(a): set_mirror_name(a),
-            '--name'         : lambda(a): set_mirror_name(a)
+            '--name'         : lambda(a): set_mirror_name(a),
+            '--available-as' : lambda(a): set_network_root(a)
         }
         on_ok = lambda: go_import()
         return self.apply_args(args,commands,on_ok)
diff --git a/docs/cobbler.pod b/docs/cobbler.pod
index 60ada41..5374aef 100644
--- a/docs/cobbler.pod
+++ b/docs/cobbler.pod
@@ -42,7 +42,7 @@ in network booting via PXE and just want to use koan to install virtual systems,
 
 This first step towards configurating what you want to provision is to add a distribution to the cobbler's configuration.  
 
-If there is an rsync mirror, DVD, or filesystem tree available that you would rather import instead, skip down to the documentation about the "import" command.  It's really a lot easier, and it only requires waiting for the mirror content to be copied.  Imported mirrors also save time during install since they don't have to hit external install sources.   However if you have the content locally already and don't want to create another copy of it, you'll want to use the manual distro add commands.
+If there is an rsync mirror, DVD, NFS, or filesystem tree available that you would rather import instead, skip down to the documentation about the "import" command.  It's really a lot easier, and it only requires waiting for the mirror content to be copied.  Imported mirrors also save time during install since they don't have to hit external install sources.   However if you have the content locally already and don't want to create another copy of it, you'll want to use the manual distro add commands.
 
 B<cobbler distro add --name=string --kernel=path --initrd=path [--kopts=string] [--ksmeta=string] [--arch=x86|x86_64|ia64] [--breed=redhat|debian|suse]>
 
@@ -364,15 +364,20 @@ This example shows how to create a provisioning infrastructure from a distributi
 Then a default PXE configuration is created, so that by default systems will PXE boot into 
 a fully automated install process for that distribution.
 
-You can use a network rsync mirror or a mounted DVD location.
+You can use a network rsync mirror, a mounted DVD location, or a tree you have available
+via a network filesystem.
 
 B<cobbler check>
 
-B<cobbler import --mirror=rsync://yourfavoritemirror.com/foo --name=anyname>
+B<cobbler import --path=rsync://yourfavoritemirror.com/foo --name=anyname>
 
 # OR 
 
-B<cobbler import --mirror=/mnt/dvd --name=anyname>
+B<cobbler import --path=/mnt/dvd --name=anyname>
+
+# OR (using an eternal NAS box without mirroring)
+
+B<cobbler import --path=/path/where/filer/is/mounted --name=anyname --available-as=nfs://nfs.example.org:/where/mounted/>
 
 # wait for mirror to rsync...
 
@@ -540,7 +545,7 @@ Enterprising users will notice this is not much more than a wrapper around SSH c
 =head2 IMPORTING TREES
 
 
-Cobbler can auto-add distributions and profiles from remote sources, whether this is a filesystem path or an rsync mirror.  This can save a lot of time when setting up a new provisioning environment.  This is a feature that many users will want to take advantage of, and is very simple to use.
+Cobbler can auto-add distributions and profiles from remote sources, whether this is a filesystem path or an rsync mirror.  This can save a lot of time when setting up a new provisioning environment.  Import is a feature that many users will want to take advantage of, and is very simple to use.
   
 After an import is run, cobbler will try to detect the distribution type and automatically assign kickstarts.  By default, it will provision the system by erasing the hard drive, setting up eth0 for dhcp, and using a default password of "cobbler".  If this is undesirable, edit the kickstart files in /etc/cobbler to do something else or change the kickstart setting after cobbler creates the profile.
 
@@ -554,10 +559,14 @@ Example3:  B<cobbler import --mirror=/mnt/dvd --name=baz>
 
 Example4:  B<cobbler import --mirror=/path/to/stuff --name=glorp>
 
+Example5:  B<cobbler import --path=/path/where/filer/is/mounted --name=anyname --available-as=nfs://nfs.example.org:/where/mounted/>
+
 Once imported, run a "cobbler list" or "cobbler report" to see what you've added.
 
 By default, the rsync operations will exclude PPC content, debug RPMs, and ISO images -- to change what is excluded during an import, see /etc/cobbler/rsync.exclude.
 
+Note that all of the import commands will mirror install tree content into /var/www/cobbler unless a network accessible location is given with --available-as.  --available-as will be primarily used when importing distros stored on an external NAS box, or potentially on another partition on the same machine that is already accessible via http:// or ftp://.
+
 =head2 DEFAULT PXE BOOT BEHAVIOR
 
 What happens when PXE booting a system when cobbler has no record
diff --git a/website/new/docs/cobbler-import.html b/website/new/docs/cobbler-import.html
index ce71cf6..500087e 100644
--- a/website/new/docs/cobbler-import.html
+++ b/website/new/docs/cobbler-import.html
@@ -5,7 +5,7 @@
 <a name="import">
 <h2>About</h2>
 <p>
-Suppose you want to set up a network boot server for a given distro, really really quickly. For this example, we'll use Fedora Core 6 as an example. What's the best way to do this? It's a command called "cobbler import", that can mirror content based on a DVD image or even an external rsync mirror or SSH location.
+Suppose you want to set up a network boot server for a given distro, really really quickly. For this example, we'll use Fedora Core 6 as an example. What's the best way to do this? It's a command called "cobbler import", that can mirror content based on a DVD image, a tree on a filesystem, or even an external rsync mirror or SSH location.
 </p>
 
 <a name="provserver">
@@ -37,6 +37,16 @@ This would mirror from a public rsync server without needing the DVD image.
 <p>
 The kickstarts used above will provision machines with a default password of "cobbler" and a really basic set of packages. For something more complicated, you may wish to edit the default kickstarts in /etc/cobbler and then re-run "cobbler sync" to apply those changes.
 </p>
+<p>
+What if you don't want to mirror the content?  Say you already have the trees from all your DVDs and/or CDs extracted on a Filer mounted over NFS somewhere.  This works too, with the addition of one more argument, in versions 0.5.3 and later...
+</p>
+<p>
+<blockquote>
+cobbler import --mirror=/path/where/filer/is/mounted --name=filer --available-as=nfs://nfsserver.example.org:/is/mounted/here
+</blockquote>
+</p>
+<p>
+The above command will set up cobbler automatically using all of the above distros (stored on the filer) as a basis -- but will keep the trees on NFS.  This saves disk space on the Cobbler server.  As you add more distros over time to the filer, you can keep running the above commands to add them to Cobbler.  So, whether using some data you already have on the network, or letting cobbler create an install mirror for you, there are lots of useful options.  </p>
 
 <a name="#reinstalls">
 <h2>Using That Server For Reinstallation</h2>