From 28ebf5f306b0f60b7113d602c58e44acdc131ebc Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Tue, 10 Apr 2007 18:27:39 -0400 Subject: This commit adds a --rpm-list parameter to "cobbler repo add". --rpm-list allows for partial mirroring of RPM content from a yum repository using yumdownloader. An example of this would be wanting to have a local mirror of useful tools from FC6 Extras (cobbler and koan, possibly?) while not pulling down content that just takes up time/space (like 3D games). This will work for http:// and ftp:// repositories, but not RHN at this point. Incidentally this feature doesn't resolve dependencies at this point because yumdownloader is currently broken. See https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=232183 So, for now, give all dependencies for --rpm-list and when yumdownloader gets fixed, the "--resolve" argument can be reinserted and this will be a lot more useful. --- CHANGELOG | 2 + cobbler/action_reposync.py | 98 ++++++++++++++++++++++++++++++++++++++++------ cobbler/cobbler.py | 3 +- cobbler/item_repo.py | 26 ++++++++++-- docs/cobbler.pod | 12 ++++++ 5 files changed, 126 insertions(+), 15 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 99446d1..43b30ce 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,8 @@ Cobbler CHANGELOG * TBA - 0.4.7 - Disable mod_python tracker piece for RHEL5 (replacement eventual). +- Added support for --rpm-list parameter to "repo add" for download of partial content from repositories + (ex: cobbler and koan from FC6extras, w/o games). * Thu Apr 05 2007 - 0.4.6 - Bind cobbler_syslogd to all addresses diff --git a/cobbler/action_reposync.py b/cobbler/action_reposync.py index fc01061..1801071 100644 --- a/cobbler/action_reposync.py +++ b/cobbler/action_reposync.py @@ -32,6 +32,8 @@ class RepoSync: Handles conversion of internal state to the tftpboot tree layout """ + # ================================================================================== + def __init__(self,config): """ Constructor @@ -43,6 +45,8 @@ class RepoSync: self.systems = config.systems() self.settings = config.settings() self.repos = config.repos() + + # ================================================================================== def run(self,verbose=True): """ @@ -64,6 +68,8 @@ class RepoSync: self.do_rsync(repo) return True + + # ================================================================================== def do_reposync(self,repo): @@ -72,12 +78,25 @@ class RepoSync: FIXME: support for mirrorlist? """ + # warn about not having yum-utils. We don't want to require it in the package because + # RHEL4 and RHEL5U0 don't have it. + if not os.path.exists("/usr/bin/reposync"): raise cexceptions.CobblerException("no /usr/bin/reposync found, please install yum-utils") - is_rhn = False + cmds = [] # queues up commands to run + is_rhn = False # RHN repositories require extra black magic + has_rpm_list = False # flag indicating not to pull the whole repo + + # detect cases that require special handling + if repo.mirror.lower().startswith("rhn://"): is_rhn = True + if repo.rpm_list != "": + has_rpm_list = True + + # user might have disabled repo updates in the config file for whatever reason. + # if so, don't update this one. if not repo.keep_updated: print "- %s is set to not be updated" % repo.name @@ -89,35 +108,86 @@ class RepoSync: temp_path = os.path.join(store_path, ".origin") if not os.path.isdir(temp_path) and not is_rhn: # if doing the rhn sync, reposync will make the directory + # otherwise, we need to do it explicitly os.makedirs(temp_path) + # how we invoke yum-utils depends on whether this is RHN content or not. + if not is_rhn: - # rhn sync takes different params + + # this is the simple non-RHN case. + # create the config file that yum will use for the copying + temp_file = self.create_local_file(repo, temp_path, output=False) - cmd = "/usr/bin/reposync --config=%s --repoid=%s --download_path=%s" % (temp_file, repo.name, store_path) - print "- %s" % cmd + + if not has_rpm_list: + + # if we have not requested only certain RPMs, use reposync + cmd = "/usr/bin/reposync --config=%s --repoid=%s --download_path=%s" % (temp_file, repo.name, store_path) + print "- %s" % cmd + cmds.append(cmd) + + else: + + # create the output directory if it doesn't exist + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + # if we only want certain RPMs, use yumdownloader (likely more than once) + # FIXME: yumdownloader has a current bug where --resolve blows up + # removing --resolve until I get the email from bugzilla saying it's fixed. + cmd = "/usr/bin/yumdownloader --config=%s --destdir=%s %s" %(temp_file, dest_path, " ".join(repo.rpm_list)) + print "- %s" % cmd + cmds.append(cmd) else: - # this requires that you have entitlements for the server and you give the mirror as rhn://$channelname - rest = repo.mirror[6:] + + # this is the somewhat more-complex RHN case. + # NOTE: this requires that you have entitlements for the server and you give the mirror as rhn://$channelname + + if has_rpm_list: + print "- warning: --rpm-list is not supported for RHN content" + rest = repo.mirror[6:] # everything after rhn:// cmd = "/usr/bin/reposync -r %s --download_path=%s" % (rest, store_path) print "- %s" % cmd - # downloads using -r use the value given for -r as part of the output dir, so create a symlink with the name the user + cmds.append(cmd) + + # downloads using -r use the value given for -r as part of the output dir, + # so create a symlink with the name the user # gave such that everything still works as intended and the sync code still works + # this doesn't happen for the http:// and ftp:// mirrors. + if not os.path.exists(dest_path): from1 = os.path.join(self.settings.webdir, "repo_mirror", rest) print "- symlink: %s -> %s" % (from1, dest_path) os.symlink(from1, dest_path) - rc = sub_process.call(cmd, shell=True) + + # now regardless of whether we're doing yumdownloader or reposync + # or whether the repo was http://, ftp://, or rhn://, execute all queued + # commands here. Any failure at any point stops the operation. + + for cmd in cmds: + rc = sub_process.call(cmd, shell=True) + if rc !=0: + raise cexceptions.CobblerException("cobbler reposync failed") + + # some more special case handling for RHN. + # create the config file now, because the directory didn't exist earlier + if is_rhn: - # now that the directory exists, we can create the config file. this is different from the normal case. temp_file = self.create_local_file(repo, temp_path, output=False) - if rc !=0: - raise cexceptions.CobblerException("cobbler reposync failed") + + # now run createrepo to rebuild the index + arg = None os.path.walk(dest_path, self.createrepo_walker, arg) + # create the config file the hosts will use to access the repository. + self.create_local_file(repo, dest_path) + + # ================================================================================== + def do_rsync(self,repo): """ @@ -127,6 +197,8 @@ class RepoSync: if not repo.keep_updated: print "- %s is set to not be updated" % repo.name return True + if repo.rpm_list != "": + print "- warning: --rpm-list is not supported for rsync'd repositories" dest_path = os.path.join(self.settings.webdir, "repo_mirror", repo.name) spacer = "" if not repo.mirror.startswith("rsync://") and not repo.mirror.startswith("/"): @@ -142,6 +214,8 @@ class RepoSync: print "- walking: %s" % dest_path os.path.walk(dest_path, self.createrepo_walker, arg) self.create_local_file(repo, dest_path) + + # ================================================================================== def create_local_file(self, repo, dest_path, output=True): """ @@ -167,6 +241,8 @@ class RepoSync: config_file.close() return fname + # ================================================================================== + def createrepo_walker(self, arg, dirname, fname): """ Used to run createrepo on a copied mirror. diff --git a/cobbler/cobbler.py b/cobbler/cobbler.py index 03a07e7..36c457f 100755 --- a/cobbler/cobbler.py +++ b/cobbler/cobbler.py @@ -359,7 +359,8 @@ class BootCLI: '--mirror-name' : lambda(a): repo.set_name(a), '--mirror' : lambda(a): repo.set_mirror(a), '--keep-updated' : lambda(a): repo.set_keep_updated(a), - '--local-filename' : lambda(a): repo.set_local_filename(a) + '--local-filename' : lambda(a): repo.set_local_filename(a), + '--rpm-list' : lambda(a): repo.set_rpm_list(a) } on_ok = lambda: self.api.repos().add(repo) return self.apply_args(args,commands,on_ok) diff --git a/cobbler/item_repo.py b/cobbler/item_repo.py index b90811d..79654cd 100644 --- a/cobbler/item_repo.py +++ b/cobbler/item_repo.py @@ -30,12 +30,14 @@ class Repo(item.Item): self.mirror = None # is required self.keep_updated = 1 # has reasonable defaults self.local_filename = "" # off by default + self.rpm_list = "" # just get selected RPMs + deps def from_datastruct(self,seed_data): - self.name = self.load_item(seed_data,'name') - self.mirror = self.load_item(seed_data,'mirror') + self.name = self.load_item(seed_data, 'name') + self.mirror = self.load_item(seed_data, 'mirror') self.keep_updated = self.load_item(seed_data, 'keep_updated') self.local_filename = self.load_item(seed_data, 'local_filename') + self.rpm_list = self.load_item(seed_data, 'rpm_list') return self def set_name(self,name): @@ -80,6 +82,22 @@ class Repo(item.Item): self.local_filename = fname return True + def set_rpm_list(self,rpms): + """ + Rather than mirroring the entire contents of a repository (Fedora Extras, for instance, + contains games, and we probably don't want those), make it possible to list the packages + one wants out of those repos, so only those packages + deps can be mirrored. + """ + if type(rpms) != list: + rpmlist = rpms.split(None) + else: + rpmlist = rpms + try: + rpmlist.remove('') + except: + pass + self.rpm_list = rpmlist + def is_valid(self): """ A repo is valid if it has a name and a mirror URL @@ -95,7 +113,8 @@ class Repo(item.Item): 'name' : self.name, 'mirror' : self.mirror, 'keep_updated' : self.keep_updated, - 'local_filename' : self.local_filename + 'local_filename' : self.local_filename, + 'rpm_list' : self.rpm_list } def printable(self): @@ -103,5 +122,6 @@ class Repo(item.Item): buf = buf + "mirror : %s\n" % self.mirror buf = buf + "keep updated : %s\n" % self.keep_updated buf = buf + "local filename : %s\n" % self.local_filename + buf = buf + "rpm list : %s\n" % self.rpm_list return buf diff --git a/docs/cobbler.pod b/docs/cobbler.pod index 5bc8021..dfe8985 100644 --- a/docs/cobbler.pod +++ b/docs/cobbler.pod @@ -205,6 +205,8 @@ Experimental support is also provided for mirroring (RHEL5) and later RHN conten a fast local mirror. The mirror syntax for this is --mirror=rhn://channel-name and you must have entitlements for this to work. +http://, ftp:// and rhn:// mirrors require yum-utils be installed. + =item name This name is used as the save location for the mirror. If the mirror represented, say, Fedora Core @@ -230,6 +232,16 @@ still be used for installation, it just won't get installed automatically in /et See /etc/cobbler/kickstart_fc6.ks for an example of how to employ this within a kickstart template. +=item rpm-list + +By specifying a space-delimited list of package names for --rpm-list, one can decide to mirror only a part +of a repo (the list of packages given, plus dependencies). This may be helpful in conserving time/space/bandwidth. +For intance, when mirroring FC6 Extras, it may be desired to mirror just cobbler and koan, and skip all of the +games. To do this, use --rpm-list="cobbler koan". + +This option only works for http:// and ftp:// repositories. It will be ignored for other +mirror types, such as local paths and rsync:// mirrors. This option requires yum-utils be installed. + =back =head2 DISPLAYING CONFIGURATION ENTRIES -- cgit