diff options
Diffstat (limited to 'createprestorepo')
| -rw-r--r-- | createprestorepo/COPYING | 339 | ||||
| -rw-r--r-- | createprestorepo/ChangeLog | 2 | ||||
| -rw-r--r-- | createprestorepo/Makefile | 11 | ||||
| -rw-r--r-- | createprestorepo/README | 22 | ||||
| -rwxr-xr-x | createprestorepo/createprestorepo | 2 | ||||
| -rw-r--r-- | createprestorepo/dumpMetadata.py | 496 | ||||
| -rwxr-xr-x | createprestorepo/gendeltarpms.py | 216 | ||||
| -rwxr-xr-x | createprestorepo/genprestometadata.py | 475 | ||||
| -rwxr-xr-x | createprestorepo/makedeltarepo | 2 | ||||
| -rw-r--r-- | createprestorepo/packagelist.py | 385 |
10 files changed, 0 insertions, 1950 deletions
diff --git a/createprestorepo/COPYING b/createprestorepo/COPYING deleted file mode 100644 index e77696a..0000000 --- a/createprestorepo/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 675 Mass Ave, Cambridge, MA 02139, USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) 19yy <name of author> - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) 19yy name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - <signature of Ty Coon>, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/createprestorepo/ChangeLog b/createprestorepo/ChangeLog deleted file mode 100644 index 2de29c0..0000000 --- a/createprestorepo/ChangeLog +++ /dev/null @@ -1,2 +0,0 @@ -* Tue May 7 2007 Jonathan Dieter <jdieter@gmail.com> - 0.1.0 - - Initial release diff --git a/createprestorepo/Makefile b/createprestorepo/Makefile deleted file mode 100644 index 51228f8..0000000 --- a/createprestorepo/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -clean: - rm -f *.pyc *.pyo *~ - -install: - install -m 755 createprestorepo $(DESTDIR)/usr/bin/ - install -m 755 makedeltarepo $(DESTDIR)/usr/bin/ - mkdir -p $(DESTDIR)/usr/share/createprestorepo - install -m 755 genprestometadata.py $(DESTDIR)/usr/share/createprestorepo - install -m 644 dumpMetadata.py $(DESTDIR)/usr/share/createprestorepo - install -m 755 gendeltarpms.py $(DESTDIR)/usr/share/createprestorepo - install -m 644 packagelist.py $(DESTDIR)/usr/share/createprestorepo diff --git a/createprestorepo/README b/createprestorepo/README deleted file mode 100644 index 010fdd3..0000000 --- a/createprestorepo/README +++ /dev/null @@ -1,22 +0,0 @@ -Presto: A project to add delta rpm support into yum for Fedora users -http://hosted.fedoraproject.org/projects/presto. A list of presto-enabled -repositories is available there. - -createprestorepo: The presto repository creater - -Installation: -============= -1- Untar the package -2- Run 'make install' - -Running: -1- First run 'makedeltarepo <repo dir> <drpm dir>' where <repo dir> is the base - directory for your repository and <drpm dir> is the subdirectory you want to - create the deltarpms into -2- Run 'createprestorepo <repo dir>' where <repo dir> is the base directory for - your repository. - -WARNING: createprestorepo does *NOT* yet know how to deal with the metadata - created by createrepo. You will have to manually move metadata from - .olddata to repodata after running both createprestorepo and - createrepo. diff --git a/createprestorepo/createprestorepo b/createprestorepo/createprestorepo deleted file mode 100755 index 5334a80..0000000 --- a/createprestorepo/createprestorepo +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec /usr/share/createprestorepo/genprestometadata.py "$@" diff --git a/createprestorepo/dumpMetadata.py b/createprestorepo/dumpMetadata.py deleted file mode 100644 index 6969400..0000000 --- a/createprestorepo/dumpMetadata.py +++ /dev/null @@ -1,496 +0,0 @@ -#!/usr/bin/python -t -# base classes and functions for dumping out package Metadata -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2004 Duke University - -# $Id: dumpMetadata.py,v 1.36 2006/02/21 20:10:08 pnasrat Exp $ - -import os -import rpm -import exceptions -import md5 -import sha -import types -import struct -import re -import stat - -# done to fix gzip randomly changing the checksum -import gzip -from zlib import error as zlibError -from gzip import write32u, FNAME - -__all__ = ["GzipFile","open"] - -class GzipFile(gzip.GzipFile): - def _write_gzip_header(self): - self.fileobj.write('\037\213') # magic header - self.fileobj.write('\010') # compression method - fname = self.filename[:-3] - flags = 0 - if fname: - flags = FNAME - self.fileobj.write(chr(flags)) - write32u(self.fileobj, long(0)) - self.fileobj.write('\002') - self.fileobj.write('\377') - if fname: - self.fileobj.write(fname + '\000') - - -def _gzipOpen(filename, mode="rb", compresslevel=9): - return GzipFile(filename, mode, compresslevel) - - - -def returnFD(filename): - try: - fdno = os.open(filename, os.O_RDONLY) - except OSError: - raise MDError, "Error opening file" - return fdno - -def returnHdr(ts, package): - """hand back the rpm header or raise an Error if the pkg is fubar""" - opened_here = 0 - try: - if type(package) is types.StringType: - opened_here = 1 - fdno = os.open(package, os.O_RDONLY) - else: - fdno = package # let's assume this is an fdno and go with it :) - except OSError: - raise MDError, "Error opening file" - ts.setVSFlags((rpm._RPMVSF_NOSIGNATURES|rpm.RPMVSF_NOMD5|rpm.RPMVSF_NEEDPAYLOAD)) - try: - hdr = ts.hdrFromFdno(fdno) - except rpm.error: - raise MDError, "Error opening package" - if type(hdr) != rpm.hdr: - raise MDError, "Error opening package" - ts.setVSFlags(0) - - if opened_here: - os.close(fdno) - del fdno - - return hdr - -def getChecksum(sumtype, file, CHUNK=2**16): - """takes filename, hand back Checksum of it - sumtype = md5 or sha - filename = /path/to/file - CHUNK=65536 by default""" - - # chunking brazenly lifted from Ryan Tomayko - opened_here = 0 - try: - if type(file) is not types.StringType: - fo = file # assume it's a file-like-object - else: - opened_here = 1 - fo = open(file, 'rb', CHUNK) - - if sumtype == 'md5': - sum = md5.new() - elif sumtype == 'sha': - sum = sha.new() - else: - raise MDError, 'Error Checksumming file, wrong checksum type %s' % sumtype - chunk = fo.read - while chunk: - chunk = fo.read(CHUNK) - sum.update(chunk) - - if opened_here: - fo.close() - del fo - - return sum.hexdigest() - except: - raise MDError, 'Error opening file for checksum: %s' % file - - -def utf8String(string): - """hands back a unicoded string""" - if string is None: - return '' - elif isinstance(string, unicode): - return string - try: - x = unicode(string, 'ascii') - return string - except UnicodeError: - encodings = ['utf-8', 'iso-8859-1', 'iso-8859-15', 'iso-8859-2'] - for enc in encodings: - try: - x = unicode(string, enc) - except UnicodeError: - pass - else: - if x.encode(enc) == string: - return x.encode('utf-8') - newstring = '' - for char in string: - if ord(char) > 127: - newstring = newstring + '?' - else: - newstring = newstring + char - return newstring - - -def byteranges(file): - """takes an rpm file or fileobject and returns byteranges for location of the header""" - opened_here = 0 - if type(file) is not types.StringType: - fo = file - else: - opened_here = 1 - fo = open(file, 'r') - #read in past lead and first 8 bytes of sig header - fo.seek(104) - # 104 bytes in - binindex = fo.read(4) - # 108 bytes in - (sigindex, ) = struct.unpack('>I', binindex) - bindata = fo.read(4) - # 112 bytes in - (sigdata, ) = struct.unpack('>I', bindata) - # each index is 4 32bit segments - so each is 16 bytes - sigindexsize = sigindex * 16 - sigsize = sigdata + sigindexsize - # we have to round off to the next 8 byte boundary - disttoboundary = (sigsize % 8) - if disttoboundary != 0: - disttoboundary = 8 - disttoboundary - # 112 bytes - 96 == lead, 8 = magic and reserved, 8 == sig header data - hdrstart = 112 + sigsize + disttoboundary - - fo.seek(hdrstart) # go to the start of the header - fo.seek(8,1) # read past the magic number and reserved bytes - - binindex = fo.read(4) - (hdrindex, ) = struct.unpack('>I', binindex) - bindata = fo.read(4) - (hdrdata, ) = struct.unpack('>I', bindata) - - # each index is 4 32bit segments - so each is 16 bytes - hdrindexsize = hdrindex * 16 - # add 16 to the hdrsize to account for the 16 bytes of misc data b/t the - # end of the sig and the header. - hdrsize = hdrdata + hdrindexsize + 16 - - # header end is hdrstart + hdrsize - hdrend = hdrstart + hdrsize - if opened_here: - fo.close() - del fo - return (hdrstart, hdrend) - - -class MDError(exceptions.Exception): - def __init__(self, args=None): - exceptions.Exception.__init__(self) - self.args = args - - - -class RpmMetaData: - """each drpm is one object, you pass it an rpm file - it opens the file, and pulls the information out in bite-sized chunks :) - """ - - mode_cache = {} - - def __init__(self, ts, basedir, filename, options, is_drpm): - try: - stats = os.stat(os.path.join(basedir, filename)) - self.size = stats[6] - self.mtime = stats[8] - del stats - except OSError, e: - raise MDError, "Error Stat'ing file %s %s" % (basedir, filename) - self.options = options - self.localurl = options['baseurl'] - self.relativepath = filename - fd = returnFD(os.path.join(basedir, filename)) - self.hdr = returnHdr(ts, fd) - os.lseek(fd, 0, 0) - fo = os.fdopen(fd, 'rb') - self.pkgid = self.doChecksumCache(fo) - fo.seek(0) - (self.rangestart, self.rangeend) = byteranges(fo) - self.is_drpm = False - if is_drpm: - fo.seek(self.rangeend) - self._getOldInfo(fo) - self.is_drpm = True - del fo - del fd - - def arch(self): - if self.tagByName('sourcepackage') == 1: - return 'src' - else: - return self.tagByName('arch') - - def _stringToNEVR(self, string): - i = string.rfind("-", 0, string.rfind("-")-1) - name = string[:i] - (epoch, ver, rel) = self._stringToVersion(string[i+1:]) - return (name, epoch, ver, rel) - - def _getLength(self, in_data): - length = 0 - for val in in_data: - length = length * 256 - length += ord(val) - return length - - def _getOldInfo(self, fo): - try: - compobj = gzip.GzipFile("", "rb", 9, fo) - except: - raise zlibError("Data not stored in gzip format") - - if compobj.read(4)[:3] != "DLT": - raise Exception("Not a deltarpm") - - nevr_length = self._getLength(compobj.read(4)) - nevr = compobj.read(nevr_length).strip("\x00") - seq_length = self._getLength(compobj.read(4)) - seq = compobj.read(seq_length) - hex_seq = "" - for char in seq: - hex_seq += str("%02x" % ord(char)) - self.oldnevrstring = nevr - self.oldnevr = self._stringToNEVR(nevr) - self.sequence = hex_seq - compobj.close() - - def _stringToVersion(self, strng): - i = strng.find(':') - if i != -1: - epoch = strng[:i] - else: - epoch = '0' - j = strng.find('-') - if j != -1: - if strng[i + 1:j] == '': - version = None - else: - version = strng[i + 1:j] - release = strng[j + 1:] - else: - if strng[i + 1:] == '': - version = None - else: - version = strng[i + 1:] - release = None - return (epoch, version, release) - - ########### - # Title: Remove duplicates from a sequence - # Submitter: Tim Peters - # From: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560 - - def _uniq(self,s): - """Return a list of the elements in s, but without duplicates. - - For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3], - unique("abcabc") some permutation of ["a", "b", "c"], and - unique(([1, 2], [2, 3], [1, 2])) some permutation of - [[2, 3], [1, 2]]. - - For best speed, all sequence elements should be hashable. Then - unique() will usually work in linear time. - - If not possible, the sequence elements should enjoy a total - ordering, and if list(s).sort() doesn't raise TypeError it's - assumed that they do enjoy a total ordering. Then unique() will - usually work in O(N*log2(N)) time. - - If that's not possible either, the sequence elements must support - equality-testing. Then unique() will usually work in quadratic - time. - """ - - n = len(s) - if n == 0: - return [] - - # Try using a dict first, as that's the fastest and will usually - # work. If it doesn't work, it will usually fail quickly, so it - # usually doesn't cost much to *try* it. It requires that all the - # sequence elements be hashable, and support equality comparison. - u = {} - try: - for x in s: - u[x] = 1 - except TypeError: - del u # move on to the next method - else: - return u.keys() - - # We can't hash all the elements. Second fastest is to sort, - # which brings the equal elements together; then duplicates are - # easy to weed out in a single pass. - # NOTE: Python's list.sort() was designed to be efficient in the - # presence of many duplicate elements. This isn't true of all - # sort functions in all languages or libraries, so this approach - # is more effective in Python than it may be elsewhere. - try: - t = list(s) - t.sort() - except TypeError: - del t # move on to the next method - else: - assert n > 0 - last = t[0] - lasti = i = 1 - while i < n: - if t[i] != last: - t[lasti] = last = t[i] - lasti += 1 - i += 1 - return t[:lasti] - - # Brute force is all that's left. - u = [] - for x in s: - if x not in u: - u.append(x) - return u - - def tagByName(self, tag): - data = self.hdr[tag] - if type(data) is types.ListType: - if len(data) > 0: - return data[0] - else: - return '' - else: - return data - - def listTagByName(self, tag): - """take a tag that should be a list and make sure it is one""" - lst = [] - data = self.hdr[tag] - if data is None: - return lst - - if type(data) is types.ListType: - lst.extend(data) - else: - lst.append(data) - return lst - - def epoch(self): - if self.hdr['epoch'] is None: - return 0 - else: - return self.tagByName('epoch') - - def doChecksumCache(self, fo): - """return a checksum for a package: - - check if the checksum cache is enabled - if not - return the checksum - if so - check to see if it has a cache file - if so, open it and return the first line's contents - if not, grab the checksum and write it to a file for this pkg - """ - if not self.options['cache']: - return getChecksum(self.options['sumtype'], fo) - - csumtag = os.path.basename(self.relativepath) + ".cache" - csumfile = '%s/%s' % (self.options['cachedir'], csumtag) - if os.path.exists(csumfile) and self.mtime <= os.stat(csumfile)[8]: - csumo = open(csumfile, 'r') - checksum = csumo.readline() - csumo.close() - - else: - checksum = getChecksum(self.options['sumtype'], fo) - csumo = open(csumfile, 'w') - csumo.write(checksum) - csumo.close() - - return checksum - - - -def generateXML(doc, node, formatns, drpmObj, sumtype, pkgDeltas): - """takes an xml doc object and a package metadata entry node, populates a - package node with the md information""" - name = drpmObj.tagByName('name') - arch = drpmObj.arch() - epoch = str(drpmObj.epoch()) - ver = str(drpmObj.tagByName('version')) - rel = str(drpmObj.tagByName('release')) - if not pkgDeltas.has_key('%s-%s:%s-%s.%s' % (name, epoch, ver, rel, arch)): - pkgNode = node.newChild(None, "package", None) - pkgNode.newProp('type', 'rpm') - pkgNode.newChild(None, 'name', name) - pkgNode.newChild(None, 'arch', arch) - version = pkgNode.newChild(None, 'version', None) - version.newProp('epoch', epoch) - version.newProp('ver', ver) - version.newProp('rel', rel) - deltas = pkgNode.newChild(None, 'deltas', None) - pkgDeltas['%s-%s:%s-%s.%s' % (name, epoch, ver, rel, arch)] = deltas - else: - deltas = pkgDeltas['%s-%s:%s-%s.%s' % (name, epoch, ver, rel, arch)] - (oldname, oldepoch, oldver, oldrel) = drpmObj.oldnevr - drpmNode = deltas.newChild(None, "oldrpm", None) - if name != oldname: - drpmNode.newChild(None, 'name', oldname) - # oldrpm arch is not stored in drpm, so we can only work within same arch - version = drpmNode.newChild(None, 'version', None) - if epoch != oldepoch: - version.newProp('epoch', oldepoch) - if ver != oldver: - version.newProp('ver', oldver) - version.newProp('rel', oldrel) - drpmNode.newChild(None, 'drpm_filename', drpmObj.relativepath) - drpmNode.newChild(None, 'size', str(drpmObj.size)) - drpmNode.newChild(None, 'sequence', '%s-%s' % (drpmObj.oldnevrstring, drpmObj.sequence)) - checksum = drpmNode.newChild(None, 'checksum', drpmObj.pkgid) - checksum.newProp('type', drpmObj.options['sumtype']) - - -def repoXML(node, cmds): - """generate the repomd.xml file that stores the info on the other files""" - sumtype = cmds['sumtype'] - workfiles = [(cmds['prestofile'], 'deltas')] - - - for (file, ftype) in workfiles: - zfo = _gzipOpen(os.path.join(cmds['outputdir'], cmds['tempdir'], file)) - uncsum = getChecksum(sumtype, zfo) - zfo.close() - csum = getChecksum(sumtype, os.path.join(cmds['outputdir'], cmds['tempdir'], file)) - timestamp = os.stat(os.path.join(cmds['outputdir'], cmds['tempdir'], file))[8] - data = node.newChild(None, 'data', None) - data.newProp('type', ftype) - location = data.newChild(None, 'location', None) - if cmds['baseurl'] is not None: - location.newProp('xml:base', cmds['baseurl']) - location.newProp('href', os.path.join(cmds['finaldir'], file)) - checksum = data.newChild(None, 'checksum', csum) - checksum.newProp('type', sumtype) - timestamp = data.newChild(None, 'timestamp', str(timestamp)) - unchecksum = data.newChild(None, 'open-checksum', uncsum) - unchecksum.newProp('type', sumtype) diff --git a/createprestorepo/gendeltarpms.py b/createprestorepo/gendeltarpms.py deleted file mode 100755 index 26d795b..0000000 --- a/createprestorepo/gendeltarpms.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/python -t -# -*- mode: Python; indent-tabs-mode: nil; -*- -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import errno, os, sys, gzip -import fnmatch, re -import rpmUtils.transaction, rpmUtils.miscutils -import commands, libxml2 -import string -from dumpMetadata import byteranges -from packagelist import RpmItem, DrpmItem, PackageList - -import gzip -from zlib import error as zlibError -from gzip import write32u, FNAME - - -DRPMWORTHKEEPINGTHRESH=0.7 -DEBUG=0 - -def getDeltaNevr(filename): - """Get nevr for src rpm of a deltarpm. Return a tuple of (n, e, v, r)""" - def _getLength(in_data): - length = 0 - for val in in_data: - length = length * 256 - length += ord(val) - return length - - def _stringToVersion(strng): - i = strng.find(':') - if i != -1: - epoch = strng[:i] - else: - epoch = '0' - j = strng.find('-') - if j != -1: - if strng[i + 1:j] == '': - version = None - else: - version = strng[i + 1:j] - release = strng[j + 1:] - else: - if strng[i + 1:] == '': - version = None - else: - version = strng[i + 1:] - release = None - return (epoch, version, release) - - def _stringToNEVR(string): - i = string.rfind("-", 0, string.rfind("-")-1) - name = string[:i] - (epoch, ver, rel) = _stringToVersion(string[i+1:]) - return (name, epoch, ver, rel) - - (range_start, range_end) = byteranges(filename) - fd = open(filename, "r") - fd.seek(range_end) - try: - compobj = gzip.GzipFile("", "rb", 9, fd) - except: - raise zlibError("Data not stored in gzip format") - - if compobj.read(4)[:3] != "DLT": - raise Exception("Not a deltarpm") - - nevr_length = _getLength(compobj.read(4)) - nevr = compobj.read(nevr_length).strip("\x00") - oldnevr = _stringToNEVR(nevr) - compobj.close() - return oldnevr - - -def getFileList(path, ext, filelist): - """Return all files in path matching ext, store them in filelist, - recurse dirs. Returns a list object""" - - extlen = len(ext) - totalpath = os.path.normpath(path) - try: - dir_list = os.listdir(totalpath) - except OSError, e: - errorprint(_('Error accessing directory %s, %s') % (totalpath, e)) - sys.exit(1) - - for d in dir_list: - if os.path.isdir(totalpath + '/' + d): - filelist = getFileList(os.path.join(totalpath, d), ext, filelist) - elif string.lower(d[-extlen:]) == '%s' % (ext): - filelist.append(os.path.join(totalpath, d)) - - return filelist - -def createPrestoRepo(base_dir, drpm_dir, dst_dir=None): - ts = rpmUtils.transaction.initReadOnlyTransaction() - changed = False - - # Create list of .rpm files. - # We don't use "glob", so sub-directories are supported. - print 'Using base dir: %s' % base_dir - print 'Using destination dir: %s' % drpm_dir - - rpm_files = getFileList(base_dir, ".rpm", []) - if dst_dir != None: - dst_rpm_files = getFileList(dst_dir, ".rpm", []) - drpm_files = getFileList(drpm_dir, ".drpm", []) - - if not len(rpm_files): - print ' Nothing found.' - return changed - - if dst_dir != None and not len(dst_rpm_files): - print ' No new rpms found.' - return changed - - # Check whether rel_dir exists, and if it doesn't, create it - if not os.access(drpm_dir, os.F_OK): - os.makedirs(drpm_dir, 0755) - elif not os.access(drpm_dir, os.W_OK): - print 'ERROR: Unable to write to %s' % drpm_dir - sys.exit(1) - - # Add all rpms to PackageList - rpm_list = PackageList(base_dir, drpm_dir, DRPMWORTHKEEPINGTHRESH, DEBUG) - for f in rpm_files: - try: - hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) - except: - print "Unable to open %s" % f - else: - nm = hdr['name'] + "." + hdr['arch'] - e = hdr['epoch'] - if e is None: - e = "0" - - nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch']) - rpm = RpmItem(nevra, f) - rpm_list.addRpm(rpm) - - # If we have a new distro, add its rpms with epoch bumped +5 - if dst_dir != None: - for f in dst_rpm_files: - try: - hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) - except: - print "Unable to open %s" % f - else: - nm = hdr['name'] + "." + hdr['arch'] - e = hdr['epoch'] - if e is None: - e = "0" - e = str(int(e) + 5) - - nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch']) - rpm = RpmItem(nevra, f) - rpm_list.addRpm(rpm) - - # Add all deltarpms to PackageList - for f in drpm_files: - try: - hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) - except: - print "Unable to open %s" % f - else: - (sn, se, sv, sr) = getDeltaNevr(f) - - nm = hdr['name'] + "." + hdr['arch'] - e = hdr['epoch'] - if e is None: - e = "0" - - dst_nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch']) - src_nevra = (sn, se, sv, sr, hdr['arch']) - drpm = DrpmItem(src_nevra, dst_nevra, f) - rpm_list.addDrpm(drpm) - - if DEBUG: - rpm_list.dump() - - # Build deltarpms - if dst_dir != None: - rpm_list.makeDeltaRpms(DrpmsPerPackage=1, DoFirst=False) - else: - rpm_list.makeDeltaRpms(DrpmsPerPackage=5) - -if __name__ == '__main__': - if len(sys.argv) < 2 or (sys.argv[1] == "-d" and len(sys.argv) < 4): - print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir>' % os.path.basename(sys.argv[0]) - print ' %s -d <old_dist_dir> <new_dist_dir> <delta_rpm_dir>' % os.path.basename(sys.argv[0]) - sys.exit(errno.EINVAL) - if sys.argv[1] != "-d": - bin_rpm_path = sys.argv[1] - delta_rpm_path = sys.argv[2] - createPrestoRepo(bin_rpm_path, delta_rpm_path) - else: - src_bin_rpm_path = sys.argv[2] - dst_bin_rpm_path = sys.argv[3] - delta_rpm_path = sys.argv[4] - createPrestoRepo(src_bin_rpm_path, delta_rpm_path, dst_bin_rpm_path) - - sys.exit(0) - diff --git a/createprestorepo/genprestometadata.py b/createprestorepo/genprestometadata.py deleted file mode 100755 index bbce09d..0000000 --- a/createprestorepo/genprestometadata.py +++ /dev/null @@ -1,475 +0,0 @@ -#!/usr/bin/python -t -# primary functions and glue for generating the repository metadata -# - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2004 Duke University -# Copyright 2007 Jonathan Dieter - - -import os -import sys -import getopt -import rpm -import libxml2 -import string -import fnmatch -import urlgrabber - -import dumpMetadata -from dumpMetadata import _gzipOpen -__version__ = '0.4.3' - -def errorprint(stuff): - print >> sys.stderr, stuff - -def _(args): - """Stub function for translation""" - return args - -def usage(retval=1): - print _(""" - createrepo [options] directory-of-packages - - Options: - -u, --baseurl <url> = optional base url location for all files - -o, --outputdir <dir> = optional directory to output to - -x, --exclude = files globs to exclude, can be specified multiple times - -q, --quiet = run quietly - -v, --verbose = run verbosely - -c, --cachedir <dir> = specify which dir to use for the checksum cache - -h, --help = show this help - -V, --version = output version - -p, --pretty = output xml files in pretty format. - """) - - sys.exit(retval) - -class MetaDataGenerator: - def __init__(self, cmds): - self.cmds = cmds - self.ts = rpm.TransactionSet() - self.pkgcount = 0 - self.newrpms = {} - self.files = [] - - def getFileList(self, basepath, path, ext, filelist): - """Return all files in path matching ext, store them in filelist, - recurse dirs. Returns a list object""" - - extlen = len(ext) - totalpath = os.path.normpath(os.path.join(basepath, path)) - try: - dir_list = os.listdir(totalpath) - except OSError, e: - errorprint(_('Error accessing directory %s, %s') % (totalpath, e)) - sys.exit(1) - - for d in dir_list: - if os.path.isdir(totalpath + '/' + d): - filelist = self.getFileList(basepath, os.path.join(path, d), ext, filelist) - else: - if string.lower(d[-extlen:]) == '%s' % (ext): - if totalpath.find(basepath) == 0: - relativepath = totalpath.replace(basepath, "", 1) - relativepath = relativepath.lstrip("/") - filelist.append(os.path.join(relativepath, d)) - else: - raise "basepath '%s' not found in path '%s'" % (basepath, totalpath) - - return filelist - - - def trimRpms(self, files): - badrpms = [] - for file in files: - for glob in self.cmds['excludes']: - if fnmatch.fnmatch(file, glob): - # print 'excluded: %s' % file - if file not in badrpms: - badrpms.append(file) - for file in badrpms: - if file in files: - files.remove(file) - return files - - def doPkgMetadata(self, directory): - """all the heavy lifting for the package metadata""" - - # rpms we're going to be dealing with - files = self.getFileList(self.cmds['basedir'], directory, '.drpm', []) - files = self.trimRpms(files) - self.pkgcount = len(files) - self.openMetadataDocs() - self.writeMetadataDocs(files) - self.closeMetadataDocs() - - - def openMetadataDocs(self): - self._setupPresto() - - def _setupPresto(self): - # setup the base metadata doc - self.prestodoc = libxml2.newDoc("1.0") - self.prestoroot = self.prestodoc.newChild(None, "metadata", None) - basens = self.prestoroot.newNs('http://linux.duke.edu/metadata/common', None) - self.formatns = self.prestoroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') - self.prestoroot.setNs(basens) - prestofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile']) - self.prestofile = _gzipOpen(prestofilepath, 'w') - self.prestofile.write('<?xml version="1.0" encoding="UTF-8"?>\n') - self.prestofile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' % - self.pkgcount) - - - def writeMetadataDocs(self, files, current=0): - for file in files: - current+=1 - try: - mdobj = dumpMetadata.RpmMetaData(self.ts, self.cmds['basedir'], file, self.cmds, True) - if not self.cmds['quiet']: - if self.cmds['verbose']: - print '%d/%d - %s' % (current, len(files), file) - else: - sys.stdout.write('\r' + ' ' * 80) - sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file)) - sys.stdout.flush() - except dumpMetadata.MDError, e: - errorprint('\n%s - %s' % (e, file)) - continue - else: - try: - dumpMetadata.generateXML(self.prestodoc, self.prestoroot, self.formatns, mdobj, self.cmds['sumtype'], self.newrpms) - except dumpMetadata.MDError, e: - errorprint(_('\nAn error occurred creating presto metadata: %s') % e) - continue - return current - - - def closeMetadataDocs(self): - if not self.cmds['quiet']: - print '' - - # save them up to the tmp locations: - if not self.cmds['quiet']: - print _('Saving Presto metadata') - output = self.prestoroot.serialize('UTF-8', self.cmds['pretty']) - output = output[output.find("\n")+1:] - self.prestofile.write(output) - self.prestofile.write("\n") - self.prestofile.close() - self.prestodoc.freeDoc() - - def doRepoMetadata(self): - """wrapper to generate the prestomd.xml file that stores the info on the other files""" - repodoc = libxml2.newDoc("1.0") - reporoot = repodoc.newChild(None, "repomd", None) - repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) - reporoot.setNs(repons) - repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile']) - - try: - dumpMetadata.repoXML(reporoot, self.cmds) - except dumpMetadata.MDError, e: - errorprint(_('Error generating repo xml file: %s') % e) - sys.exit(1) - - try: - repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) - except: - errorprint(_('Error saving temp file for rep xml: %s') % repofilepath) - sys.exit(1) - - del repodoc - -class SplitMetaDataGenerator(MetaDataGenerator): - - def __init__(self, cmds): - MetaDataGenerator.__init__(self, cmds) - self.initialdir = self.cmds['basedir'] - - def _getFragmentUrl(self, url, fragment): - import urlparse - urlparse.uses_fragment.append('media') - if not url: - return url - (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url) - return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment))) - - def doPkgMetadata(self, directories): - """all the heavy lifting for the package metadata""" - import types - if type(directories) == types.StringType: - MetaDataGenerator.doPkgMetadata(self, directories) - return - filematrix = {} - for mydir in directories: - filematrix[mydir] = self.getFileList(os.path.join(self.initialdir, mydir), '.', '.rpm', []) - self.trimRpms(filematrix[mydir]) - self.pkgcount += len(filematrix[mydir]) - - mediano = 1 - current = 0 - self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano) - self.cmds['basedir'] = os.path.join(self.initialdir, directories[0]) - self.openMetadataDocs() - for mydir in directories: - self.cmds['basedir'] = os.path.join(self.initialdir, mydir) - self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano) - current = self.writeMetadataDocs(filematrix[mydir], current) - mediano += 1 - self.cmds['basedir'] = os.path.join(self.initialdir, directories[0]) - self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], 1) - self.closeMetadataDocs() - - -def checkAndMakeDir(dir): - """ - check out the dir and make it, if possible, return 1 if done, else return 0 - """ - if os.path.exists(dir): - if not os.path.isdir(dir): - errorprint(_('%s is not a dir') % dir) - result = False - else: - if not os.access(dir, os.W_OK): - errorprint(_('%s is not writable') % dir) - result = False - else: - result = True - else: - try: - os.mkdir(dir) - except OSError, e: - errorprint(_('Error creating dir %s: %s') % (dir, e)) - result = False - else: - result = True - return result - -def parseArgs(args): - """ - Parse the command line args return a commands dict and directory. - Sanity check all the things being passed in. - """ - cmds = {} - cmds['quiet'] = 0 - cmds['verbose'] = 0 - cmds['excludes'] = [] - cmds['baseurl'] = None - cmds['sumtype'] = 'sha' - cmds['pretty'] = 0 - cmds['cachedir'] = None - cmds['basedir'] = os.getcwd() - cmds['cache'] = False - cmds['split'] = False - cmds['outputdir'] = "" - cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] - cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*'] - - try: - gopts, argsleft = getopt.getopt(args, 'phqVvs:x:u:c:o:', ['help', 'exclude=', - 'quiet', 'verbose', 'cachedir=', 'basedir=', - 'baseurl=', 'checksum=', - 'version', 'pretty', 'split', 'outputdir=']) - except getopt.error, e: - errorprint(_('Options Error: %s.') % e) - usage() - - try: - for arg,a in gopts: - if arg in ['-h','--help']: - usage(retval=0) - elif arg in ['-V', '--version']: - print '%s' % __version__ - sys.exit(0) - elif arg == '--split': - cmds['split'] = True - except ValueError, e: - errorprint(_('Options Error: %s') % e) - usage() - - - # make sure our dir makes sense before we continue - if len(argsleft) > 1 and not cmds['split']: - errorprint(_('Error: Only one directory allowed per run.')) - usage() - elif len(argsleft) == 0: - errorprint(_('Error: Must specify a directory to index.')) - usage() - else: - directories = argsleft - - try: - for arg,a in gopts: - if arg in ['-v', '--verbose']: - cmds['verbose'] = 1 - elif arg in ["-q", '--quiet']: - cmds['quiet'] = 1 - elif arg in ['-u', '--baseurl']: - if cmds['baseurl'] is not None: - errorprint(_('Error: Only one baseurl allowed.')) - usage() - else: - cmds['baseurl'] = a - elif arg in ['-x', '--exclude']: - cmds['excludes'].append(a) - elif arg in ['-p', '--pretty']: - cmds['pretty'] = 1 - elif arg in ['-c', '--cachedir']: - cmds['cache'] = True - cmds['cachedir'] = a - elif arg == '--basedir': - cmds['basedir'] = a - elif arg in ['-o','--outputdir']: - cmds['outputdir'] = a - - except ValueError, e: - errorprint(_('Options Error: %s') % e) - usage() - - directory = directories[0] -# Fix paths - directory = os.path.normpath(directory) - if cmds['split']: - pass - elif os.path.isabs(directory): - cmds['basedir'] = directory - directory = '.' - else: - cmds['basedir'] = os.path.realpath(os.path.join(cmds['basedir'], directory)) - directory = '.' - if not cmds['outputdir']: - cmds['outputdir'] = cmds['basedir'] - if cmds['cachedir']: - a = cmds ['cachedir'] - if not os.path.isabs(a): - a = os.path.join(cmds['basedir'] ,a) - if not checkAndMakeDir(a): - errorprint(_('Error: cannot open/write to cache dir %s' % a)) - usage() - cmds['cachedir'] = a - - #setup some defaults - cmds['prestofile'] = 'presto.xml.gz' - cmds['prestomdfile'] = 'prestomd.xml' - cmds['tempdir'] = '.repodata' - cmds['finaldir'] = 'repodata' - cmds['olddir'] = '.olddata' - - # Fixup first directory - directories[0] = directory - return cmds, directories - -def main(args): - cmds, directories = parseArgs(args) - directory = directories[0] - # start the sanity/stupidity checks - if not os.path.exists(os.path.join(cmds['basedir'], directory)): - errorprint(_('Directory must exist')) - sys.exit(1) - - if not os.path.isdir(os.path.join(cmds['basedir'], directory)): - errorprint(_('Directory of packages must be a directory.')) - sys.exit(1) - - if not os.access(cmds['outputdir'], os.W_OK): - errorprint(_('Directory must be writable.')) - sys.exit(1) - - if cmds['split']: - oldbase = cmds['basedir'] - cmds['basedir'] = os.path.join(cmds['basedir'], directory) - if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['tempdir'])): - sys.exit(1) - - if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['finaldir'])): - sys.exit(1) - - if os.path.exists(os.path.join(cmds['outputdir'], cmds['olddir'])): - errorprint(_('Old data directory exists, please remove: %s') % cmds['olddir']) - sys.exit(1) - - # make sure we can write to where we want to write to: - for direc in ['tempdir', 'finaldir']: - for file in ['prestofile', 'prestomdfile']: - filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file]) - if os.path.exists(filepath): - if not os.access(filepath, os.W_OK): - errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath) - usage() - - if cmds['split']: - cmds['basedir'] = oldbase - mdgen = SplitMetaDataGenerator(cmds) - mdgen.doPkgMetadata(directories) - else: - mdgen = MetaDataGenerator(cmds) - mdgen.doPkgMetadata(directory) - mdgen.doRepoMetadata() - - if os.path.exists(os.path.join(cmds['outputdir'], cmds['finaldir'])): - try: - os.rename(os.path.join(cmds['outputdir'], cmds['finaldir']), - os.path.join(cmds['outputdir'], cmds['olddir'])) - except: - errorprint(_('Error moving final %s to old dir %s' % (os.path.join(cmds['outputdir'], cmds['finaldir']), - os.path.join(cmds['outputdir'], cmds['olddir'])))) - sys.exit(1) - - try: - os.rename(os.path.join(cmds['outputdir'], cmds['tempdir']), - os.path.join(cmds['outputdir'], cmds['finaldir'])) - except: - errorprint(_('Error moving final metadata into place')) - # put the old stuff back - os.rename(os.path.join(cmds['outputdir'], cmds['olddir']), - os.path.join(cmds['outputdir'], cmds['finaldir'])) - sys.exit(1) - - for file in ['prestofile', 'prestomdfile']: - if cmds[file]: - fn = os.path.basename(cmds[file]) - else: - continue - oldfile = os.path.join(cmds['outputdir'], cmds['olddir'], fn) - if os.path.exists(oldfile): - try: - os.remove(oldfile) - except OSError, e: - errorprint(_('Could not remove old metadata file: %s') % oldfile) - errorprint(_('Error was %s') % e) - sys.exit(1) - - -#XXX: fix to remove tree as we mung basedir - try: - os.rmdir(os.path.join(cmds['outputdir'], cmds['olddir'])) - except OSError, e: - errorprint(_('Could not remove old metadata dir: %s') % cmds['olddir']) - errorprint(_('Error was %s') % e) - errorprint(_('Please clean up this directory manually.')) - -if __name__ == "__main__": - if len(sys.argv) > 1: - if sys.argv[1] == 'profile': - import hotshot - p = hotshot.Profile(os.path.expanduser("~/createprestorepo.prof")) - p.run('main(sys.argv[2:])') - p.close() - else: - main(sys.argv[1:]) - else: - main(sys.argv[1:]) diff --git a/createprestorepo/makedeltarepo b/createprestorepo/makedeltarepo deleted file mode 100755 index 99bedcc..0000000 --- a/createprestorepo/makedeltarepo +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec /usr/share/createprestorepo/gendeltarpms.py "$@" diff --git a/createprestorepo/packagelist.py b/createprestorepo/packagelist.py deleted file mode 100644 index c057f3f..0000000 --- a/createprestorepo/packagelist.py +++ /dev/null @@ -1,385 +0,0 @@ -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# Copyright (c) 2007 Jonathan Dieter - -import rpmUtils.miscutils -import os -import commands - -class RpmItem: - def __init__(self, nevra, f): - (self.name, self.epoch, self.version, self.release, self.arch) = nevra - self.epoch = str(self.epoch) - self.filename = f - - def getNevra(self): - return (self.name, self.epoch, self.version, self.release, self.arch) - - def __eq__(self, b): - if b == None: - return False - try: - # An RpmItem is equal to another RpmItem if the nevra's are equal - if self.getNevra() == b.getNevra(): - return True - else: - return False - except: - # An RpmItem is equal to a DrpmItem if the nevra is equal to the drpm's destination nevra - if self.getNevra() == b.getDstNevra(): - return True - else: - return False - -class DrpmItem: - def __init__(self, src_nevra, dst_nevra, f): - (self.src_name, self.src_epoch, self.src_version, self.src_release, self.src_arch) = src_nevra - (self.dst_name, self.dst_epoch, self.dst_version, self.dst_release, self.dst_arch) = dst_nevra - self.src_epoch = str(self.src_epoch) - self.dst_epoch = str(self.dst_epoch) - self.filename = f - - def exists(self): - return self.filename != None - - def getSrcNevra(self): - return (self.src_name, self.src_epoch, self.src_version, self.src_release, self.src_arch) - - def getDstNevra(self): - return (self.dst_name, self.dst_epoch, self.dst_version, self.dst_release, self.dst_arch) - - def getSrcDstNevra(self): - return [self.getSrcNevra(), self.getDstNevra()] - - def __eq__(self, b): - if b == None: - return False - try: - # A DrpmItem is equal to another DrpmItem if both source and destination nevra's are equal - if self.getSrcDstNevra() == b.getSrcDstNevra(): - return True - else: - return False - except: - # A DrpmItem is equal to an RpmItem if the destination nevra is equal to the rpm's nevra - if self.getDstNevra() == b.getNevra(): - return True - else: - return False - - -class PackageList: - def __init__(self, base_dir, drpm_dir, threshold, debug): - self.base_dir = base_dir - self.drpm_dir = drpm_dir - self.threshold = threshold - self.debug = debug - self.rpm_list = {} - self.sorted = False - - def addRpm(self, rpm): - """Add RpmItem to PackageList""" - self.sorted = False - rpm_key = "%s.%s" % (rpm.name, rpm.arch) - self.rpm_list.setdefault(rpm_key, {}) - self.rpm_list[rpm_key].setdefault("rpms", []) - found = False - for found_rpm in self.rpm_list[rpm_key]["rpms"]: - if rpm == found_rpm: - found = True - break - if not found: - self.rpm_list[rpm_key]["rpms"].append(rpm) - - def addDrpm(self, drpm): - """Add DrpmItem to PackageList""" - self.sorted = False - rpm_key = "%s.%s" % (drpm.src_name, drpm.src_arch) - self.rpm_list.setdefault(rpm_key, {}) - self.rpm_list[rpm_key].setdefault("drpms", []) - found = False - for found_drpm in self.rpm_list[rpm_key]["drpms"]: - if drpm == found_drpm: - found = True - break - if not found: - self.rpm_list[rpm_key]["drpms"].append(drpm) - - def sort(self): - """Sort all rpms and deltarpms in reverse order. This is necessary before running any - methods that try to work with rpms and deltarpms""" - self.sorted = True - for rpm_type in self.rpm_list.values(): - for key in rpm_type.keys(): - if key == "rpms" and len(rpm_type["rpms"]) > 1: - def sortByEVR(rpm1, rpm2): - (n1,e1,v1,r1,a1) = rpm1.getNevra() - (n2,e2,v2,r2,a2) = rpm2.getNevra() - rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2)) - if rc == 0: - return 0 - if rc > 0: - return -1 - if rc < 0: - return 1 - rpm_type["rpms"].sort(sortByEVR) # highest first in list - elif key == "drpms" and len(rpm_type["drpms"]) > 1: - def sortByEVR(drpm1, drpm2, src=False): - if src: - (n1,e1,v1,r1,a1) = drpm1.getSrcNevra() - (n2,e2,v2,r2,a2) = drpm2.getSrcNevra() - else: - (n1,e1,v1,r1,a1) = drpm1.getDstNevra() - (n2,e2,v2,r2,a2) = drpm2.getDstNevra() - rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2)) - if rc == 0: - if src: - return 0 - else: - return sortByEVR(drpm1, drpm2, True) - if rc > 0: - return -1 - if rc < 0: - return 1 - rpm_type["drpms"].sort(sortByEVR) # highest first in list - - def dump(self): - """Dump list of all packages and their rpms and deltarpms""" - if not self.sorted: - self.sort() - temp_list = [] - for x in self.rpm_list.keys(): - temp_list.append(x) - temp_list.sort() - for x in temp_list: - print x - if self.rpm_list[x].has_key("rpms"): - print " RPMS:" - for rpm in self.rpm_list[x]["rpms"]: - print " %s:%s-%s" % (rpm.epoch, rpm.version, rpm.release) - if self.rpm_list[x].has_key("drpms"): - print " DRPMS:" - for drpm in self.rpm_list[x]["drpms"]: - print " %s:%s-%s => %s:%s-%s" % (drpm.src_epoch, drpm.src_version, drpm.src_release, drpm.dst_epoch, drpm.dst_version, drpm.dst_release) - - def __doWork(self, nevra1, nevra2, f1, deltaCommand): - (n1,e1,v1,r1,a1) = nevra1 - (n2,e2,v2,r2,a2) = nevra2 - - deltaRPMName = os.path.join(self.drpm_dir, '%s-%s-%s_%s-%s.%s.drpm' % (n1, v2, r2, v1, r1, a1)) - deltaCommand += deltaRPMName - if self.debug: - print deltaCommand - - if os.path.exists("%s.dontdelta" % deltaRPMName): - f = open("%s.dontdelta" % deltaRPMName, "r") - drpmsize = f.readline()[:-1] - drpmsize = int(drpmsize) - if not self.__drpmIsWorthKeeping(drpmsize, f1): - return False - else: - if self.debug: - print "Original drpm is now worth creating...deleting %s%s.dontdelta" % (self.base_dir, deltaRPMName) - try: - os.unlink("%s.dontdelta" % deltaRPMName) - except Exception, e: - print "Error deleting %s.dontdelta" % os.path.basename(deltaRPMName), str(e) - - (code, out) = commands.getstatusoutput(deltaCommand) - if code: - print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out) - return False - else: - # Check whether or not we should keep the drpm - drpmsize = os.path.getsize(deltaRPMName) - if not self.__drpmIsWorthKeeping(drpmsize, f1): - if self.debug: - print 'deleting %s' % (deltaRPMName) - f = open("%s.dontdelta" % deltaRPMName, "w") - f.write("%i\n" % os.path.getsize(deltaRPMName)) - f.close() - try: - os.unlink(deltaRPMName) - except Exception, e: - print "Error deleting deltarpm %s" % os.path.basename(deltaRPMName), str(e) - return False - else: - if e1 == e2: - print 'Generated delta rpm for %s.%s - %s-%s => %s-%s' % (n1, a1, v2, r2, v1, r1) - else: - print 'Generated delta rpm for %s.%s - %s:%s-%s => %s:%s-%s' % (n1, a1, e2, v2, r2, e1, v1, r1) - drpm = DrpmItem(nevra2, nevra1, deltaRPMName) - self.addDrpm(drpm) - self.sort() - if self.debug: - self.dump() - return True - - def __combineDeltaRpm(self, combine_list, dstrpm): - nevra1 = combine_list[-1].getDstNevra() - nevra2 = combine_list[0].getSrcNevra() - f1 = dstrpm.filename - - deltaCommand = 'combinedeltarpm ' - for drpm in combine_list: - deltaCommand += "%s " % drpm.filename - - return self.__doWork(nevra1, nevra2, f1, deltaCommand) - - - def __buildDeltaRpm(self, srcrpm, dstrpm): - nevra1 = dstrpm.getNevra() - nevra2 = srcrpm.getNevra() - - f1 = dstrpm.filename - f2 = srcrpm.filename - deltaCommand = 'makedeltarpm %s %s ' % (f2, f1) - - return self.__doWork(nevra1, nevra2, f1, deltaCommand) - - def __makeDeltaRpm(self, src_nevra, dstrpm, package): - foundSrc = False - foundDst = False - tempQueue = [] - if package.has_key("drpms"): - for drpm in package["drpms"]: - if drpm.getDstNevra() == dstrpm.getNevra() and drpm.getSrcNevra() == src_nevra: - return True - if drpm.getSrcNevra() == src_nevra: - for item in tempQueue: - if drpm.getDstNevra() == item.getSrcNevra(): - combine_list = [drpm, item] - return self.__combineDeltaRpm(combine_list, dstrpm) - elif drpm.getDstNevra() == dstrpm.getNevra(): - tempQueue.append(drpm) - - for rpm in package["rpms"]: - if rpm.getNevra() == src_nevra: - return self.__buildDeltaRpm(rpm, dstrpm) - - print "Found nothing!" - return False - - - def makeDeltaRpms(self, DrpmsPerPackage=0, OnlyLastPackage=True, DoFirst=True): - """Create deltarpms for packages. - Attributes: - OnlyLastPackage (Boolean): Only create deltarpms to point to last destination package - DrpmsPerPackage (Int): Number of deltarpms to make for each package (0 = all possible) - DoFirst (Boolean): Create deltarpm for first source package""" - if not self.sorted: - self.sort() - temp_list = [] - for x in self.rpm_list.keys(): - temp_list.append(x) - temp_list.sort() - if self.debug: - print "Building drpms" - for rpm_name in temp_list: - package = self.rpm_list[rpm_name] - if not package.has_key("rpms") or len(package["rpms"]) == 0: - if self.debug: - print "No rpms for %s: Doing nothing" % rpm_name - continue - - if (not package.has_key("drpms") or len(package["drpms"]) == 0) and len(package["rpms"]) == 1: - if self.debug: - print "Only one rpm available for %s: Doing nothing" % rpm_name - continue - - build_list = [] - - for rpm in package["rpms"]: - if [rpm.getNevra(), 1] not in build_list: - build_list.append([rpm.getNevra(), 1]) - - if package.has_key("drpms"): - for drpm in package["drpms"]: - if [drpm.getSrcNevra(), 1] not in build_list and [drpm.getSrcNevra(), 0] not in build_list: - build_list.append([drpm.getSrcNevra(), 0]) - if [drpm.getDstNevra(), 1] not in build_list and [drpm.getDstNevra(), 0] not in build_list: - build_list.append([drpm.getDstNevra(), 0]) - - def sortByEVR(rpm1, rpm2): - (n1,e1,v1,r1,a1) = rpm1[0] - (n2,e2,v2,r2,a2) = rpm2[0] - rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2)) - if rc == 0: - return 0 - if rc > 0: - return -1 - if rc < 0: - return 1 - build_list.sort(sortByEVR) - - if OnlyLastPackage: - if build_list[0][1] != 1: - continue - - rpm = None - for x in package["rpms"]: - if x.getNevra() == build_list[0][0]: - rpm = x - break - if rpm == None: - continue - - newpkg_list = [rpm] - else: - newpkg_list = [] - for build in build_list[:-1]: - if build[1] != 1: - continue - - rpm = None - for x in package["rpms"]: - if x.getNevra() == build[0]: - rpm = x - break - if rpm == None: - continue - - newpkg_list.append(rpm) - - for newpkg in newpkg_list: - start = False - count = 0 - do_last = True - for build in build_list[:-1]: - if start: - self.__makeDeltaRpm(build[0], newpkg, package) - if not self.sorted: - self.sort() - count += 1 - if DrpmsPerPackage != 0: - if count >= DrpmsPerPackage and not DoFirst: - do_last = False - break - elif count >= DrpmsPerPackage - 1 and DoFirst: - break - if build[0] == newpkg.getNevra(): - start = True - - if DoFirst == 1 and start and do_last: - self.__makeDeltaRpm(build_list[-1][0], newpkg, package) - if not self.sorted: - self.sort() - return - - def __drpmIsWorthKeeping(self, drpmsize, newrpm): - newsize = os.path.getsize(newrpm) - # Delete the drpm if it's too large - if drpmsize > self.threshold * newsize: - return False - return True |