summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Dieter <jdieter@gmail.com>2007-07-11 18:37:03 +0300
committerJonathan Dieter <jdieter@gmail.com>2007-07-11 18:37:03 +0300
commitec53c5c2b2f6ea31b22c47e4ae8f71c4eff2cd37 (patch)
tree8f6deaef1c82c121fc71d5b7b10fa2b0ac0fa853
parentdce0600bc64c793ba6e8f67c56c286d8d97e7c4c (diff)
downloadpresto-ec53c5c2b2f6ea31b22c47e4ae8f71c4eff2cd37.zip
presto-ec53c5c2b2f6ea31b22c47e4ae8f71c4eff2cd37.tar.gz
presto-ec53c5c2b2f6ea31b22c47e4ae8f71c4eff2cd37.tar.xz
Complete rewrite of yum-presto. New xml format.
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
-rw-r--r--presto-utils/ChangeLog4
-rw-r--r--presto-utils/Makefile20
-rw-r--r--presto-utils/README17
-rwxr-xr-xpresto-utils/createdeltarpms2
-rwxr-xr-xpresto-utils/createprestorepo2
-rwxr-xr-xpresto-utils/createprestorepo-0.22
-rwxr-xr-xpresto-utils/createprestorepo-0.32
-rwxr-xr-xpresto-utils/doprunedrpms.py (renamed from prunedrpms/PruneDrpms.py)0
-rw-r--r--presto-utils/dumpMetadata.py1
-rwxr-xr-xpresto-utils/genpresto.py313
-rwxr-xr-xpresto-utils/prunedrpms2
-rw-r--r--presto-utils/relaxng/presto.rnc20
-rw-r--r--prunedrpms/ChangeLog2
-rw-r--r--prunedrpms/Makefile7
-rw-r--r--prunedrpms/README16
-rwxr-xr-xprunedrpms/prunedrpms2
-rw-r--r--yum-presto-legacy/COPYING (renamed from prunedrpms/COPYING)0
-rw-r--r--yum-presto-legacy/ChangeLog68
-rw-r--r--yum-presto-legacy/Makefile17
-rw-r--r--yum-presto-legacy/README31
-rw-r--r--yum-presto-legacy/presto.conf16
-rw-r--r--yum-presto-legacy/presto.py163
-rw-r--r--yum-presto-legacy/shared/deltarpm.py (renamed from yum-presto/shared/deltarpm.py)0
-rw-r--r--yum-presto-legacy/shared/prestoDownload.py (renamed from yum-presto/shared/prestoDownload.py)0
-rw-r--r--yum-presto-legacy/shared/prestoLog.py (renamed from yum-presto/shared/prestoLog.py)0
-rw-r--r--yum-presto-legacy/shared/prestoRepo.py (renamed from yum-presto/shared/prestoRepo.py)0
-rw-r--r--yum-presto-legacy/shared/prestoThread.py (renamed from yum-presto/shared/prestoThread.py)0
-rw-r--r--yum-presto-legacy/shared/prestoTransaction.py (renamed from yum-presto/shared/prestoTransaction.py)0
-rw-r--r--yum-presto-legacy/shared/prestomdparser.py (renamed from yum-presto/shared/prestomdparser.py)0
-rw-r--r--yum-presto/ChangeLog5
-rw-r--r--yum-presto/Makefile9
-rw-r--r--yum-presto/presto.conf9
-rw-r--r--yum-presto/presto.py464
33 files changed, 1018 insertions, 176 deletions
diff --git a/presto-utils/ChangeLog b/presto-utils/ChangeLog
index 1640457..8bc2ac3 100644
--- a/presto-utils/ChangeLog
+++ b/presto-utils/ChangeLog
@@ -1,3 +1,7 @@
+* Wed Jul 11 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.0
+ - New XML format by Jeremy Katz
+ - Added prunedrpm script
+
* Tue Jun 19 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.0
- Now works with createrepo and modifyrepo
- Many bugfixes
diff --git a/presto-utils/Makefile b/presto-utils/Makefile
index 9e8c6eb..643539c 100644
--- a/presto-utils/Makefile
+++ b/presto-utils/Makefile
@@ -2,10 +2,18 @@ clean:
rm -f *.pyc *.pyo *~
install:
- install -m 755 createprestorepo $(DESTDIR)/usr/bin/
+ mkdir -p $(DESTDIR)/usr/bin
+ install -m 755 createprestorepo-0.2 $(DESTDIR)/usr/bin/
+ install -m 755 createprestorepo-0.3 $(DESTDIR)/usr/bin/
+ ln -s createprestorepo-0.3 $(DESTDIR)/usr/bin/createprestorepo
install -m 755 createdeltarpms $(DESTDIR)/usr/bin/
- mkdir -p $(DESTDIR)/usr/share/createprestorepo
- install -m 755 genprestometadata.py $(DESTDIR)/usr/share/createprestorepo
- install -m 644 dumpMetadata.py $(DESTDIR)/usr/share/createprestorepo
- install -m 755 gendeltarpms.py $(DESTDIR)/usr/share/createprestorepo
- install -m 644 packagelist.py $(DESTDIR)/usr/share/createprestorepo
+ install -m 755 prunedrpms $(DESTDIR)/usr/bin/
+ mkdir -p $(DESTDIR)/usr/share/presto-utils
+ install -m 755 genprestometadata.py $(DESTDIR)/usr/share/presto-utils/
+ install -m 755 genpresto.py $(DESTDIR)/usr/share/presto-utils/
+ install -m 644 dumpMetadata.py $(DESTDIR)/usr/share/presto-utils/
+ install -m 755 gendeltarpms.py $(DESTDIR)/usr/share/presto-utils/
+ install -m 644 packagelist.py $(DESTDIR)/usr/share/presto-utils/
+ install -m 755 doprunedrpms.py $(DESTDIR)/usr/share/presto-utils/
+ mkdir -p $(DESTDIR)/usr/share/presto-utils/relaxng
+ install -m 644 relaxng/presto.rnc $(DESTDIR)/usr/share/presto-utils/relaxng/
diff --git a/presto-utils/README b/presto-utils/README
index a81187a..ddb3faa 100644
--- a/presto-utils/README
+++ b/presto-utils/README
@@ -2,21 +2,24 @@ Presto: A project to add delta rpm support into yum for Fedora users
http://hosted.fedoraproject.org/projects/presto. A list of presto-enabled
repositories is available there.
-createprestorepo: The presto repository creater
+presto-utils: Tools for working on presto repositories
Installation:
=============
1- Untar the package
2- Run 'make install'
-Running:
+To create a presto repository:
1- First run 'createdeltarpms <repo dir> <drpm dir>' where <repo dir> is the
base directory for your repository and <drpm dir> is the subdirectory you
want to create the deltarpms into
2- Run 'createprestorepo <repo dir>' where <repo dir> is the base directory for
- your repository.
+ your repository. Run 'createprestorepo -m <repo dir>' instead if you want to
+ merge the presto information into the normal repository information.
+
+To remove old deltarpms:
+1- First remove old rpms from your repository using the tools available in
+ yum-utils
+2- Run 'prunedrpms <repo dir>' where <repo dir> is the base directory for your
+ repository.
-WARNING: createprestorepo does *NOT* yet know how to deal with the metadata
- created by createrepo. You will have to manually move metadata from
- .olddata to repodata after running both createprestorepo and
- createrepo.
diff --git a/presto-utils/createdeltarpms b/presto-utils/createdeltarpms
index 99bedcc..18cee00 100755
--- a/presto-utils/createdeltarpms
+++ b/presto-utils/createdeltarpms
@@ -1,2 +1,2 @@
#!/bin/sh
-exec /usr/share/createprestorepo/gendeltarpms.py "$@"
+exec /usr/share/presto-utils/gendeltarpms.py "$@"
diff --git a/presto-utils/createprestorepo b/presto-utils/createprestorepo
deleted file mode 100755
index 5334a80..0000000
--- a/presto-utils/createprestorepo
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-exec /usr/share/createprestorepo/genprestometadata.py "$@"
diff --git a/presto-utils/createprestorepo-0.2 b/presto-utils/createprestorepo-0.2
new file mode 100755
index 0000000..ed25c95
--- /dev/null
+++ b/presto-utils/createprestorepo-0.2
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/share/presto-utils/genprestometadata.py "$@"
diff --git a/presto-utils/createprestorepo-0.3 b/presto-utils/createprestorepo-0.3
new file mode 100755
index 0000000..a8f5200
--- /dev/null
+++ b/presto-utils/createprestorepo-0.3
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/share/presto-utils/genpresto.py "$@"
diff --git a/prunedrpms/PruneDrpms.py b/presto-utils/doprunedrpms.py
index 629970a..629970a 100755
--- a/prunedrpms/PruneDrpms.py
+++ b/presto-utils/doprunedrpms.py
diff --git a/presto-utils/dumpMetadata.py b/presto-utils/dumpMetadata.py
index 0ec2c20..61b30d6 100644
--- a/presto-utils/dumpMetadata.py
+++ b/presto-utils/dumpMetadata.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python -t
# base classes and functions for dumping out package Metadata
#
# This program is free software; you can redistribute it and/or modify
diff --git a/presto-utils/genpresto.py b/presto-utils/genpresto.py
new file mode 100755
index 0000000..24330a1
--- /dev/null
+++ b/presto-utils/genpresto.py
@@ -0,0 +1,313 @@
+#!/usr/bin/python
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# Copyright 2007 Red Hat, Inc. -- Jeremy Katz <katzj@redhat.com>
+# Based on genprestometadata.py which was based on genmetadata.py
+# Copyright 2007 Jonathan Dieter, Copyright 2004 Duke University
+
+import os, sys, string
+import optparse
+import gzip
+import rpm
+import types
+import sha
+import struct
+import libxml2
+
+__version__ = '0.3.0'
+
+class MDError(Exception):
+ pass
+
+def getFileList(directory, ext):
+ extlen = len(ext)
+
+ def extension_visitor(arg, dirname, names):
+ for fn in names:
+ if os.path.isdir(fn):
+ continue
+ elif string.lower(fn[-extlen:]) == '%s' % (ext):
+ reldir = os.path.basename(dirname)
+ if reldir == os.path.basename(directory):
+ reldir = ""
+ arg.append(os.path.join(reldir,fn))
+
+ rpmlist = []
+ startdir = os.path.join(directory)
+ os.path.walk(startdir, extension_visitor, rpmlist)
+ return rpmlist
+
+def generateXML(doc, node, drpmObj, sumtype, pkgDeltas):
+ """takes an xml doc object and a package metadata entry node, populates a
+ package node with the md information"""
+ name = drpmObj.tagByName('name')
+ arch = drpmObj.tagByName('arch')
+ epoch = str(drpmObj.epoch())
+ ver = str(drpmObj.tagByName('version'))
+ rel = str(drpmObj.tagByName('release'))
+ if not pkgDeltas.has_key('%s-%s:%s-%s.%s' % (name, epoch, ver, rel, arch)):
+ pkgNode = node.newChild(None, "newpackage", None)
+ pkgNode.newProp('name', name)
+ pkgNode.newProp('epoch', epoch)
+ pkgNode.newProp('version', ver)
+ pkgNode.newProp('release', rel)
+ pkgNode.newProp('arch', arch)
+ pkgDeltas['%s-%s:%s-%s.%s' % (name, epoch, ver, rel, arch)] = pkgNode
+ else:
+ pkgNode = pkgDeltas['%s-%s:%s-%s.%s' % (name, epoch, ver, rel, arch)]
+ delta = pkgNode.newChild(None, "delta", None)
+ delta.newChild(None, 'filename', drpmObj.relativepath)
+ delta.newChild(None, 'sequence', "%s-%s" %(drpmObj.oldnevrstring, drpmObj.sequence))
+ delta.newChild(None, 'size', str(drpmObj.size))
+ sum = delta.newChild(None, 'checksum', drpmObj.pkgid)
+ sum.newProp('type', 'sha')
+ (oldname, oldepoch, oldver, oldrel) = drpmObj.oldnevr
+ delta.newProp('oldepoch', oldepoch)
+ delta.newProp('oldversion', oldver)
+ delta.newProp('oldrelease', oldrel)
+
+
+def byteranges(file):
+ """takes an rpm file or fileobject and returns byteranges for location of the header"""
+ opened_here = 0
+ if type(file) is not types.StringType:
+ fo = file
+ else:
+ opened_here = 1
+ fo = open(file, 'r')
+ #read in past lead and first 8 bytes of sig header
+ fo.seek(104)
+ # 104 bytes in
+ binindex = fo.read(4)
+ # 108 bytes in
+ (sigindex, ) = struct.unpack('>I', binindex)
+ bindata = fo.read(4)
+ # 112 bytes in
+ (sigdata, ) = struct.unpack('>I', bindata)
+ # each index is 4 32bit segments - so each is 16 bytes
+ sigindexsize = sigindex * 16
+ sigsize = sigdata + sigindexsize
+ # we have to round off to the next 8 byte boundary
+ disttoboundary = (sigsize % 8)
+ if disttoboundary != 0:
+ disttoboundary = 8 - disttoboundary
+ # 112 bytes - 96 == lead, 8 = magic and reserved, 8 == sig header data
+ hdrstart = 112 + sigsize + disttoboundary
+
+ fo.seek(hdrstart) # go to the start of the header
+ fo.seek(8,1) # read past the magic number and reserved bytes
+
+ binindex = fo.read(4)
+ (hdrindex, ) = struct.unpack('>I', binindex)
+ bindata = fo.read(4)
+ (hdrdata, ) = struct.unpack('>I', bindata)
+
+ # each index is 4 32bit segments - so each is 16 bytes
+ hdrindexsize = hdrindex * 16
+ # add 16 to the hdrsize to account for the 16 bytes of misc data b/t the
+ # end of the sig and the header.
+ hdrsize = hdrdata + hdrindexsize + 16
+
+ # header end is hdrstart + hdrsize
+ hdrend = hdrstart + hdrsize
+ if opened_here:
+ fo.close()
+ del fo
+ return (hdrstart, hdrend)
+
+class DrpmMetaData:
+ """each drpm is one object, you pass it an rpm file
+ it opens the file, and pulls the information out in bite-sized chunks :)
+ """
+
+ mode_cache = {}
+
+ def __init__(self, ts, basedir, filename):
+ try:
+ stats = os.stat(os.path.join(basedir, filename))
+ self.size = stats[6]
+ self.mtime = stats[8]
+ del stats
+ except OSError, e:
+ raise MDError, "Error Stat'ing file %s %s" % (basedir, filename)
+ self.relativepath = filename
+ fd = os.open(os.path.join(basedir, filename), os.O_RDONLY)
+ self.hdr = ts.hdrFromFdno(fd)
+ os.lseek(fd, 0, 0)
+ fo = os.fdopen(fd, 'rb')
+ self.pkgid = self.getChecksum("sha", fo)
+ fo.seek(0)
+ (start, end) = byteranges(fo)
+ fo.seek(end)
+ self._getOldInfo(fo)
+ del fo
+ del fd
+
+ def arch(self):
+ if self.tagByName('sourcepackage') == 1:
+ return 'src'
+ else:
+ return self.tagByName('arch')
+
+ def _stringToNEVR(self, string):
+ i = string.rfind("-", 0, string.rfind("-")-1)
+ name = string[:i]
+ (epoch, ver, rel) = self._stringToVersion(string[i+1:])
+ return (name, epoch, ver, rel)
+
+ def _getLength(self, in_data):
+ length = 0
+ for val in in_data:
+ length = length * 256
+ length += ord(val)
+ return length
+
+ def _getOldInfo(self, fo):
+ try:
+ compobj = gzip.GzipFile("", "rb", 9, fo)
+ except:
+ raise zlibError("Data not stored in gzip format")
+
+ if compobj.read(4)[:3] != "DLT":
+ raise Exception("Not a deltarpm")
+
+ nevr_length = self._getLength(compobj.read(4))
+ nevr = compobj.read(nevr_length).strip("\x00")
+ seq_length = self._getLength(compobj.read(4))
+ seq = compobj.read(seq_length)
+ hex_seq = ""
+ for char in seq:
+ hex_seq += str("%02x" % ord(char))
+ self.oldnevrstring = nevr
+ self.oldnevr = self._stringToNEVR(nevr)
+ self.sequence = hex_seq
+ compobj.close()
+
+ def _stringToVersion(self, strng):
+ i = strng.find(':')
+ if i != -1:
+ epoch = strng[:i]
+ else:
+ epoch = '0'
+ j = strng.find('-')
+ if j != -1:
+ if strng[i + 1:j] == '':
+ version = None
+ else:
+ version = strng[i + 1:j]
+ release = strng[j + 1:]
+ else:
+ if strng[i + 1:] == '':
+ version = None
+ else:
+ version = strng[i + 1:]
+ release = None
+ return (epoch, version, release)
+
+ def tagByName(self, tag):
+ data = self.hdr[tag]
+ if type(data) is types.ListType:
+ if len(data) > 0:
+ return data[0]
+ else:
+ return ''
+ else:
+ return data
+
+ def epoch(self):
+ if self.hdr['epoch'] is None:
+ return 0
+ else:
+ return self.tagByName('epoch')
+
+ def getChecksum(self, sumtype, file, CHUNK=2**16):
+ """takes filename, hand back Checksum of it
+ sumtype = md5 or sha
+ filename = /path/to/file
+ CHUNK=65536 by default"""
+
+ # chunking brazenly lifted from Ryan Tomayko
+ opened_here = 0
+ try:
+ if type(file) is not types.StringType:
+ fo = file # assume it's a file-like-object
+ else:
+ opened_here = 1
+ fo = open(file, 'rb', CHUNK)
+
+ if sumtype == 'md5':
+ sum = md5.new()
+ elif sumtype == 'sha':
+ sum = sha.new()
+ else:
+ raise MDError, 'Error Checksumming file, wrong checksum type %s' % sumtype
+ chunk = fo.read
+ while chunk:
+ chunk = fo.read(CHUNK)
+ sum.update(chunk)
+
+ if opened_here:
+ fo.close()
+ del fo
+
+ return sum.hexdigest()
+ except Exception, e:
+ print e
+ raise MDError, 'Error opening file for checksum: %s' % file
+
+def writePrestoData(deltadir, outputdir):
+ files = getFileList(deltadir, ".drpm")
+
+ doc = libxml2.newDoc("1.0")
+ root = doc.newChild(None, "prestodeltas", None)
+
+ deltas = {}
+ ts = rpm.TransactionSet()
+ ts.setVSFlags(-1)
+ for f in files:
+ drpmobj = DrpmMetaData(ts, deltadir, f)
+ generateXML(doc, root, drpmobj, "sha", deltas)
+
+ prestofile = open("%s/prestodelta.xml" %(outputdir,), "w")
+ prestofile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ prestofile.write(root.serialize("UTF-8", True))
+ prestofile.close()
+
+def usage():
+ print >> sys.stderr, "Usage: %s <deltadir> <outputdir>" %(sys.argv[0])
+
+def main(args):
+ if len(args) == 0:
+ usage()
+ sys.exit(1)
+
+ deltadir = args[0]
+ if len(args) > 1:
+ outputdir = args[1]
+ else:
+ outputdir = "%s/repodata" %(deltadir,)
+
+ if not os.path.isdir(deltadir):
+ print >> sys.stderr, "Delta directory must exist."
+ sys.exit(1)
+ if not os.access(outputdir, os.W_OK):
+ print >> sys.stderr, "Output directory must be writable."
+ sys.exit(1)
+
+ writePrestoData(deltadir, outputdir)
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
diff --git a/presto-utils/prunedrpms b/presto-utils/prunedrpms
new file mode 100755
index 0000000..65ee9dc
--- /dev/null
+++ b/presto-utils/prunedrpms
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/share/presto-utils/doprunedrpms.py "$@"
diff --git a/presto-utils/relaxng/presto.rnc b/presto-utils/relaxng/presto.rnc
new file mode 100644
index 0000000..90a4c34
--- /dev/null
+++ b/presto-utils/relaxng/presto.rnc
@@ -0,0 +1,20 @@
+start = element prestodeltas { newpackage+ }
+newpackage = element newpackage {
+ attribute name { text },
+ attribute epoch { text }?,
+ attribute version { text },
+ attribute release { text },
+ attribute arch { text },
+ element delta {
+ attribute oldepoch { text }?,
+ attribute oldversion { text }?,
+ attribute oldrelease { text }?,
+ element filename { text },
+ element sequence { text },
+ element size { text },
+ element checksum {
+ attribute type { "sha" | "md5" },
+ text
+ }
+ }+
+}
diff --git a/prunedrpms/ChangeLog b/prunedrpms/ChangeLog
deleted file mode 100644
index 2de29c0..0000000
--- a/prunedrpms/ChangeLog
+++ /dev/null
@@ -1,2 +0,0 @@
-* Tue May 7 2007 Jonathan Dieter <jdieter@gmail.com> - 0.1.0
- - Initial release
diff --git a/prunedrpms/Makefile b/prunedrpms/Makefile
deleted file mode 100644
index cd13c34..0000000
--- a/prunedrpms/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-clean:
- rm -f *.pyc *.pyo *~
-
-install:
- install -m 755 prunedrpms $(DESTDIR)/usr/bin/
- mkdir -p $(DESTDIR)/usr/share/prunedrpms
- install -m 755 PruneDrpms.py $(DESTDIR)/usr/share/prunedrpms
diff --git a/prunedrpms/README b/prunedrpms/README
deleted file mode 100644
index 036a81e..0000000
--- a/prunedrpms/README
+++ /dev/null
@@ -1,16 +0,0 @@
-Presto: A project to add delta rpm support into yum for Fedora users
-http://hosted.fedoraproject.org/projects/presto. A list of presto-enabled
-repositories is available there.
-
-prunedrpms: A tool that removes all drpms that point to rpms no longer in your
- repository
-
-Installation:
-=============
-1- Untar the package
-2- Run 'make install'
-
-Running:
-========
-1- Run 'prunedrpms <repo dir>' where <repo dir> is the base directory for your
- repository
diff --git a/prunedrpms/prunedrpms b/prunedrpms/prunedrpms
deleted file mode 100755
index 7f33c00..0000000
--- a/prunedrpms/prunedrpms
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-exec /usr/share/prunedrpms/PruneDrpms.py "$@"
diff --git a/prunedrpms/COPYING b/yum-presto-legacy/COPYING
index e77696a..e77696a 100644
--- a/prunedrpms/COPYING
+++ b/yum-presto-legacy/COPYING
diff --git a/yum-presto-legacy/ChangeLog b/yum-presto-legacy/ChangeLog
new file mode 100644
index 0000000..05c5c41
--- /dev/null
+++ b/yum-presto-legacy/ChangeLog
@@ -0,0 +1,68 @@
+* Tue May 1 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.10
+ - Use new -a option to deltarpm to only check against a certain architecture.
+ This allows us to work completely correctly on x86_64.
+ - Add "*" to repository of deltarpm as it *doesn't* screw up depsolving.
+
+* Sun Apr 15 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.9
+ - Modifications to make yum-presto compatible with both FC6 and Rawhide
+ - Removed "*" next to deltarpms because it screws up depsolving
+ - Fixed bug that showed log error message when running yum list as non super-
+ user
+ - Fixed bug that caused yum to hang forever when deltarpm downloads were
+ Ctrl-C'd
+
+* Fri Apr 6 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.8
+ - Small bugfix
+
+* Thu Apr 5 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.6
+ - Some minor changes to README and configuration file
+
+* Wed Apr 4 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.5
+ - Many small bug fixes
+ - Improved logging - Large changes
+
+* Tue Apr 3 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.4
+ - Add patch from Ahmed Kamal to put rebuilding of rpms into a
+ different thread
+
+* Fri Mar 30 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.3
+ - Change to how presto.xml.gz stores sequence. An attempt to
+ save space doesn't always work, so get rid of it.
+
+* Thu Mar 29 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.2
+ - Minor fixes to make rpmlint happy
+ - Minor fix so yum doesn't die when download fails
+ - Minor fix to allow public keys to be imported properly
+ - Update README
+
+* Wed Mar 28 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.0
+ - Massive changes to downloading structure
+ - When unable to rebuild drpm, we now download full rpm
+ - Stop doing slow MD5 check and just check RPM header
+ while we have a prelink bug
+
+* Mon Mar 26 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.9
+ - Fix another mirrorlist bug
+ - Minor optimization
+ - Added logging to /var/log/presto.log
+ - Fix another mirrorlist bug
+ - Fix bug where we sometimes die if delta repository doesn't exist
+ - Properly exit when unable to rebuild drpm
+ - Do full (slow) MD5 check when checking to see if we can
+ build RPM from disk
+
+* Sat Mar 24 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.3
+ - Fixed bug that breaks yum install
+
+* Sat Mar 24 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.2
+ - Fixed "not showing download error" bug
+ - Added --disablepresto yum command-line option
+ - Added code to trap the (hopefully) unlikely scenario where applydeltarpm
+ fails
+ - Show byte savings at end of yum update
+
+* Fri Mar 23 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.1
+ - Fixed bug in handling mirrorlists in original repositories
+
+* Thu Mar 22 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.0
+ - Initial release
diff --git a/yum-presto-legacy/Makefile b/yum-presto-legacy/Makefile
new file mode 100644
index 0000000..8826b30
--- /dev/null
+++ b/yum-presto-legacy/Makefile
@@ -0,0 +1,17 @@
+clean:
+ rm -f *.pyc *.pyo *~
+ cd shared; rm -f *.pyc *.pyo *~
+
+install:
+ mkdir -p $(DESTDIR)/usr/lib/yum-plugins
+ install -m 644 presto.py $(DESTDIR)/usr/lib/yum-plugins
+ mkdir -p $(DESTDIR)/etc/yum/pluginconf.d
+ install -m 644 presto.conf $(DESTDIR)/etc/yum/pluginconf.d
+ mkdir -p $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestoRepo.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestomdparser.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestoTransaction.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestoThread.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestoLog.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestoDownload.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/deltarpm.py $(DESTDIR)/usr/share/presto
diff --git a/yum-presto-legacy/README b/yum-presto-legacy/README
new file mode 100644
index 0000000..4e53751
--- /dev/null
+++ b/yum-presto-legacy/README
@@ -0,0 +1,31 @@
+Presto: A project to add delta rpm support into yum for Fedora users
+http://hosted.fedoraproject.org/projects/presto. A list of presto-enabled
+repositories is available there.
+
+Installation:
+=============
+1- Install yum-presto on your system (yum -y install yum-presto)
+2- Either use a repository that already has presto enabled in it or modify
+ your .repo files so they contain the following:
+ deltaurl = http://your-presto-repository.com
+ where your-presto-repository.com is replaced with a valid presto repository
+3- Now install an old rpm from a presto-enabled repository using rpm, then
+ try updating it using yum. The plugin should kick in, try to download the
+ drpm, reconstruct the full rpm, and yum should install that.
+
+Notes:
+======
+Presto will read the deltaurl from two possible locations:
+1. The repository's .repo file ("deltaurl = http://your-presto-repository.com")
+2. Appended to /etc/yum/pluginconf.d/presto.conf in the form:
+ [repository]
+ deltaurl = http://your-presto-repository.com
+
+Presto.conf has the following options in [main]:
+keepdeltas=1 Always keep deltas in cache no matter what keepcache
+ is set to.
+neverkeepdeltas=1 Always remove deltas after creating full rpms.
+exitondownloadfailure=0|1 If there is a problem downloading the deltarpm, exit
+ rather than trying to download the full rpm.
+Note: If you specify neither keepdeltas nor neverkeepdeltas, presto will follow
+ the keepcache option in yum.conf.
diff --git a/yum-presto-legacy/presto.conf b/yum-presto-legacy/presto.conf
new file mode 100644
index 0000000..488f69f
--- /dev/null
+++ b/yum-presto-legacy/presto.conf
@@ -0,0 +1,16 @@
+# Please go to http://hosted.fedoraproject.org/projects/presto for a list of
+# presto-enabled repositories
+
+[main]
+enabled=1
+neverkeepdeltas=1
+
+# Setup for test server, enable if you are testing
+#[updates]
+#deltaurl=http://www.lesbg.com/jdieter/updates/fc6/i386
+#
+#[extras]
+#deltaurl=http://www.lesbg.com/jdieter/extras/fc6/i386
+#
+#[development]
+#deltaurl=http://www.lesbg.com/jdieter/updates/development/i386
diff --git a/yum-presto-legacy/presto.py b/yum-presto-legacy/presto.py
new file mode 100644
index 0000000..48db24d
--- /dev/null
+++ b/yum-presto-legacy/presto.py
@@ -0,0 +1,163 @@
+# author: Jonathan Dieter <jdieter@gmail.com>
+#
+# heavily modified from yum-deltarpm.py created by
+# Lars Herrmann <herrmann@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2005 Duke University
+# Copyright 2007 Jonathan Dieter
+
+from yum.plugins import TYPE_INTERACTIVE, PluginYumExit
+from yum import config
+
+import os
+import sys
+
+sys.path.append("/usr/share/presto")
+import deltarpm
+from prestoRepo import PrestoRepository
+from prestomdparser import PrestoMDParser
+import prestoTransaction
+import prestoLog
+import prestoDownload
+
+requires_api_version = '2.1'
+LOG_FILE = "/var/log/presto.log"
+plugin_type = (TYPE_INTERACTIVE,)
+
+rpm_size = 0
+drpm_size = 0
+drpm_count = 0
+log = None
+
+# Configuration stuff
+def config_hook(conduit):
+ # Set up repository specific deltarpm url and mirrorlist
+ config.RepoConf.deltaurl = config.UrlListOption()
+ config.RepoConf.deltamirrorlist = config.UrlOption()
+
+ # Add --disable-presto option
+ parser = conduit.getOptParser()
+ parser.add_option('', '--disablepresto', dest='disablepresto',
+ action='store_true', default=False,
+ help="disable Presto plugin and don't download any deltarpms")
+
+# Set up Presto repositories
+def postreposetup_hook(conduit):
+ opts, commands = conduit.getCmdLine()
+ if not opts.disablepresto:
+ conduit.info(2, 'Setting up Presto')
+ for active_repo in conduit.getRepos().listEnabled():
+ p_repo = PrestoRepository(active_repo, conduit)
+ p_repo.setup(conduit.getConf().cache)
+
+ conduit.info(2, 'Reading Presto metadata in from local files')
+ for active_repo in conduit.getRepos().listEnabled():
+ xml = active_repo.p_repo.getPrestoXML()
+ if active_repo.p_repo.enabled:
+ xmldata = active_repo.p_repo.repoXML.getData('deltas')
+ (ctype, csum) = xmldata.checksum
+ parser = PrestoMDParser(xml)
+ active_repo.p_repo.deltalist = parser.getDeltaList()
+ else:
+ conduit.info(5, '--disablepresto specified - Presto disabled')
+
+
+def postresolve_hook(conduit):
+ global rpm_size
+ global drpm_size
+ global drpm_count
+
+ opts, commands = conduit.getCmdLine()
+ if not opts.disablepresto:
+ # Cycle through packages to see if there's a deltarpm available
+ for newpkg in conduit.getTsInfo():
+ if newpkg.ts_state != "e":
+ (chosen_drpm, installed, local, drpm_enabled) = prestoTransaction.find_available_drpms(conduit, newpkg)
+
+ # If a drpm was found, change certain package information so it reflects
+ # the drpm, not the rpm.
+ if chosen_drpm != None:
+ newpkg.po.has_drpm = True
+ conduit.info(2, "Found deltarpm update for %s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release))
+ # In yum 3.0.x, this doesn't get defined if you run "yum update x" rather than "yum update"
+ rpm_size += int(newpkg.po.size)
+ drpm_size += int(chosen_drpm['size'])
+ newpkg.po.realpackagesize = newpkg.po.size
+ if hasattr(newpkg.po, 'packagesize'):
+ newpkg.po.packagesize = chosen_drpm['size']
+ else:
+ newpkg.po.simple['packagesize'] = chosen_drpm['size']
+ newpkg.po.deltasize = chosen_drpm['size']
+ newpkg.po.deltarelativepath = chosen_drpm['drpm_filename']
+ newpkg.po.deltachecksumtype = chosen_drpm['checksum_type']
+ newpkg.po.deltachecksum = chosen_drpm['checksum']
+ newpkg.po.deltalocalpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename'])
+ newpkg.po.to = newpkg
+ newpkg.po.hasdrpm = True
+ newpkg.repoid = newpkg.po.repo.id + " *"
+ drpm_count += 1
+ else:
+ if installed and drpm_enabled and not local:
+ try:
+ rpm_size += int(newpkg.po.size)
+ drpm_size += int(newpkg.po.size)
+ except:
+ pass
+ return
+
+
+def predownload_hook(conduit):
+ global drpm_count
+ global log
+
+ # Set up logging
+ log = prestoLog.PrestoLog(conduit, LOG_FILE)
+
+ pkglist = conduit.getDownloadPackages()
+
+ opts, commands = conduit.getCmdLine()
+ if not opts.disablepresto and drpm_count > 0:
+ conduit.info(2, "Downloading DeltaRPMs:")
+
+ # Download deltarpms
+ problems = prestoDownload.downloadPkgs(conduit, pkglist, log)
+
+ # If 'exitondownloaderror' is on, exit
+ if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0:
+ errstring = 'Error Downloading Packages:\n'
+ for key in problems.keys():
+ errors = misc.unique(problems[key])
+ for error in errors:
+ errstring += ' %s: %s\n' % (key, error)
+ raise PluginYumExit(errstring)
+
+ else:
+ conduit.info(2, "Downloading RPMs:")
+
+def posttrans_hook(conduit):
+ global rpm_size
+ global drpm_size
+ global log
+
+ log.close()
+
+ if rpm_size > 0:
+ drpm_string = prestoTransaction.format_number(drpm_size)
+ rpm_string = prestoTransaction.format_number(rpm_size)
+
+ conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %s" % drpm_string)
+ conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %s" % rpm_string)
+ conduit.info(2, "This is a savings of %i percent" % (100 - ((drpm_size * 100) / rpm_size)))
diff --git a/yum-presto/shared/deltarpm.py b/yum-presto-legacy/shared/deltarpm.py
index 97a3cc6..97a3cc6 100644
--- a/yum-presto/shared/deltarpm.py
+++ b/yum-presto-legacy/shared/deltarpm.py
diff --git a/yum-presto/shared/prestoDownload.py b/yum-presto-legacy/shared/prestoDownload.py
index 1c29608..1c29608 100644
--- a/yum-presto/shared/prestoDownload.py
+++ b/yum-presto-legacy/shared/prestoDownload.py
diff --git a/yum-presto/shared/prestoLog.py b/yum-presto-legacy/shared/prestoLog.py
index 1323346..1323346 100644
--- a/yum-presto/shared/prestoLog.py
+++ b/yum-presto-legacy/shared/prestoLog.py
diff --git a/yum-presto/shared/prestoRepo.py b/yum-presto-legacy/shared/prestoRepo.py
index bc1b188..bc1b188 100644
--- a/yum-presto/shared/prestoRepo.py
+++ b/yum-presto-legacy/shared/prestoRepo.py
diff --git a/yum-presto/shared/prestoThread.py b/yum-presto-legacy/shared/prestoThread.py
index e090910..e090910 100644
--- a/yum-presto/shared/prestoThread.py
+++ b/yum-presto-legacy/shared/prestoThread.py
diff --git a/yum-presto/shared/prestoTransaction.py b/yum-presto-legacy/shared/prestoTransaction.py
index ffaad01..ffaad01 100644
--- a/yum-presto/shared/prestoTransaction.py
+++ b/yum-presto-legacy/shared/prestoTransaction.py
diff --git a/yum-presto/shared/prestomdparser.py b/yum-presto-legacy/shared/prestomdparser.py
index 6764f71..6764f71 100644
--- a/yum-presto/shared/prestomdparser.py
+++ b/yum-presto-legacy/shared/prestomdparser.py
diff --git a/yum-presto/ChangeLog b/yum-presto/ChangeLog
index 05c5c41..0007bf7 100644
--- a/yum-presto/ChangeLog
+++ b/yum-presto/ChangeLog
@@ -1,3 +1,8 @@
+* Wed Jul 11 2007 Jonathan Dieter <jdieter@gmail.com> - 0.4.0
+ - Complete rewrite by Jeremy Katz.
+ - Many old options and features removed in preparation for use as part of
+ Fedora 8.
+
* Tue May 1 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.10
- Use new -a option to deltarpm to only check against a certain architecture.
This allows us to work completely correctly on x86_64.
diff --git a/yum-presto/Makefile b/yum-presto/Makefile
index 8826b30..c756c05 100644
--- a/yum-presto/Makefile
+++ b/yum-presto/Makefile
@@ -1,17 +1,8 @@
clean:
rm -f *.pyc *.pyo *~
- cd shared; rm -f *.pyc *.pyo *~
install:
mkdir -p $(DESTDIR)/usr/lib/yum-plugins
install -m 644 presto.py $(DESTDIR)/usr/lib/yum-plugins
mkdir -p $(DESTDIR)/etc/yum/pluginconf.d
install -m 644 presto.conf $(DESTDIR)/etc/yum/pluginconf.d
- mkdir -p $(DESTDIR)/usr/share/presto
- install -m 644 shared/prestoRepo.py $(DESTDIR)/usr/share/presto
- install -m 644 shared/prestomdparser.py $(DESTDIR)/usr/share/presto
- install -m 644 shared/prestoTransaction.py $(DESTDIR)/usr/share/presto
- install -m 644 shared/prestoThread.py $(DESTDIR)/usr/share/presto
- install -m 644 shared/prestoLog.py $(DESTDIR)/usr/share/presto
- install -m 644 shared/prestoDownload.py $(DESTDIR)/usr/share/presto
- install -m 644 shared/deltarpm.py $(DESTDIR)/usr/share/presto
diff --git a/yum-presto/presto.conf b/yum-presto/presto.conf
index 488f69f..b7dc4a9 100644
--- a/yum-presto/presto.conf
+++ b/yum-presto/presto.conf
@@ -5,12 +5,3 @@
enabled=1
neverkeepdeltas=1
-# Setup for test server, enable if you are testing
-#[updates]
-#deltaurl=http://www.lesbg.com/jdieter/updates/fc6/i386
-#
-#[extras]
-#deltaurl=http://www.lesbg.com/jdieter/extras/fc6/i386
-#
-#[development]
-#deltaurl=http://www.lesbg.com/jdieter/updates/development/i386
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index 48db24d..255372c 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -18,36 +18,342 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2005 Duke University
# Copyright 2007 Jonathan Dieter
-
-from yum.plugins import TYPE_INTERACTIVE, PluginYumExit
-from yum import config
+# Copyright 2007 Red Hat, Inc. -- Jeremy Katz <katzj@redhat.com>
import os
import sys
+import subprocess
+import gzip
+import thread
+import threading
+import Queue
+try:
+ from cElementTree import iterparse
+except:
+ from xml.etree.cElementTree import iterparse
+
+from yum.plugins import TYPE_CORE, PluginYumExit
+import yum.Errors
+import yum.misc
+from urlgrabber.grabber import URLGrabError
-sys.path.append("/usr/share/presto")
-import deltarpm
-from prestoRepo import PrestoRepository
-from prestomdparser import PrestoMDParser
-import prestoTransaction
-import prestoLog
-import prestoDownload
requires_api_version = '2.1'
-LOG_FILE = "/var/log/presto.log"
-plugin_type = (TYPE_INTERACTIVE,)
+plugin_type = (TYPE_CORE,)
+
+# mapping of repo.id -> PrestoInfo
+pinfo = {}
+
+def verifyDelta(sequence, arch):
+ if subprocess.call(["/usr/bin/applydeltarpm", "-a", arch,
+ "-C", "-s", sequence]):
+ return False
+ return True
+
+def applyDelta(deltarpmfile, newrpmfile, arch):
+ if subprocess.call(["/usr/bin/applydeltarpm", "-a", arch,
+ deltarpmfile, newrpmfile]):
+ return False
+ return True
+
+def reconstruct(conduit, rpmlocal, rpmarch, deltalocal):
+ retlist = ""
+
+ if not applyDelta(deltalocal, rpmlocal, rpmarch):
+ retlist += "Error rebuilding rpm from %s! Will download full package.\n" % os.path.basename(deltalocal)
+ try:
+ os.unlink(rpmlocal)
+ except:
+ pass
+ else:
+ # Check to see whether or not we should keep the drpms
+ # FIXME: Is there any way to see whether or not a Boolean option was not set?
+ if conduit.confBool('main', 'neverkeepdeltas'):
+ delete = True
+ elif conduit.confBool('main', 'keepdeltas'):
+ delete = False
+ elif conduit.getConf().keepcache != 0:
+ delete = False
+ else:
+ delete = True
+
+ if delete:
+ try:
+ os.unlink(deltalocal)
+ except:
+ pass
+ return retlist
+
+class ReconstructionThread(threading.Thread):
+ def __init__(self, queue, lock, run_function):
+ threading.Thread.__init__(self)
+ self.run_function = run_function
+ self.queue = queue
+ self.lock = lock
+ self.can_exit = False
+ self.messages = ""
+
+ def run(self):
+ while True:
+ try:
+ retval = self.queue.get(not self.can_exit)
+ except Queue.Empty:
+ # If we're done with our drpms and no more are coming, let's
+ # blow this joint
+ break
+ if retval != None:
+ messages = apply(self.run_function, retval)
+ if self.can_exit:
+ # If there are not going to be any more new drpms,
+ # send messages directly to conduit
+ conduit = retval[0]
+ if self.messages != "":
+ conduit.info(2, self.messages[:-1])
+ self.messages = ""
+ if messages != "":
+ conduit.info(2, messages[:-1])
+ else:
+ # We may be downloading drpms still, so queue messages
+ self.lock.acquire()
+ self.messages += messages
+ self.lock.release()
+
+
+def getDelta(po, presto, rpmdb):
+ """Does the package have a reasonable delta for us to use?"""
+
+ # local packages don't make sense to use a delta for...
+ if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
+ return None
+ if po.remote_url.startswith("file:/"):
+ # kind of a hack, but file:/ repos are basically local
+ return None
+
+ # if there's not presto info for the repo, we don't have a delta for
+ # the package
+ if not presto.has_key(po.repo.id):
+ return None
+ deltainfo = presto[po.repo.id]
+
+ # any deltas for the new package in the repo?
+ nevra = "%s-%s:%s-%s.%s" %(po.name, po.epoch, po.version,
+ po.release, po.arch)
+ if not deltainfo.has_key(nevra):
+ return None
+ deltas = deltainfo[nevra]
+
+ # check to see if we've already got the full package
+ local = po.localPkg()
+ if os.path.exists(local):
+ cursize = os.stat(local)[6]
+ totsize = long(po.size)
+ if po.verifyLocalPkg(): # we've got it.
+ return None
+ if cursize < totsize: # we have part of the file; do a reget
+ return None
+ os.unlink(local)
+
+ # did we have a previous package of the same arch installed?
+ installed = rpmdb.searchNevra(po.name, None, None, None, po.arch)
+ if len(installed) == 0:
+ return None
+
+ # now, let's see if there's a delta for us...
+ bestdelta = None
+
+ for oldpo in installed:
+ evr = "%s:%s-%s" %(oldpo.epoch, oldpo.version, oldpo.release)
+ if not deltas.has_key(evr):
+ continue
+ delta = deltas[evr]
+
+ # we just want to use the smallest delta
+ if bestdelta and delta['size'] >= bestdelta['size']:
+ continue
+
+ if not verifyDelta(delta['sequence'], po.arch):
+ continue
+
+ bestdelta = delta
+
+ return bestdelta
+
+
+def downloadPkgs(conduit, presto):
+ """download list of package objects handed to you, return errors"""
+
+ errors = {}
+ def adderror(po, msg):
+ errors.setdefault(po, []).append(msg)
+
+ # Set up thread for applying drpms
+ queue = Queue.Queue(0)
+ lock = thread.allocate_lock()
+ curthread = ReconstructionThread(queue, lock, reconstruct)
+ curthread.start()
+
+ remote_pkgs = []
+
+ # see which deltas we need to download; if the delta is already
+ # downloaded, we can start it reconstructing in the background
+ for po in conduit.getDownloadPackages():
+ delta = getDelta(po, presto, conduit.getRpmDB())
+ if delta is None:
+ continue
+
+ # verify the delta if it already exists
+ deltadir = os.path.join(po.repo.cachedir, 'deltas')
+ if not os.path.isdir(deltadir):
+ try:
+ os.mkdir(deltadir)
+ except OSError:
+ continue
+ deltapath = os.path.join(deltadir,
+ os.path.basename(delta['filename']))
+ if os.path.exists(deltapath):
+ try:
+ conduit._base.verifyChecksum(deltapath, delta['checksum_type'],
+ delta['checksum'])
+ except URLGrabError, e:
+ if po.repo.cache:
+ raise Errors.RepoError, "Caching enabled and local cache for %s doesn't match checksum" %(deltapath,)
+ else:
+ cursize = os.stat(deltapath)[6]
+ totsize = long(delta['size'])
+ if cursize >= totsize:
+ os.unlink(deltapath)
+
+ remote_packages.append( (po, delta) )
+ else:
+ # Deltarpm is local and good, put it in the rebuild thread.
+ conduit.info(5, "using local copy of deltarpm for %s" % po)
+ queue.put((conduit, po.localpath, po.arch, deltapath))
+ continue
+ else:
+ remote_pkgs.append( (po, delta) )
+
+ # now we need to do downloads
+ i = 0
+ for (po, delta) in remote_pkgs:
+ i += 1
+ # FIXME: verifyChecksum should handle the urlgrabber objects...
+ checkfunc = (lambda fo, csumtype, csum:
+ conduit._base.verifyChecksum(fo.filename, csumtype, csum),
+ (delta['checksum_type'],
+ delta['checksum']), {})
+
+ deltadir = os.path.join(po.repo.cachedir, 'deltas')
+ deltapath = os.path.join(deltadir,
+ os.path.basename(delta['filename']))
+
+ # FIXME: this should be moved into _getFile
+ dirstat = os.statvfs(deltadir)
+ if (dirstat.f_bavail * dirstat.f_bsize) <= long(po.size):
+ adderror(po, 'Insufficient space in download directory %s '
+ 'to download' % (deltadir,))
+ continue
+ try:
+ text = "(%s/%s): %s" %(i, len(remote_pkgs),
+ os.path.basename(delta['filename']))
+ deltafile = po.repo._getFile(url=po.basepath,
+ relative=delta['filename'],
+ local=deltapath,
+ checkfunc=checkfunc,
+ text=text,
+ cache=po.repo.cache)
+ except Errors.RepoError, e:
+ adderror(po, str(e))
+ else:
+ queue.put((conduit, po.localpath, po.arch, deltafile))
+
+ if errors.has_key(po):
+ del errors[po]
+
+ # Check for waiting messages from building thread
+ lock.acquire()
+ if curthread.messages != "":
+ conduit.info(2, curthread.messages[:-1])
+ curthread.messages = ""
+ lock.release()
+
+ conduit.info(2, "Rebuilding rpms from deltarpms")
+
+ # Tell build thread that there are no more drpms and wait for it to exit
+ curthread.can_exit = True
+ queue.put(None)
+ curthread.join()
+
+ if curthread.messages != "":
+ conduit.info(2, curthread.messages[:-1])
+ curthread.messages = ""
+
+ return errors
+
+class DeltaInfo(object):
+ def __init__(self, elem):
+ self.epoch = elem.get("oldepoch")
+ self.version = elem.get("oldversion")
+ self.release = elem.get("oldrelease")
+
+ self.filename = self.sequence = self.size = self.checksum = self.checksum_type = None
+
+ for x in elem.getchildren():
+ if x.tag == "checksum":
+ self.checksum_type = x.get("type")
+ setattr(self, x.tag, x.text)
+
+ def evr(self):
+ return "%s:%s-%s" %(self.epoch, self.version, self.release)
+
+ def __str__(self):
+ return "filename: %s, sequence: %s, size: %s, checksum (%s) = %s" % (self.filename, self.sequence, self.size, self.checksum_type, self.checksum)
+
+ def __getitem__(self, key):
+ return getattr(self, key)
+
+class NewPackage(object):
+ def __init__(self, elem):
+ for prop in ("name", "version", "release", "epoch", "arch"):
+ setattr(self, prop, elem.get(prop))
+
+ self.deltas = {}
+ for child in elem.getchildren():
+ if child.tag != "delta":
+ continue
+ d = DeltaInfo(child)
+ self.deltas[d.evr()] = d
+
+ def nevra(self):
+ return "%s-%s:%s-%s.%s" %(self.name, self.epoch, self.version,
+ self.release, self.arch)
+
+ def __str__(self):
+ return "%s <== %s" % (self.nevra(), self.deltas)
+
+ def has_key(self, key):
+ return self.deltas.has_key(key)
+ def __getitem__(self, key):
+ return self.deltas[key]
+
+class PrestoParser(object):
+ def __init__(self, filename):
+ self.deltainfo = {}
+
+ if filename.endswith(".gz"):
+ fo = gzip.open(filename)
+ else:
+ fo = open(filename, 'rt')
+ for event, elem in iterparse(fo):
+ if elem.tag == "newpackage":
+ p = NewPackage(elem)
+ self.deltainfo[p.nevra()] = p
+
+ def getDeltas(self):
+ return self.deltainfo
-rpm_size = 0
-drpm_size = 0
-drpm_count = 0
-log = None
# Configuration stuff
def config_hook(conduit):
- # Set up repository specific deltarpm url and mirrorlist
- config.RepoConf.deltaurl = config.UrlListOption()
- config.RepoConf.deltamirrorlist = config.UrlOption()
-
# Add --disable-presto option
parser = conduit.getOptParser()
parser.add_option('', '--disablepresto', dest='disablepresto',
@@ -58,106 +364,36 @@ def config_hook(conduit):
def postreposetup_hook(conduit):
opts, commands = conduit.getCmdLine()
if not opts.disablepresto:
- conduit.info(2, 'Setting up Presto')
+ conduit.info(2, 'Setting up and reading Presto delta metadata')
for active_repo in conduit.getRepos().listEnabled():
- p_repo = PrestoRepository(active_repo, conduit)
- p_repo.setup(conduit.getConf().cache)
-
- conduit.info(2, 'Reading Presto metadata in from local files')
- for active_repo in conduit.getRepos().listEnabled():
- xml = active_repo.p_repo.getPrestoXML()
- if active_repo.p_repo.enabled:
- xmldata = active_repo.p_repo.repoXML.getData('deltas')
- (ctype, csum) = xmldata.checksum
- parser = PrestoMDParser(xml)
- active_repo.p_repo.deltalist = parser.getDeltaList()
+ try:
+ deltamd = active_repo.retrieveMD("prestodelta")
+ except yum.Errors.RepoMDError:
+ conduit.info(2, "No Presto metadata available for %s" %(active_repo,))
+ continue
+ pinfo[active_repo.id] = PrestoParser(deltamd).getDeltas()
else:
conduit.info(5, '--disablepresto specified - Presto disabled')
-
-def postresolve_hook(conduit):
- global rpm_size
- global drpm_size
- global drpm_count
-
+
+def predownload_hook(conduit):
opts, commands = conduit.getCmdLine()
- if not opts.disablepresto:
- # Cycle through packages to see if there's a deltarpm available
- for newpkg in conduit.getTsInfo():
- if newpkg.ts_state != "e":
- (chosen_drpm, installed, local, drpm_enabled) = prestoTransaction.find_available_drpms(conduit, newpkg)
-
- # If a drpm was found, change certain package information so it reflects
- # the drpm, not the rpm.
- if chosen_drpm != None:
- newpkg.po.has_drpm = True
- conduit.info(2, "Found deltarpm update for %s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release))
- # In yum 3.0.x, this doesn't get defined if you run "yum update x" rather than "yum update"
- rpm_size += int(newpkg.po.size)
- drpm_size += int(chosen_drpm['size'])
- newpkg.po.realpackagesize = newpkg.po.size
- if hasattr(newpkg.po, 'packagesize'):
- newpkg.po.packagesize = chosen_drpm['size']
- else:
- newpkg.po.simple['packagesize'] = chosen_drpm['size']
- newpkg.po.deltasize = chosen_drpm['size']
- newpkg.po.deltarelativepath = chosen_drpm['drpm_filename']
- newpkg.po.deltachecksumtype = chosen_drpm['checksum_type']
- newpkg.po.deltachecksum = chosen_drpm['checksum']
- newpkg.po.deltalocalpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename'])
- newpkg.po.to = newpkg
- newpkg.po.hasdrpm = True
- newpkg.repoid = newpkg.po.repo.id + " *"
- drpm_count += 1
- else:
- if installed and drpm_enabled and not local:
- try:
- rpm_size += int(newpkg.po.size)
- drpm_size += int(newpkg.po.size)
- except:
- pass
+ if opts.disablepresto or len(conduit.getDownloadPackages()) == 0:
return
-
-def predownload_hook(conduit):
- global drpm_count
- global log
-
- # Set up logging
- log = prestoLog.PrestoLog(conduit, LOG_FILE)
+ conduit.info(2, "Downloading DeltaRPMs:")
- pkglist = conduit.getDownloadPackages()
-
- opts, commands = conduit.getCmdLine()
- if not opts.disablepresto and drpm_count > 0:
- conduit.info(2, "Downloading DeltaRPMs:")
-
- # Download deltarpms
- problems = prestoDownload.downloadPkgs(conduit, pkglist, log)
-
- # If 'exitondownloaderror' is on, exit
- if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0:
- errstring = 'Error Downloading Packages:\n'
- for key in problems.keys():
- errors = misc.unique(problems[key])
- for error in errors:
- errstring += ' %s: %s\n' % (key, error)
- raise PluginYumExit(errstring)
-
- else:
- conduit.info(2, "Downloading RPMs:")
+ # Download deltarpms
+ problems = downloadPkgs(conduit, pinfo)
-def posttrans_hook(conduit):
- global rpm_size
- global drpm_size
- global log
-
- log.close()
-
- if rpm_size > 0:
- drpm_string = prestoTransaction.format_number(drpm_size)
- rpm_string = prestoTransaction.format_number(rpm_size)
-
- conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %s" % drpm_string)
- conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %s" % rpm_string)
- conduit.info(2, "This is a savings of %i percent" % (100 - ((drpm_size * 100) / rpm_size)))
+ # If 'exitondownloaderror' is on, exit
+ if conduit.confBool('main', 'exitondownloaderror') and \
+ len(problems.keys()) > 0:
+ errstring = 'Error Downloading DeltaRPMs:\n'
+ for key in problems.keys():
+ errors = yum.misc.unique(problems[key])
+ for error in errors:
+ errstring += ' %s: %s\n' % (key, error)
+ raise PluginYumExit(errstring)
+
+# FIXME: would be good to give an idea to people of what they saved