path: root/scripts/splitdistro

#!/usr/bin/python2.2

# These dirs, along with RPMS, make up disc 2
disc2dirs = [ "preview" ]

# These files appear on all binary CDs
jointfiles = [ "beta_eula.txt", "RPM-GPG-KEY", "README", "GPL", "EULA"] 

targetSize = 648 * 1024.0 * 1024.0

# Leave about 1.2MB of space on the disc
fudgeFactor = 1.2 * 1024 * 1024

# rough estimate of the size of the comps package -- we don't create it
# until after we run splitdistro but need to leave room for it on CD1
compsSize = 10.0 * 1024 * 1024

#-----------------

import sys
import os
import os.path
import string
import getopt
import time
import types

def stamp(path, releasestr, num, arch, startedAt, dirInfo):
    try:
        os.unlink("%s/.discinfo" % path)
    except:
        pass
    f = open("%s/.discinfo" % path, "w")
    f.write("%f\n" % startedAt)
    f.write("%s\n" % releasestr)
    f.write("%s\n" % arch)
    f.write("%s\n" % num)
    if dirInfo is not None:
        for item in dirInfo:
            f.write("%s\n" % item)
    f.close()

def moveFiles(srcDir, destDir, files):
    for fn in files:
	src = "%s/%s" % (srcDir, fn)
	dest = "%s/%s" % (destDir, fn)
	try:
		os.rename(src, dest)
	except:
		print "in moveFiles, src was %s, dest was %s" % (src, dest)
		raise RuntimeError

def linkFile(srcDir, destDir, file):
    for file in jointfiles:
        src = "%s/%s" % (srcDir, file)
        dest = "%s/%s" %(destDir, file)
        try:
            os.link(src, dest)
        except OSError, (errno, msg):
            pass
#            print "**** WARNING linking %s to %s: %s" % (src, dest, msg)

def excessFiles(path, fileList, maxSize):
    total = 0
    moveList = []
    found = 0

    for fns in fileList:
	if type(fns) == types.StringType:
	    fns = [ fns ]

	size = 0
	for fn in fns:
	    thisSize = os.path.getsize(path + "/" + fn)
	    size = size + thisSize + ((thisSize + 2047) % 2048)

	if size + total  < maxSize:
	    total = total + size
	else:
	    # once we're done, we're done
            if found == 0:
#                print "straw was %s" % (fns,)
                found = 1
	    total = maxSize
	    moveList.extend(fns)

    return moveList

class DirectoryIterator:
    def __init__(self):
        self.inodes = {}
    
    def traverse (self, arg, dirname, names):
        for name in names:
            sb = os.lstat (dirname + "/" + name)
            if not self.inodes.has_key(sb[1]):
                self.inodes[sb[1]] = sb[6]

    def total(self):
        total = 0
        for size in self.inodes.values():
            total = total + size
	return total

def spaceUsed(path):
    foo = DirectoryIterator()
    os.path.walk (path, foo.traverse, None)
    return foo.total()

fns = {}

(args, extra) = getopt.getopt(sys.argv[1:], '', [ 'fileorder=', 'release=' ])

if len(extra) != 2:
    print "splitdistro --fileorder <file> [--release <comment>] <toppath> <arch>"
    sys.exit(1)

releasestr = ""
packageOrderFile = None
for n in args:
    (str, arg) = n
    if str == '--fileorder':
	packageOrderFile = arg
    if str == '--release':
	releasestr = arg

if not packageOrderFile:
    print "splitdistro --fileorder <file> [--release <comment>] <toppath> <arch>"
    sys.exit(1)


arch = extra[1]
distDir = os.path.normpath(extra[0] + "/" + arch)
srcDir = os.path.normpath(extra[0] + "/" + arch + "/SRPMS")

if not os.access("%s/.discinfo" %(distDir,), os.O_RDONLY):
    startedAt = time.time()
    dirInfo = None
else:
    f = open("%s/.discinfo" %(distDir,), "r")
    startedAt = string.atof(f.readline()[:-1])
    releasestr = f.readline()[:-1]
    newarch = f.readline()[:-1]
    if newarch != arch:
        raise RuntimeError, "Passed arch doesn't match arch in .discinfo file"
    # throw out discnum
    f.readline()
    # basedir, packagedir, and pixmapdir
    dirInfo = [ f.readline()[:-1], f.readline()[:-1], f.readline()[:-1] ]

if not os.path.isdir(distDir):
    print "error: %s is not a directory" % distDir
    sys.exit(1)

if not os.path.isdir(srcDir):
    print "error: %s is not a directory" % srcDir
    sys.exit(1)

disc1Dir = distDir + "-disc1"
disc2Dir = distDir + "-disc2"
disc3Dir = distDir + "-disc3"
disc1SrcDir = distDir + "-disc4"
disc2SrcDir = distDir + "-disc5"
disc3SrcDir = distDir + "-disc6"

id = 0

files = os.listdir(distDir + "/RedHat/RPMS")
files.sort()
packages = {}
for file in files:
    l = string.split(file, ".")
    pkg = string.join(l[:-2], ".")
    if packages.has_key(pkg):
	packages[pkg].append(file)
    else:
	packages[pkg] = [ file ]

f = open(packageOrderFile, "r")
binPkgList = []
for pkg in f.readlines():
    # chop
    pkg = pkg[:len(pkg) - 1]
    if pkg[0:8] != "warning:":
	binPkgList.append(packages[pkg])

del f

print "Splitting tree..."

totalsize = spaceUsed(distDir)
rpmsize = spaceUsed(distDir + "/RedHat/RPMS")
dirsize = 0
for dir in disc2dirs:
    what = distDir + "/" + dir
    if os.access(what, os.R_OK):
        dirsize = dirsize + spaceUsed(distDir + "/" + dir)
srpmsize = spaceUsed(distDir + "/SRPMS")

disc1used = totalsize - rpmsize - dirsize - srpmsize

os.system("rm -rf %s %s %s %s" % ( disc1Dir, disc2Dir, disc1SrcDir,
				   disc2SrcDir))
os.system("mkdir -p %s %s/SRPMS %s/SRPMS %s/RedHat/RPMS %s/RedHat/RPMS %s/SRPMS" % 
	    (disc1Dir, disc1SrcDir, disc2SrcDir, disc2Dir, 
             disc3Dir, disc3SrcDir))

print "Creating disc1..."

os.system("cp -al %s/. %s" % (distDir, disc1Dir))
stamp(disc1Dir, releasestr, "1", arch, startedAt, dirInfo)

# remove the srpms dir from disc1 since it shouldn't be there
os.system("rm -rf %s/SRPMS" % (disc1Dir,))

print "Creating disc2..."
stamp(disc2Dir, releasestr, "2", arch, startedAt, dirInfo)

for file in jointfiles:
    linkFile(disc1Dir, disc2Dir, file)

print targetSize / (1024.0 * 1024.0)
print disc1used / (1024.0 * 1024.0)
print fudgeFactor / (1024.0 * 1024.0)
print compsSize / (1024.0 * 1024.0)
print (targetSize - disc1used - fudgeFactor - compsSize) / (1024.0 * 1024.0)
disc2pkgs = excessFiles(distDir + "/RedHat/RPMS", binPkgList, 
			targetSize - disc1used - fudgeFactor - compsSize)
print "first to move to disc2 is", disc2pkgs[0]
moveFiles("%s/RedHat/RPMS" % disc1Dir, 
	  "%s/RedHat/RPMS" % disc2Dir, 
	  disc2pkgs);
disc1Size = spaceUsed(disc1Dir)

print "Creating disc3..."
stamp(disc3Dir, releasestr, "3", arch, startedAt, dirInfo)

for file in jointfiles:
    linkFile(disc1Dir, disc2Dir, file)

disc3pkgs = excessFiles(distDir + "/RedHat/RPMS", binPkgList, 
			targetSize + disc1Size - disc1used - fudgeFactor)

print "first to move to disc3 is", disc3pkgs[0]
moveFiles("%s/RedHat/RPMS" % disc2Dir, 
	  "%s/RedHat/RPMS" % disc3Dir, 
	  disc3pkgs);
disc3used = spaceUsed(disc3Dir)

print "Creating first source disc..."
stamp(disc1SrcDir, releasestr, "4", arch, startedAt, dirInfo)
os.system("cp -al %s %s" % (srcDir, disc1SrcDir))

print "Creating second source disc..."
stamp(disc2SrcDir, releasestr, "5", arch, startedAt, dirInfo)

srcPkgList = os.listdir("%s/SRPMS" % disc1SrcDir)
srcPkgList.sort()
#srcPkgList.reverse()
disc2pkgs = excessFiles(srcDir, srcPkgList,
			targetSize - fudgeFactor)
print "first src pkg to move to 2nd src cd is", disc2pkgs[0]
moveFiles("%s/SRPMS" % disc1SrcDir, 
	  "%s/SRPMS" % disc2SrcDir, 
	  disc2pkgs);
srcDisc1Size = spaceUsed(disc1SrcDir)

print "Dropping remainder of sources on third disc..."
stamp(disc3SrcDir, releasestr, "6", arch, startedAt, dirInfo)
disc3pkgs = excessFiles(srcDir, srcPkgList,
                        srcDisc1Size + targetSize - fudgeFactor)
print "first src pkg to move to 3rd src cd is", disc3pkgs[0]
moveFiles("%s/SRPMS" % disc2SrcDir,
          "%s/SRPMS" % disc3SrcDir,
          disc3pkgs)

#sys.exit(0)

sys.stdout.flush()

os.system("du -sh %s %s %s %s %s %s" % (disc1Dir, disc2Dir, disc3Dir,
                                        disc1SrcDir, disc2SrcDir, disc3SrcDir))