From 27018517a13241ff9639999386d5d51145d3d041 Mon Sep 17 00:00:00 2001 From: Elio Maldonado Date: Sun, 30 Dec 2012 11:03:46 -0800 Subject: First checked in - scripts to build nss and depeendencies using mock - adapted from scripts written dor dogtag and ipa by Kevin Wright --- python-scripts/build-nss.py | 305 +++++++++++++++ python-scripts/build_functions.py | 799 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 1104 insertions(+) create mode 100755 python-scripts/build-nss.py create mode 100755 python-scripts/build_functions.py diff --git a/python-scripts/build-nss.py b/python-scripts/build-nss.py new file mode 100755 index 0000000..70a9533 --- /dev/null +++ b/python-scripts/build-nss.py @@ -0,0 +1,305 @@ +#!/usr/bin/python +import sys +import os +import argparse +import json +import build_functions +print os.environ.get("USER") +####################################################### + +parser = argparse.ArgumentParser(description='Program to automate nss builds using mock.', + version = '0.9 beta', + add_help=True, + conflict_handler='resolve', + epilog='This program is designed to be used both on the command line or by cron') + +#parser = argparse.ArgumentParser(description='Program to automate nss builds using mock.', version = '0.9 beta')#, add_help=True) + + +parser.add_argument("-d", + "--debug", + action="store_true", + default=True, + help="show debug output") + +parser.add_argument("-t", + "--test", + action="store_true", + default=False, + help="test mode (for debugging)") +parser.add_argument( + '-f', + '--flavor', + action="store", + dest="flavor", + default='nss', + help='specify the flavor of the build. [default=nss]') +#parser.add_argument( +# '-p', +# '--platform', +# action="store", +# dest="platform", +# default=None, +# help='specify a platform to use for mock config. [default=None] NOT IMPLEMENTED') +parser.add_argument( + '-r', + '--rel', + action="store", + dest="rel", + default=None, + choices=("6", "7" ), + help='specify a release to use for mock config. [default=None]') +parser.add_argument( + '-a', + '--arch', + action="store", + dest="arch", + default=None, + choices=("i386", "x86_64"), + help='specify an arch to use for mock config. [default=None]') +parser.add_argument( + '-l', + '--local', + action="store_true", + dest="localBuild", + default=False, + help='perform a build in the current working diretory. [default=False]') +parser.add_argument( + '-mm', + '--mailme', + action="store_true", + dest="mailMe", + #default=True, + default=False, + help='send email to kwright@redhat.com. [default=False') +parser.add_argument( + '-sb', + '--skipbuild', + action="store_true", + dest="skipBuild", + default=False, + help='skip the build portion of the program. [default=False] ') +parser.add_argument( + '-sc', + '--skipcopy', + action="store_true", + dest="skipCopy", + default=False, + help='skip the copy portion of the program. [default=False] ') +parser.add_argument( + '-sw', + '--skipwget', + action="store_true", + dest="skipWget", + default=False, + help='skip the wget portion of the program. [default=False] ') +parser.add_argument( + '-b', + '--build', + action='store', + dest='nssPackageList', + nargs='+', + default=['nspr', 'nss-util', 'nss-softokn', 'nss', 'certmonger', 'crypto-utils', 'curl', 'mod_nss', 'mod_revocator' ], + help='specify a list of one of more builds to performm. ', + choices=('nspr', 'nss-util', 'nss-softokn', 'nss')) +parser.add_argument( + '-m', + '--mock_cfg', + action="store", + dest="mockCfgList", + nargs='+', + default=["rhel-6-i386", "rhel-6-x86_64"], + help='spcify a list of one or more mock configs to use. [default = rhel-6-i386", "rhel-6-x86_64]', + choices=("rhel-6-i386", "rhel-6-x86_64")) +options = parser.parse_args() +#if args.servername is None and not args.dry_run: + + +def print_debug(msg): + if options.debug: + print "DEBUG: %s" % (msg) + +#TODO: figure out some logic about which parameters can be used together +####################################################### +workbase = os.environ['HOME'] +scriptsDir = workbase + "/scripts" +pythonDir=scriptsDir + "/build" +sys.path.append(pythonDir) +from build_functions import * +####################################################### +if options.test: + options.debug = True + options.mailMe = True + +if options.skipCopy: + options.skipWget = True + +if options.skipBuild and options.nssPackageList: + print "INFO: --skipBuild and --build cannot be used together" + +print_debug("test = %s " % options.test) +print_debug("debug = %s " % options.debug) +#print_debug("platform = %s " % options.platform) +print_debug("release = %s " % options.rel) +print_debug("arch = %s " % options.arch) +print_debug("flavor = %s " % options.flavor) +print_debug("mailMe = %s " % options.mailMe) +print_debug("localBuild = %s " % options.localBuild) +print_debug("skipBuild = %s " % options.skipBuild) +print_debug("skipCopy = %s " % options.skipCopy) +print_debug("skipWget = %s " % options.skipWget) +print_debug("nssPackageList = %s " % options.nssPackageList) +print_debug("mockCfgList = %s " % options.mockCfgList) +####################################################### +mockCfgList=[] +for mock_cfg in options.mockCfgList: + [platform, rel, arch] = mock_cfg.split("-") + if options.rel is not None: + print_debug("Release = %s" % options.rel) + mockCfgList.append("-".join([platform, options.rel, arch])) + else: + print_debug("Release not defind") + mockCfgList = options.mockCfgList + print_debug("mockCfgList = %s" % mockCfgList) + + +for mock_cfg in options.mockCfgList: + [platform, rel, arch] = mock_cfg.split("-") + if options.arch is not None: + print_debug("arch = %s" % options.arch) + mockCfgList = [("-".join([platform, rel, options.arch]))] + else: + print_debug("arch not defind") + mockCfgList = options.mockCfgList + +print_debug("mockCfgList = %s" % mockCfgList) + +####################################################### +#TODO use MAILTO or mailTo or receiver to pass as a arg +#test to see if a variable is defined +#try: + #thevariable +#except NameError: + #print "well, it WASN'T defined after all!" +#else: + #print "sure, it was defined." +####################################################### +#TODO use MAILTO or mailTo or receiver to pass as a arg +if options.mailMe: + mailTo="emaldona@redhat.com" +else: + #mailTo="rhcs-dev-list@redhat.com" + mailTo="nss-nspr-devel@redhat.com" +####################################################### +if options.localBuild is False: + import datetime + now = datetime.datetime.now() + DT=now.strftime('%Y%m%d-%H%M%S') + str(DT) + buildDir = workbase + "/" + DT + print_debug("DT = %s" % DT) +else: + buildDir = os.getcwd() + DT=os.path.basename(buildDir) + #need the DT for later when copying files. + print_debug("DT = %s" % DT) +print_debug("buildDir = %s" % buildDir) + +nssPackageList = options.nssPackageList +flavor = options.flavor +####################################################### +progname = os.path.basename(__file__) +message = "Subject: INFO: starting a build of from %s\n\nThe build directory is %s " % (progname, buildDir) +#TODO. figure out how to send this email only to me. (ugh!) +#email(message, mailTo) +####################################################### + +print_debug("Verifying the existence of the mock config file") +mockDir = '/etc/mock' +fileNameExt = '.cfg' +for mock_cfg in mockCfgList: + if os.path.isfile(os.path.join(mockDir, mock_cfg + fileNameExt)): + print_debug("%s" % os.path.join(mockDir, mock_cfg + fileNameExt)) + else: + print "ERROR: %s doesn't exist." % os.path.join(mockDir, mock_cfg + fileNameExt) + message = "Subject: ERROR: %s missing from /etc/mock\n\nERROR: %s missing from /etc/mock. Killing the build." % (mock_cfg + fileNameExt, mock_cfg) + email(message, mailTo) + sys.exit() +####################################################### + +buildStatusDict={} +if options.skipBuild is False: + for mock_cfg in mockCfgList: + [platform, rel, arch] = mock_cfg.split("-") + if rel == "18": + nssPackageList.append("pki-tps") + print_debug("Adding pki-tps to list of packages to build for %s %s" % (platform, rel)) + print_debug("nssPackageList = %s" % nssPackageList) + for package in nssPackageList: + #status = build_pki(package, buildDir, mock_cfg, flavor, scriptsDir, options.debug, mailTo) + status = build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options.debug, mailTo) + buildStatusDict[(package,mock_cfg)]=status +#################### +#hope this line works + print_debug("%s" % json.dumps(str(buildStatusDict), sort_keys=True, indent=4)) +#################### + print_debug("package %s completed with status %s for %s" % (package, status, mock_cfg)) + + + +# TODO fix issue with failed build not stopping creating the repo +# NOTE the problem appears to be if the x86_64 bit builds all complete +# successfully, then the dict only shows those builds then it thinks the +# build is okay. also, dogtag uses build_pki, whereas nss +# uses build_generic_pkg -- this may not apply to nss +#print_debug("%s" % buildStatusDict) +print_debug("%s" % json.dumps(str(buildStatusDict), sort_keys=True, indent=4)) +#TODO write the output of buildStatusDict to a file in the home directory. +#This way, I can easily tell if anything failed. + +################################################################################ +#NOTE: handle whether or not to copy the files over and create the rpm repo. +#The logic is as follows: +#if one of the packages failed to build, skipCopy is enalbed, don't copy +#if the commandline option skipCopy is enalbled, don't copy +#else skipCopy is disabled and the files get copied then the yum repos are created. +################################################################################ +skipCopy=False +for mock_cfg in mockCfgList: + for (package, status) in buildStatusDict: + print_debug("%s" % buildStatusDict[package,mock_cfg]) + if buildStatusDict[package,mock_cfg]=="FAILED": + skipCopy=True + print "ERROR: skipCopy is %s (enabled) due to a failure of package %s on %s. The repo will NOT be created)" % (skipCopy, package, mock_cfg) + #TODO Add the code here to copy the build logs over possibly only for the packages the failed? + + if options.skipCopy is True: + message = "Subject: INFO: skipCopy was enabled from the commandline. Skipping copying to and/or creating the repo.\n\nThe build directory is %s." % (buildDir) + print message + email(message, mailTo) + elif skipCopy is True: + status_message="" + for (package, status) in buildStatusDict: + status_message += "build status of package: %s on %s: %s\n" % (package, status, buildStatusDict[package,mock_cfg]) + message = "Subject: ERROR: %s skipping copying to and/or creating the repo.\n\nERROR: The repo was not created due to a failure of one or more packages.\n\nHere's the list of builds and their status:\n\n%s\n" % (flavor, status_message) +#TODO add code to copy the log files over somewhere when the build failes. + createRepoDirs(mock_cfg, flavor, DT, buildDir, options.test, options.debug, mailTo) + + print_debug("%s" % message) + email(message, mailTo) + elif skipCopy is False: + #for mock_cfg in mockCfgList: + print_debug("copy the files to the repo for %s" % mock_cfg) + copyToRepo(mock_cfg, flavor, DT, buildDir, options.test, options.debug, options.skipWget, mailTo) + +sys.exit() + + +if __name__ == '__main__': + import sys + try: + main(sys.argv[1]) + except IndexError: + main() + + diff --git a/python-scripts/build_functions.py b/python-scripts/build_functions.py new file mode 100755 index 0000000..adad8f2 --- /dev/null +++ b/python-scripts/build_functions.py @@ -0,0 +1,799 @@ +#!/usr/bin/python + +"""porting build functions from bash script""" + +import re +import os +import subprocess +import commands +import shutil +import git +import sys +import distutils.dir_util + + +########################################################################################## +def email(message, mailTo): + import smtplib + import socket + mail_server='smtp.corp.redhat.com' + mail_server_port=25 + + receivers=mailTo + + #if options_mailMe: + # receivers = 'kwright@redhat.com' + #else: + # receivers = 'ipa-and-samba-team-automation@redhat.com' + + sender = 'kwright@redhat.com' + + #print socket.gethostname() + hostname = socket.gethostname() + dashes = "+" + 70 * "-" + "+" + header = "" + header = """\ +To: %s +From: %s +%s +%s +""" % (receivers, sender, message, header) + message = header + "\n\n\n" + dashes + "\nINFO: mailed from " + hostname + "\n" + dashes + "\n" + + try: + smtpObj = smtplib.SMTP('smtp.corp.redhat.com', 25) + #smtpObj = smtplib.SMTP('localhost', 25) + smtpObj.sendmail(sender, receivers, message) + print "INFO: Successfully sent email to %s" % receivers + except smtplib.SMTPException: + print "Error: unable to send email" + + +########################################################################################## +def tail2(f, n, offset=None): + """Reads a n lines from f with an offset of offset lines. The return + value is a tuple in the form ``(lines, has_more)`` where `has_more` is + an indicator that is `True` if there are more lines in the file. + """ + avg_line_length = 74 + to_read = n + (offset or 0) + + while 1: + try: + f.seek(-(avg_line_length * to_read), 2) + except IOError: + # woops. apparently file is smaller than what we want + # to step back, go to the beginning instead + f.seek(0) + pos = f.tell() + lines = f.read().splitlines() + #lines = f.read() + if len(lines) >= to_read or pos == 0: + return lines[-to_read:offset and -offset or None], len(lines) > to_read or pos > 0 + avg_line_length *= 1.3 + +########################################################################################## +def tail( f, window=20 ): + #BUFSIZ = 1024 + BUFSIZ = 2048 + f.seek(0, 2) + bytes = f.tell() + size = window + block = -1 + data = [] + while size > 0 and bytes > 0: + if (bytes - BUFSIZ > 0): + # Seek back one whole BUFSIZ + f.seek(block*BUFSIZ, 2) + # read BUFFER + data.append(f.read(BUFSIZ)) + else: + # file too small, start from begining + f.seek(0,0) + # only read what was not read + # kwright commented out the next line trying to fix a bug + #data.append(f.read(bytes)) + linesFound = data[-1].count('\n') + size -= linesFound + bytes -= BUFSIZ + block -= 1 + return '\n'.join(''.join(data).splitlines()[-window:]) + + +########################################################################################## +def runProcess(exe): + #print "runProcess" + p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + while(True): + retcode = p.poll() #returns None while subprocess is running + line = p.stdout.readline() + yield line + if(retcode is not None): + break + +########################################################################################## +def run(command): + #if options.verbose: + # print command + rv, output = commands.getstatusoutput(command) + if rv: + #raise RuntimeError, "error running %s, %s" % (command, rv) + pass + return output + +########################################################################################## +def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mailTo): + dashes = "+" + 70 * "-" + "+" + pounds = 70 * "#" + + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + + print_debug("package %s" % package) + print_debug("mock_cfg %s" % mock_cfg) + print_debug("group %s" % group) + if len(gitrev) > 0: + gitrev = "git" + gitrev + print_debug("gitrev %s" % gitrev) + print_debug("buildDir %s" % buildDir) + #if os.path.isdir(os.path.join(buildDir, package, ".git")): + #if os.path.isdir(resultsDir): + buildLogs = os.path.join(buildDir,"build_logs", mock_cfg) + print_debug("buildLogs: %s" % buildLogs) + if not os.path.exists(buildLogs): + os.makedirs(buildLogs) + os.chdir(buildLogs) + + print_debug("cwd %s" % os.getcwd()) + print pounds + print "INFO: build_package() -> building %s for %s" % (package, mock_cfg) + print pounds + + + SRPMS = buildDir + "/packages/SRPMS/" + if not os.path.exists(SRPMS): + os.makedirs(SRPMS) + os.chdir(SRPMS) + print_debug("cwd %s" % os.getcwd()) + srpmsdir = os.listdir(SRPMS) + srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package + print_debug("srpmsdir %s" % srpmsdir) + print_debug("package = %s" % package) + print_debug("SRPMS = %s" % SRPMS) + print_debug("srpmPattern = %s" % srpmPattern) + SRPM=None + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print_debug("fileName %s" % fileName) + SRPM = fileName + + if SRPM == None: + message = "Subject: %s ERROR: unable to build %s .\n\nThe src.rpm file for the package %s is not available. " % (group, package, package) + email(message, mailTo) + failure_message="ERROR: The src.rpm file for %s is not available!" % package + print_debug(failure_message) + print "INFO: %s " % dashes + print os.listdir(SRPMS) + print "INFO: %s " % dashes + #message = "Subject: %s The package %s build for %s failed\n\n%s" % (group, package, mock_cfg, failure_message ) + #email(message, mailTo) + status="FAILED" + return(status) + + else: + import datetime + now = datetime.datetime.now() + YEAR = now.strftime("%Y") + str(YEAR) + workbase = os.environ['HOME'] + scriptsDir = workbase + "/scripts" + + + if not re.search(YEAR, SRPM): + print "INFO: we need to redo this src.rpm file." + os.chdir(SRPMS) + print_debug("SRPM = %s, gitrev = %s" % (SRPM, gitrev)) + os.listdir(SRPMS) + script = "/srpm_timestamp.sh" + scriptPath = scriptsDir + script + + #for line in runProcess(['/home/kwright/scripts/srpm_timestamp.sh', SRPM, gitrev]): + for line in runProcess([scriptPath, SRPM, gitrev]): + print line, + os.remove(SRPM) + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print_debug("fileName %s" % fileName) + SRPM = fileName + else: + print "INFO: This srpm has already been converted.\n" + + #resultsDir= '/var/lib/mock/' + mock_cfg + '/result' + resultsDir= os.path.join('/var/lib/mock', mock_cfg, 'result') + print "%s\nINFO: cleaning up anything in %s\n%s\n" % (dashes, resultsDir, dashes) + if os.path.isdir(resultsDir): + fileList = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ] + print_debug("\nfileList %s" % fileList) + os.chdir(resultsDir) + for f in fileList: + print_debug("deleting %s" % f) + os.remove(f) + + os.chdir(SRPMS) + print_debug("cwd %s" % os.getcwd()) + print pounds + print "INFO: doing the mock build now of %s on %s" % (SRPM, mock_cfg) + print pounds + #logFilePath = buildLogs + "/build_" + package + ".log" + #buildLogs = os.path.join(buildDir,"build_logs", mock_cfg) + logFilePath = os.path.join(buildLogs, package + ".log") + logFile = open(logFilePath, "w") + for line in runProcess(['mock', '-v', '-r', mock_cfg, '--rebuild', SRPM]): + logFile.write(line,) + print line, + logFile.write("\n") + logFile.close() + + RPM=None + fileName=None + rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg) + RPMList = [] + builtRPMS="" + srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package + if not os.path.exists(rpmsDir): + os.makedirs(rpmsDir) + print "INFO: Creating dir %s" % rpmsDir + filelist = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ] + print "\n\n" + print_debug("filelist = %s" % filelist) + for RPM in filelist: + builtRPMS += RPM + "\n" + pathname = os.path.join(resultsDir, RPM) + if options_debug == True: + print_debug("copying:%s to %s" % (pathname, rpmsDir)) + shutil.copy2(pathname, rpmsDir) + + mockBuildLogs = os.path.join(buildDir,"build_logs", mock_cfg, package) + if not os.path.exists(mockBuildLogs): + print "INFO: Creating dir %s" % mockBuildLogs + os.makedirs(mockBuildLogs) + for basename in os.listdir(resultsDir): + if basename.endswith('.log'): + pathname = os.path.join(resultsDir, basename) + print_debug("Copying log file %s to %s" % (pathname, mockBuildLogs)) + if os.path.isfile(pathname): + shutil.copy2(pathname, mockBuildLogs) + + if builtRPMS != "": + print_debug("builtRPMS: %s" % builtRPMS.strip()) + message = "Subject: %s The package %s build for %s completed successfully\n\nINFO: Here are the rpms built:\n\n%s" % (group, package, mock_cfg, builtRPMS) + status="COMPLETED" + print "INFO: status = ", status + email(message, mailTo) + return(status) + else: + print pounds + print "RPM FILE NOT FOUND !" +#TODO move the generated build logs to this commented out line below + #not sure why I was trying to access the mock log file here. Since + #it's not currently being used, I'm commenting it out + #mockLogFilePath= os.path.join(buildLogs, package, "build.log") + + logTail = "" + print pounds + logTail=subprocess.Popen([r"tail","-n 40", logFilePath], stdout=subprocess.PIPE).communicate()[0] + print logTail + print pounds + + message = "Subject: %s The package %s build for %s failed\n\nINFO: Here are the last lines from the log file:\n%s\n%s\n" % (group, package, mock_cfg, dashes, logTail ) + email(message, mailTo) + status="FAILED" + print "INFO: status = ", status + return(status) +#end build_package() + +########################################################################################## +def build_pki(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo): + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + print "INFO: Building package %s in %s for %s: %s" % (package, buildDir, mock_cfg, flavor) + if not os.path.exists(buildDir): + os.makedirs(buildDir) + os.chdir(buildDir) + pkiScriptsDir = buildDir + "/pki/scripts" + if not os.path.exists(pkiScriptsDir): + print "INFO: Getting the sources for %s" % package + git.Git().clone("git://git.fedorahosted.org/git/pki.git") + #else: + pkiGitDir = buildDir + "/pki" + os.chdir(pkiGitDir) + g = git.Git(pkiGitDir) + print g.log('--pretty=%h','-1','--').split('\n') + gitrev = g.log('--pretty=%h','-1','--').split('\n')[0] + print_debug(gitrev) + print_debug(type(gitrev)) + #[gitrev, stuff1, stuff2] = g.log('--pretty=%h','-1','--').split('\n') + #gitrev2=gitrev[0] + #print type(gitrev2) + #gitrev=gitrev2 + #sys.exit() + print "INFO: git rev = %s" % gitrev + + srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package + SRPMS = buildDir + "/packages/SRPMS/" + if not os.path.exists(SRPMS): + os.makedirs(SRPMS) + os.chdir(SRPMS) + print "INFO: List of src.rpms:\n" + os.listdir(SRPMS) + SRPM=None + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print fileName + SRPM = fileName + if SRPM == None: + if package == "dogtag-pki": + srpmScript = scriptsDir + "/dogtag/build_" + package + "_srpm.sh" + print_debug("srpmScript: %s" % srpmScript) + shutil.copy(srpmScript, buildDir) + #cd ${BuildDir} && bash -x ./build_${PKG}_srpm.sh 2>&1 |tee ${BuildDir}/build_$PKG.srpm.log +#TODO add logging to a file + os.chdir(buildDir) + for line in runProcess([srpmScript]): + print line, + else: + print "INFO: package name to convert = %s" % package + under = re.compile('-') + srpmScript = under.sub('_', package) + #print srpmScript #TODO add the -x for DEBUG + #cd ${BuildDir}/pki/scripts/ && bash -x ${BuildDir}/pki/scripts/compose_${PKG_SCRIPT}_packages hybrid_srpm + os.chdir(pkiScriptsDir) + srpmScript = pkiScriptsDir + "/compose_" + srpmScript + "_packages" + print srpmScript + for line in runProcess([srpmScript, 'hybrid_srpm']): + print line, + status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo) + print "Build status = %s for %s " % (status, package) + return status +#end build_pki() + +########################################################################################## +def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo): + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + print "package = %s " % package + #if not os.path.isdir(os.path.join(buildDir,package)): + # print "INFO: Creating directory %s " % os.path.join(buildDir,package) + # os.mkdir(os.path.join(buildDir,package)) + print "+-----------------------------------------------------+" + print "Starting to build %s" % package + print "+-----------------------------------------------------+" + srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package + #SRPMS = buildDir + "/packages/SRPMS/" + SRPMS = os.path.join(buildDir,"packages/SRPMS") + print "SRPMS = %s" % SRPMS + if not os.path.exists(SRPMS): + os.makedirs(SRPMS) + os.chdir(SRPMS) + print "INFO: List of src.rpms:" + os.listdir(SRPMS) + SRPM=None + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print fileName + SRPM = fileName + if SRPM == None: + srpmScript = scriptsDir + "/build_" + package + ".sh" + print_debug("srpmScript = %s" % srpmScript) + shutil.copy(srpmScript, buildDir) + os.chdir(buildDir) + print os.getcwd() + srpmScript = buildDir + "/build_" + package + ".sh" + print "srpmScript = %s" % srpmScript + for line in runProcess([srpmScript]): + print line, + + #check for exitence of .git directory + if os.path.isdir(os.path.join(buildDir, package, ".git")): + print ".git dir found %s" % os.path.join(buildDir, package, ".git") + os.chdir(os.path.join(buildDir, package)) + print os.path.join(buildDir, package) + g = git.Git(os.path.join(buildDir,package)) + gitrev = g.log('--pretty=%h','-1','--').split('\n') + gitrev2=gitrev[0] + gitrev=gitrev2 + print "INFO: git rev = %s" % gitrev + else: + print ".git dir not found %s" % os.path.join(buildDir,package, ".git") + gitrev ="" + #NOTE: the ipa package already has the gitrev embedded in the SRPM and doesn't need to be converted + + status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo) + print "Build status = %s for %s " % (status, package) + return status +#end build_generic_pkg() + + + +#TODO some of these 3 todo's are duplicates of those down at the symlink area +#TODO add checking to see if the source checked out okay and that the pki/scripts dir exists +#TODO put some code in here to verify that all the packages build succeded. if not, then we shouldn't even create the directories + +########################################################################################## +def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, mailTo): + """ function for the repo directories + Currently this is called only when the build fails. + + """ + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + print_debug("function: createRepoDir") + print_debug("mock_cfg: %s" % mock_cfg) + print_debug("flavor: %s" % flavor) + print_debug("DT: %s" % DT) + print_debug("buildDir: %s" % buildDir) + print_debug("options_test: %s" % options_test) + + workbase = os.environ['HOME'] + scriptsDir = workbase + "/scripts" + + dashes = "+" + 70 * "-" + "+" + longDashes = "+" + 80 * "-" + "+" + URL="http://mickey.dsdev.sjc.redhat.com" + print_debug("URL: %s" % URL) +#TODO find a better name than repoPackage + repoPackage="pki" + print_debug("repoPackage: %s" % repoPackage) + + [platform, rel, arch] = mock_cfg.split("-") + + if arch == "i386": + arch = "i686" + print_debug("arch: %s" % arch) + + if platform == "fedora": + dist="F" + rel + print_debug("rel: %s" % rel) + + if platform == "rhel": + dist=platform + rel + print_debug("dist: %s" % dist) + +#TODO send an email the the build faild due to not being able to mount repos + if options_test: + repos = "/tmp/repos" + else: + repos = "/repos" + if not os.path.ismount(repos): + print "WARN: %s is not mounted" % repos + print "INFO: attempting to mount %s ... " % repos + for line in runProcess(["sudo", "/bin/mount", repos]): + print line + if not os.path.ismount(repos): + print "ERROR: unable to mount %s " % repos + sys.exit() + + #TODO: handle other flavor names for future builds + if flavor == "DOGTAG 10": + #/repos/pki/dogtag/10 + [flavorName, flavorGroup] = flavor.split() + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + else: + flavorName="IPA" + flavorGroup="devel" + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + print_debug("repoBase: %s" % repoBase) + print_debug("repoFull: %s" % repoFull) + + + #check for case when a new dist is released and create the dir + ################################################################# + #create the repo tree + ################################################################# + if os.path.exists(repoBase) and not os.path.exists(repoFull): + print "INFO: Creating dir", repoFull + os.makedirs(repoFull) + elif not os.path.exists(repoBase): + os.makedirs(repoBase) + #TODO add some logic here to exit if unable to create the directory + #print "ERROR: %s doesn't exist" % repoBase + #sys.exit(2) + if not os.path.exists(os.path.join(repoFull, DT)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT) + os.makedirs(os.path.join(repoFull, DT)) + for dirType in (arch, "noarch", "src"): + if not os.path.exists(os.path.join(repoFull, DT, dirType)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType) + os.makedirs(os.path.join(repoFull, DT, dirType)) + if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs") + os.makedirs(os.path.join(repoFull, DT, arch, "logs")) + +#def copyBuildLogs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo): + """still need a function for over the build logs only. in the case that one of the builds fails""" + sourceLogDir = os.path.join(buildDir,"build_logs", mock_cfg) + #repoBuildLogsDir = os.path.join(repoFull, DT, arch, "logs") + targetLogDir = os.path.join(repoFull, DT, arch, "logs", "build_logs", mock_cfg) + #destDir = os.path.join(repoFull, DT, rpmType) + #print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes)) + #shutil.copy2(os.path.join(rpmsDir, RPM), destDir) + copyResult = distutils.dir_util.copy_tree(sourceLogDir, targetLogDir, preserve_mode=0, verbose=1) + print_debug("copyResult %s" % copyResult) + #print buildLogs + URL="http://mickey.dsdev.sjc.redhat.com" + #import json + test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".") + #message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch") + #print_debug("path =%s" % message_url_path) + + failMessage= """ +You can find the build logs here: +%s + %s%s + %s%s +%s +%s +""" % ( + longDashes, + URL, os.path.join(URL, repoFull, DT, arch), + URL, os.path.join(URL, repoFull, DT, "noarch"), + longDashes, + test_message, + ) + buildStatus="FAILED" + message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, failMessage ) + print(failMessage) + email(message, mailTo) +#end createRepoDirs +########################################################################################## + +def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo): + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + + print_debug("mock_cfg: %s" % mock_cfg) + print_debug("flavor: %s" % flavor) + print_debug("DT: %s" % DT) + print_debug("buildDir: %s" % buildDir) + print_debug("options_test: %s" % options_test) + + workbase = os.environ['HOME'] + scriptsDir = workbase + "/scripts" + + dashes = "+" + 70 * "-" + "+" + longDashes = "+" + 80 * "-" + "+" + URL="http://mickey.dsdev.sjc.redhat.com" + print_debug("URL: %s" % URL) +#TODO find a better name than repoPackage + repoPackage="pki" + print_debug("repoPackage: %s" % repoPackage) + + [platform, rel, arch] = mock_cfg.split("-") + + if arch == "i386": + arch = "i686" + print_debug("arch: %s" % arch) + + if platform == "fedora": + dist="F" + rel + print_debug("rel: %s" % rel) + + if platform == "rhel": + dist=platform + rel + print_debug("dist: %s" % dist) + +#TODO send an email the the build faild due to not being able to mount repos + if options_test: + repos = "/tmp/repos" + else: + repos = "/repos" + if not os.path.ismount(repos): + print "WARN: %s is not mounted" % repos + print "INFO: attempting to mount %s ... " % repos + for line in runProcess(["sudo", "/bin/mount", repos]): + print line + if not os.path.ismount(repos): + print "ERROR: unable to mount %s " % repos + sys.exit() + + #TODO: handle other flavor names for future builds + if flavor == "DOGTAG 10": + [flavorName, flavorGroup] = flavor.split() + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + else: + flavorName="IPA" + flavorGroup="devel" + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + print_debug("repoBase: %s" % repoBase) + print_debug("repoFull: %s" % repoFull) + if not options_skipWget: + #COPY over the download_builds.sh script + downloadScript=os.path.join(scriptsDir, flavorName, "download_builds.sh") + print_debug("downloadScript = %s" % downloadScript) + shutil.copy(downloadScript, buildDir) + os.chdir(buildDir) + downloadScript=os.path.join(buildDir, "download_builds.sh") + print_debug("downloadScript = %s" % downloadScript) + for line in runProcess([downloadScript, arch]): + print line, + + + #check for case when a new dist is released and create the dir + ################################################################# + #create the repo tree + ################################################################# + if os.path.exists(repoBase) and not os.path.exists(repoFull): + print "INFO: Creating dir", repoFull + os.makedirs(repoFull) + elif not os.path.exists(repoBase): + os.makedirs(repoBase) + #print "ERROR: %s doesn't exist" % repoBase + #sys.exit(2) +#TODO add some logic here to exit if unable to create the directory + if not os.path.exists(os.path.join(repoFull, DT)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT) + os.makedirs(os.path.join(repoFull, DT)) + for dirType in (arch, "noarch", "src"): + if not os.path.exists(os.path.join(repoFull, DT, dirType)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType) + os.makedirs(os.path.join(repoFull, DT, dirType)) + if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs") + os.makedirs(os.path.join(repoFull, DT, arch, "logs")) + buildLogs = os.path.join(repoFull, DT, arch, "logs") + print buildLogs + + os.chdir(repoFull) +#TODO add some logic that if any of the builds fail, +#Currently, if the build fails, I skip the entire creation of the repos--including the build logs +# 1) do not create the symlink - DONE +# 2) copy the build logs over - DONE +# 3) do not copy the builds -DONE +# 4) send an email with a url to the build logs - DONE (in other function) + + archSymlink = "devel_" + arch + + if os.path.exists("devel_" + arch): + os.remove("devel_" + arch) + print os.getcwd() + print "creating symlink %s_devel to %s " % (arch, DT) + os.symlink(DT, "devel_" + arch) + + buildStatus="COMPLETED" + + ################################################################# + #copy the builds over to the newly created repo tree + ################################################################# +#TODO add some logic for the case where the only build is a noarch and there are no arch build (dogtag-pki) +#currently, this creates an empty log file for fileList_i686 +# /tmp/repos/pki/dogtag/10/F17/20120913-150042/i686/logs/fileList_i686 + + rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg) + print dashes + + for rpmType in (arch, "noarch", "src"): + filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ] + filelist.sort() + logFile = open(buildLogs + "/fileList_" + rpmType , "w") + destDir = os.path.join(repoFull, DT, rpmType) + for RPM in filelist: + print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes)) + shutil.copy2(os.path.join(rpmsDir, RPM), destDir) + logFile.write(RPM+"\n") + logFile.close() +#handle i386 rpm files that are part of the i686 repo + rpmType="i386" + filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ] + filelist.sort() + logFile = open(buildLogs + "/fileList_" + "i686" , "a") + destDir = os.path.join(repoFull, DT, "i686") + for RPM in filelist: + print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes)) + shutil.copy2(os.path.join(rpmsDir, RPM), destDir) + logFile.write(RPM+"\n") + logFile.close() + + #sort the file into a temp file then copy it back. this is needed for the email with the list of files + logFile = os.path.join(buildLogs, "fileList_" + "i686") + logFileTmp = os.path.join(buildLogs, "fileList_" + "i686.out") + inLogFile = open(logFile , "r") + outLogFile = open(logFileTmp , "w") + map(outLogFile.write, sorted(inLogFile.readlines())) + inLogFile.close() + outLogFile.close() + shutil.copy2(logFileTmp, logFile) + os.remove(logFileTmp) + + + + +#when I need to create a repo: +#add logic to not do this if build failed + for rpmType in (arch, "noarch", "src"): + destDir = os.path.join(repoFull, DT, rpmType) + logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "w") + print_debug("logFile = %s" % logFile) + cmd = "createrepo -vv --no-database --skip-stat " + destDir + output=run(cmd) + print output + logFile.write(output+"\n") + logFile.close() + numPkgDict={} + for rpmType in (arch, "noarch", "src"): + destDir = os.path.join(repoFull, DT, rpmType) + logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "r") + for line in logFile: + if "Spawning" in line: + #print line + pkgNum=line.split()[-2] + print_debug("pkgNum: %s" % pkgNum) + numPkgDict[rpmType]=pkgNum + + print_debug("numPkgDict: %s" % numPkgDict) + + +#TODO copy the log files over to the build_logs directory + rpmInRepoDict={} + for rpmType in (arch, "noarch", "src"): + rpmInRepoDict[rpmType] = open(buildLogs + "/fileList_" + rpmType, "r").read().split() + #" ".join(REPOLOG) + #print REPOLOG + #close(buildLogs + "/fileList_" + rpmType) + + import json + + #TODO add some logic here to check for test mode and alter the email accordingly + #NOTE this is partially implemented below: + test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".") + message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch") + print_debug("path =%s" % message_url_path) + + releaseMessage= """ +You can find the build repos here: +%s + %s%s + %s%s + + %s%s + %s%s + +Summary of the number of RPMS that were built: +%s +%s + +Here are the RPMs that were built: +%s +%s +%s + """ % ( + longDashes, + URL, os.path.join(URL, repoFull, DT, arch), + URL, os.path.join(URL, repoFull, DT, "noarch"), + URL, os.path.join(URL, repoFull, "devel_" + arch, arch ), + URL, os.path.join(URL, repoFull, "devel_" + arch, "noarch"), + longDashes, + json.dumps(numPkgDict, indent=4), + longDashes, + json.dumps(rpmInRepoDict, indent=4), + test_message, + ) + print releaseMessage + + message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, releaseMessage ) + + email(message, mailTo) + +#end copyToRepo -- cgit