summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorElio Maldonado <emaldona@localhost.localdomain>2012-12-30 11:03:46 -0800
committerElio Maldonado <emaldona@localhost.localdomain>2012-12-30 11:03:46 -0800
commit27018517a13241ff9639999386d5d51145d3d041 (patch)
tree3840aa081d54d3a472c790b6ce72a0e3c4b31719
parentd4919e8833b7223f97e6716b846d0ef4653894b5 (diff)
downloadnssmockbuilds4fedora-27018517a13241ff9639999386d5d51145d3d041.tar.gz
nssmockbuilds4fedora-27018517a13241ff9639999386d5d51145d3d041.tar.xz
nssmockbuilds4fedora-27018517a13241ff9639999386d5d51145d3d041.zip
First checked in
- scripts to build nss and depeendencies using mock - adapted from scripts written dor dogtag and ipa by Kevin Wright
-rwxr-xr-xpython-scripts/build-nss.py305
-rwxr-xr-xpython-scripts/build_functions.py799
2 files changed, 1104 insertions, 0 deletions
diff --git a/python-scripts/build-nss.py b/python-scripts/build-nss.py
new file mode 100755
index 0000000..70a9533
--- /dev/null
+++ b/python-scripts/build-nss.py
@@ -0,0 +1,305 @@
+#!/usr/bin/python
+import sys
+import os
+import argparse
+import json
+import build_functions
+print os.environ.get("USER")
+#######################################################
+
+parser = argparse.ArgumentParser(description='Program to automate nss builds using mock.',
+ version = '0.9 beta',
+ add_help=True,
+ conflict_handler='resolve',
+ epilog='This program is designed to be used both on the command line or by cron')
+
+#parser = argparse.ArgumentParser(description='Program to automate nss builds using mock.', version = '0.9 beta')#, add_help=True)
+
+
+parser.add_argument("-d",
+ "--debug",
+ action="store_true",
+ default=True,
+ help="show debug output")
+
+parser.add_argument("-t",
+ "--test",
+ action="store_true",
+ default=False,
+ help="test mode (for debugging)")
+parser.add_argument(
+ '-f',
+ '--flavor',
+ action="store",
+ dest="flavor",
+ default='nss',
+ help='specify the flavor of the build. [default=nss]')
+#parser.add_argument(
+# '-p',
+# '--platform',
+# action="store",
+# dest="platform",
+# default=None,
+# help='specify a platform to use for mock config. [default=None] NOT IMPLEMENTED')
+parser.add_argument(
+ '-r',
+ '--rel',
+ action="store",
+ dest="rel",
+ default=None,
+ choices=("6", "7" ),
+ help='specify a release to use for mock config. [default=None]')
+parser.add_argument(
+ '-a',
+ '--arch',
+ action="store",
+ dest="arch",
+ default=None,
+ choices=("i386", "x86_64"),
+ help='specify an arch to use for mock config. [default=None]')
+parser.add_argument(
+ '-l',
+ '--local',
+ action="store_true",
+ dest="localBuild",
+ default=False,
+ help='perform a build in the current working diretory. [default=False]')
+parser.add_argument(
+ '-mm',
+ '--mailme',
+ action="store_true",
+ dest="mailMe",
+ #default=True,
+ default=False,
+ help='send email to kwright@redhat.com. [default=False')
+parser.add_argument(
+ '-sb',
+ '--skipbuild',
+ action="store_true",
+ dest="skipBuild",
+ default=False,
+ help='skip the build portion of the program. [default=False] ')
+parser.add_argument(
+ '-sc',
+ '--skipcopy',
+ action="store_true",
+ dest="skipCopy",
+ default=False,
+ help='skip the copy portion of the program. [default=False] ')
+parser.add_argument(
+ '-sw',
+ '--skipwget',
+ action="store_true",
+ dest="skipWget",
+ default=False,
+ help='skip the wget portion of the program. [default=False] ')
+parser.add_argument(
+ '-b',
+ '--build',
+ action='store',
+ dest='nssPackageList',
+ nargs='+',
+ default=['nspr', 'nss-util', 'nss-softokn', 'nss', 'certmonger', 'crypto-utils', 'curl', 'mod_nss', 'mod_revocator' ],
+ help='specify a list of one of more builds to performm. ',
+ choices=('nspr', 'nss-util', 'nss-softokn', 'nss'))
+parser.add_argument(
+ '-m',
+ '--mock_cfg',
+ action="store",
+ dest="mockCfgList",
+ nargs='+',
+ default=["rhel-6-i386", "rhel-6-x86_64"],
+ help='spcify a list of one or more mock configs to use. [default = rhel-6-i386", "rhel-6-x86_64]',
+ choices=("rhel-6-i386", "rhel-6-x86_64"))
+options = parser.parse_args()
+#if args.servername is None and not args.dry_run:
+
+
+def print_debug(msg):
+ if options.debug:
+ print "DEBUG: %s" % (msg)
+
+#TODO: figure out some logic about which parameters can be used together
+#######################################################
+workbase = os.environ['HOME']
+scriptsDir = workbase + "/scripts"
+pythonDir=scriptsDir + "/build"
+sys.path.append(pythonDir)
+from build_functions import *
+#######################################################
+if options.test:
+ options.debug = True
+ options.mailMe = True
+
+if options.skipCopy:
+ options.skipWget = True
+
+if options.skipBuild and options.nssPackageList:
+ print "INFO: --skipBuild and --build cannot be used together"
+
+print_debug("test = %s " % options.test)
+print_debug("debug = %s " % options.debug)
+#print_debug("platform = %s " % options.platform)
+print_debug("release = %s " % options.rel)
+print_debug("arch = %s " % options.arch)
+print_debug("flavor = %s " % options.flavor)
+print_debug("mailMe = %s " % options.mailMe)
+print_debug("localBuild = %s " % options.localBuild)
+print_debug("skipBuild = %s " % options.skipBuild)
+print_debug("skipCopy = %s " % options.skipCopy)
+print_debug("skipWget = %s " % options.skipWget)
+print_debug("nssPackageList = %s " % options.nssPackageList)
+print_debug("mockCfgList = %s " % options.mockCfgList)
+#######################################################
+mockCfgList=[]
+for mock_cfg in options.mockCfgList:
+ [platform, rel, arch] = mock_cfg.split("-")
+ if options.rel is not None:
+ print_debug("Release = %s" % options.rel)
+ mockCfgList.append("-".join([platform, options.rel, arch]))
+ else:
+ print_debug("Release not defind")
+ mockCfgList = options.mockCfgList
+ print_debug("mockCfgList = %s" % mockCfgList)
+
+
+for mock_cfg in options.mockCfgList:
+ [platform, rel, arch] = mock_cfg.split("-")
+ if options.arch is not None:
+ print_debug("arch = %s" % options.arch)
+ mockCfgList = [("-".join([platform, rel, options.arch]))]
+ else:
+ print_debug("arch not defind")
+ mockCfgList = options.mockCfgList
+
+print_debug("mockCfgList = %s" % mockCfgList)
+
+#######################################################
+#TODO use MAILTO or mailTo or receiver to pass as a arg
+#test to see if a variable is defined
+#try:
+ #thevariable
+#except NameError:
+ #print "well, it WASN'T defined after all!"
+#else:
+ #print "sure, it was defined."
+#######################################################
+#TODO use MAILTO or mailTo or receiver to pass as a arg
+if options.mailMe:
+ mailTo="emaldona@redhat.com"
+else:
+ #mailTo="rhcs-dev-list@redhat.com"
+ mailTo="nss-nspr-devel@redhat.com"
+#######################################################
+if options.localBuild is False:
+ import datetime
+ now = datetime.datetime.now()
+ DT=now.strftime('%Y%m%d-%H%M%S')
+ str(DT)
+ buildDir = workbase + "/" + DT
+ print_debug("DT = %s" % DT)
+else:
+ buildDir = os.getcwd()
+ DT=os.path.basename(buildDir)
+ #need the DT for later when copying files.
+ print_debug("DT = %s" % DT)
+print_debug("buildDir = %s" % buildDir)
+
+nssPackageList = options.nssPackageList
+flavor = options.flavor
+#######################################################
+progname = os.path.basename(__file__)
+message = "Subject: INFO: starting a build of from %s\n\nThe build directory is %s " % (progname, buildDir)
+#TODO. figure out how to send this email only to me. (ugh!)
+#email(message, mailTo)
+#######################################################
+
+print_debug("Verifying the existence of the mock config file")
+mockDir = '/etc/mock'
+fileNameExt = '.cfg'
+for mock_cfg in mockCfgList:
+ if os.path.isfile(os.path.join(mockDir, mock_cfg + fileNameExt)):
+ print_debug("%s" % os.path.join(mockDir, mock_cfg + fileNameExt))
+ else:
+ print "ERROR: %s doesn't exist." % os.path.join(mockDir, mock_cfg + fileNameExt)
+ message = "Subject: ERROR: %s missing from /etc/mock\n\nERROR: %s missing from /etc/mock. Killing the build." % (mock_cfg + fileNameExt, mock_cfg)
+ email(message, mailTo)
+ sys.exit()
+#######################################################
+
+buildStatusDict={}
+if options.skipBuild is False:
+ for mock_cfg in mockCfgList:
+ [platform, rel, arch] = mock_cfg.split("-")
+ if rel == "18":
+ nssPackageList.append("pki-tps")
+ print_debug("Adding pki-tps to list of packages to build for %s %s" % (platform, rel))
+ print_debug("nssPackageList = %s" % nssPackageList)
+ for package in nssPackageList:
+ #status = build_pki(package, buildDir, mock_cfg, flavor, scriptsDir, options.debug, mailTo)
+ status = build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options.debug, mailTo)
+ buildStatusDict[(package,mock_cfg)]=status
+####################
+#hope this line works
+ print_debug("%s" % json.dumps(str(buildStatusDict), sort_keys=True, indent=4))
+####################
+ print_debug("package %s completed with status %s for %s" % (package, status, mock_cfg))
+
+
+
+# TODO fix issue with failed build not stopping creating the repo
+# NOTE the problem appears to be if the x86_64 bit builds all complete
+# successfully, then the dict only shows those builds then it thinks the
+# build is okay. also, dogtag uses build_pki, whereas nss
+# uses build_generic_pkg -- this may not apply to nss
+#print_debug("%s" % buildStatusDict)
+print_debug("%s" % json.dumps(str(buildStatusDict), sort_keys=True, indent=4))
+#TODO write the output of buildStatusDict to a file in the home directory.
+#This way, I can easily tell if anything failed.
+
+################################################################################
+#NOTE: handle whether or not to copy the files over and create the rpm repo.
+#The logic is as follows:
+#if one of the packages failed to build, skipCopy is enalbed, don't copy
+#if the commandline option skipCopy is enalbled, don't copy
+#else skipCopy is disabled and the files get copied then the yum repos are created.
+################################################################################
+skipCopy=False
+for mock_cfg in mockCfgList:
+ for (package, status) in buildStatusDict:
+ print_debug("%s" % buildStatusDict[package,mock_cfg])
+ if buildStatusDict[package,mock_cfg]=="FAILED":
+ skipCopy=True
+ print "ERROR: skipCopy is %s (enabled) due to a failure of package %s on %s. The repo will NOT be created)" % (skipCopy, package, mock_cfg)
+ #TODO Add the code here to copy the build logs over possibly only for the packages the failed?
+
+ if options.skipCopy is True:
+ message = "Subject: INFO: skipCopy was enabled from the commandline. Skipping copying to and/or creating the repo.\n\nThe build directory is %s." % (buildDir)
+ print message
+ email(message, mailTo)
+ elif skipCopy is True:
+ status_message=""
+ for (package, status) in buildStatusDict:
+ status_message += "build status of package: %s on %s: %s\n" % (package, status, buildStatusDict[package,mock_cfg])
+ message = "Subject: ERROR: %s skipping copying to and/or creating the repo.\n\nERROR: The repo was not created due to a failure of one or more packages.\n\nHere's the list of builds and their status:\n\n%s\n" % (flavor, status_message)
+#TODO add code to copy the log files over somewhere when the build failes.
+ createRepoDirs(mock_cfg, flavor, DT, buildDir, options.test, options.debug, mailTo)
+
+ print_debug("%s" % message)
+ email(message, mailTo)
+ elif skipCopy is False:
+ #for mock_cfg in mockCfgList:
+ print_debug("copy the files to the repo for %s" % mock_cfg)
+ copyToRepo(mock_cfg, flavor, DT, buildDir, options.test, options.debug, options.skipWget, mailTo)
+
+sys.exit()
+
+
+if __name__ == '__main__':
+ import sys
+ try:
+ main(sys.argv[1])
+ except IndexError:
+ main()
+
+
diff --git a/python-scripts/build_functions.py b/python-scripts/build_functions.py
new file mode 100755
index 0000000..adad8f2
--- /dev/null
+++ b/python-scripts/build_functions.py
@@ -0,0 +1,799 @@
+#!/usr/bin/python
+
+"""porting build functions from bash script"""
+
+import re
+import os
+import subprocess
+import commands
+import shutil
+import git
+import sys
+import distutils.dir_util
+
+
+##########################################################################################
+def email(message, mailTo):
+ import smtplib
+ import socket
+ mail_server='smtp.corp.redhat.com'
+ mail_server_port=25
+
+ receivers=mailTo
+
+ #if options_mailMe:
+ # receivers = 'kwright@redhat.com'
+ #else:
+ # receivers = 'ipa-and-samba-team-automation@redhat.com'
+
+ sender = 'kwright@redhat.com'
+
+ #print socket.gethostname()
+ hostname = socket.gethostname()
+ dashes = "+" + 70 * "-" + "+"
+ header = ""
+ header = """\
+To: %s
+From: %s
+%s
+%s
+""" % (receivers, sender, message, header)
+ message = header + "\n\n\n" + dashes + "\nINFO: mailed from " + hostname + "\n" + dashes + "\n"
+
+ try:
+ smtpObj = smtplib.SMTP('smtp.corp.redhat.com', 25)
+ #smtpObj = smtplib.SMTP('localhost', 25)
+ smtpObj.sendmail(sender, receivers, message)
+ print "INFO: Successfully sent email to %s" % receivers
+ except smtplib.SMTPException:
+ print "Error: unable to send email"
+
+
+##########################################################################################
+def tail2(f, n, offset=None):
+ """Reads a n lines from f with an offset of offset lines. The return
+ value is a tuple in the form ``(lines, has_more)`` where `has_more` is
+ an indicator that is `True` if there are more lines in the file.
+ """
+ avg_line_length = 74
+ to_read = n + (offset or 0)
+
+ while 1:
+ try:
+ f.seek(-(avg_line_length * to_read), 2)
+ except IOError:
+ # woops. apparently file is smaller than what we want
+ # to step back, go to the beginning instead
+ f.seek(0)
+ pos = f.tell()
+ lines = f.read().splitlines()
+ #lines = f.read()
+ if len(lines) >= to_read or pos == 0:
+ return lines[-to_read:offset and -offset or None], len(lines) > to_read or pos > 0
+ avg_line_length *= 1.3
+
+##########################################################################################
+def tail( f, window=20 ):
+ #BUFSIZ = 1024
+ BUFSIZ = 2048
+ f.seek(0, 2)
+ bytes = f.tell()
+ size = window
+ block = -1
+ data = []
+ while size > 0 and bytes > 0:
+ if (bytes - BUFSIZ > 0):
+ # Seek back one whole BUFSIZ
+ f.seek(block*BUFSIZ, 2)
+ # read BUFFER
+ data.append(f.read(BUFSIZ))
+ else:
+ # file too small, start from begining
+ f.seek(0,0)
+ # only read what was not read
+ # kwright commented out the next line trying to fix a bug
+ #data.append(f.read(bytes))
+ linesFound = data[-1].count('\n')
+ size -= linesFound
+ bytes -= BUFSIZ
+ block -= 1
+ return '\n'.join(''.join(data).splitlines()[-window:])
+
+
+##########################################################################################
+def runProcess(exe):
+ #print "runProcess"
+ p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ while(True):
+ retcode = p.poll() #returns None while subprocess is running
+ line = p.stdout.readline()
+ yield line
+ if(retcode is not None):
+ break
+
+##########################################################################################
+def run(command):
+ #if options.verbose:
+ # print command
+ rv, output = commands.getstatusoutput(command)
+ if rv:
+ #raise RuntimeError, "error running %s, %s" % (command, rv)
+ pass
+ return output
+
+##########################################################################################
+def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mailTo):
+ dashes = "+" + 70 * "-" + "+"
+ pounds = 70 * "#"
+
+ def print_debug(msg):
+ if options_debug == True:
+ print "DEBUG: %s" % (msg)
+
+ print_debug("package %s" % package)
+ print_debug("mock_cfg %s" % mock_cfg)
+ print_debug("group %s" % group)
+ if len(gitrev) > 0:
+ gitrev = "git" + gitrev
+ print_debug("gitrev %s" % gitrev)
+ print_debug("buildDir %s" % buildDir)
+ #if os.path.isdir(os.path.join(buildDir, package, ".git")):
+ #if os.path.isdir(resultsDir):
+ buildLogs = os.path.join(buildDir,"build_logs", mock_cfg)
+ print_debug("buildLogs: %s" % buildLogs)
+ if not os.path.exists(buildLogs):
+ os.makedirs(buildLogs)
+ os.chdir(buildLogs)
+
+ print_debug("cwd %s" % os.getcwd())
+ print pounds
+ print "INFO: build_package() -> building %s for %s" % (package, mock_cfg)
+ print pounds
+
+
+ SRPMS = buildDir + "/packages/SRPMS/"
+ if not os.path.exists(SRPMS):
+ os.makedirs(SRPMS)
+ os.chdir(SRPMS)
+ print_debug("cwd %s" % os.getcwd())
+ srpmsdir = os.listdir(SRPMS)
+ srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package
+ print_debug("srpmsdir %s" % srpmsdir)
+ print_debug("package = %s" % package)
+ print_debug("SRPMS = %s" % SRPMS)
+ print_debug("srpmPattern = %s" % srpmPattern)
+ SRPM=None
+ for fileName in os.listdir(SRPMS):
+ if re.search(srpmPattern, fileName):
+ print_debug("fileName %s" % fileName)
+ SRPM = fileName
+
+ if SRPM == None:
+ message = "Subject: %s ERROR: unable to build %s .\n\nThe src.rpm file for the package %s is not available. " % (group, package, package)
+ email(message, mailTo)
+ failure_message="ERROR: The src.rpm file for %s is not available!" % package
+ print_debug(failure_message)
+ print "INFO: %s " % dashes
+ print os.listdir(SRPMS)
+ print "INFO: %s " % dashes
+ #message = "Subject: %s The package %s build for %s failed\n\n%s" % (group, package, mock_cfg, failure_message )
+ #email(message, mailTo)
+ status="FAILED"
+ return(status)
+
+ else:
+ import datetime
+ now = datetime.datetime.now()
+ YEAR = now.strftime("%Y")
+ str(YEAR)
+ workbase = os.environ['HOME']
+ scriptsDir = workbase + "/scripts"
+
+
+ if not re.search(YEAR, SRPM):
+ print "INFO: we need to redo this src.rpm file."
+ os.chdir(SRPMS)
+ print_debug("SRPM = %s, gitrev = %s" % (SRPM, gitrev))
+ os.listdir(SRPMS)
+ script = "/srpm_timestamp.sh"
+ scriptPath = scriptsDir + script
+
+ #for line in runProcess(['/home/kwright/scripts/srpm_timestamp.sh', SRPM, gitrev]):
+ for line in runProcess([scriptPath, SRPM, gitrev]):
+ print line,
+ os.remove(SRPM)
+ for fileName in os.listdir(SRPMS):
+ if re.search(srpmPattern, fileName):
+ print_debug("fileName %s" % fileName)
+ SRPM = fileName
+ else:
+ print "INFO: This srpm has already been converted.\n"
+
+ #resultsDir= '/var/lib/mock/' + mock_cfg + '/result'
+ resultsDir= os.path.join('/var/lib/mock', mock_cfg, 'result')
+ print "%s\nINFO: cleaning up anything in %s\n%s\n" % (dashes, resultsDir, dashes)
+ if os.path.isdir(resultsDir):
+ fileList = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ]
+ print_debug("\nfileList %s" % fileList)
+ os.chdir(resultsDir)
+ for f in fileList:
+ print_debug("deleting %s" % f)
+ os.remove(f)
+
+ os.chdir(SRPMS)
+ print_debug("cwd %s" % os.getcwd())
+ print pounds
+ print "INFO: doing the mock build now of %s on %s" % (SRPM, mock_cfg)
+ print pounds
+ #logFilePath = buildLogs + "/build_" + package + ".log"
+ #buildLogs = os.path.join(buildDir,"build_logs", mock_cfg)
+ logFilePath = os.path.join(buildLogs, package + ".log")
+ logFile = open(logFilePath, "w")
+ for line in runProcess(['mock', '-v', '-r', mock_cfg, '--rebuild', SRPM]):
+ logFile.write(line,)
+ print line,
+ logFile.write("\n")
+ logFile.close()
+
+ RPM=None
+ fileName=None
+ rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg)
+ RPMList = []
+ builtRPMS=""
+ srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package
+ if not os.path.exists(rpmsDir):
+ os.makedirs(rpmsDir)
+ print "INFO: Creating dir %s" % rpmsDir
+ filelist = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ]
+ print "\n\n"
+ print_debug("filelist = %s" % filelist)
+ for RPM in filelist:
+ builtRPMS += RPM + "\n"
+ pathname = os.path.join(resultsDir, RPM)
+ if options_debug == True:
+ print_debug("copying:%s to %s" % (pathname, rpmsDir))
+ shutil.copy2(pathname, rpmsDir)
+
+ mockBuildLogs = os.path.join(buildDir,"build_logs", mock_cfg, package)
+ if not os.path.exists(mockBuildLogs):
+ print "INFO: Creating dir %s" % mockBuildLogs
+ os.makedirs(mockBuildLogs)
+ for basename in os.listdir(resultsDir):
+ if basename.endswith('.log'):
+ pathname = os.path.join(resultsDir, basename)
+ print_debug("Copying log file %s to %s" % (pathname, mockBuildLogs))
+ if os.path.isfile(pathname):
+ shutil.copy2(pathname, mockBuildLogs)
+
+ if builtRPMS != "":
+ print_debug("builtRPMS: %s" % builtRPMS.strip())
+ message = "Subject: %s The package %s build for %s completed successfully\n\nINFO: Here are the rpms built:\n\n%s" % (group, package, mock_cfg, builtRPMS)
+ status="COMPLETED"
+ print "INFO: status = ", status
+ email(message, mailTo)
+ return(status)
+ else:
+ print pounds
+ print "RPM FILE NOT FOUND !"
+#TODO move the generated build logs to this commented out line below
+ #not sure why I was trying to access the mock log file here. Since
+ #it's not currently being used, I'm commenting it out
+ #mockLogFilePath= os.path.join(buildLogs, package, "build.log")
+
+ logTail = ""
+ print pounds
+ logTail=subprocess.Popen([r"tail","-n 40", logFilePath], stdout=subprocess.PIPE).communicate()[0]
+ print logTail
+ print pounds
+
+ message = "Subject: %s The package %s build for %s failed\n\nINFO: Here are the last lines from the log file:\n%s\n%s\n" % (group, package, mock_cfg, dashes, logTail )
+ email(message, mailTo)
+ status="FAILED"
+ print "INFO: status = ", status
+ return(status)
+#end build_package()
+
+##########################################################################################
+def build_pki(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo):
+ def print_debug(msg):
+ if options_debug == True:
+ print "DEBUG: %s" % (msg)
+ print "INFO: Building package %s in %s for %s: %s" % (package, buildDir, mock_cfg, flavor)
+ if not os.path.exists(buildDir):
+ os.makedirs(buildDir)
+ os.chdir(buildDir)
+ pkiScriptsDir = buildDir + "/pki/scripts"
+ if not os.path.exists(pkiScriptsDir):
+ print "INFO: Getting the sources for %s" % package
+ git.Git().clone("git://git.fedorahosted.org/git/pki.git")
+ #else:
+ pkiGitDir = buildDir + "/pki"
+ os.chdir(pkiGitDir)
+ g = git.Git(pkiGitDir)
+ print g.log('--pretty=%h','-1','--').split('\n')
+ gitrev = g.log('--pretty=%h','-1','--').split('\n')[0]
+ print_debug(gitrev)
+ print_debug(type(gitrev))
+ #[gitrev, stuff1, stuff2] = g.log('--pretty=%h','-1','--').split('\n')
+ #gitrev2=gitrev[0]
+ #print type(gitrev2)
+ #gitrev=gitrev2
+ #sys.exit()
+ print "INFO: git rev = %s" % gitrev
+
+ srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package
+ SRPMS = buildDir + "/packages/SRPMS/"
+ if not os.path.exists(SRPMS):
+ os.makedirs(SRPMS)
+ os.chdir(SRPMS)
+ print "INFO: List of src.rpms:\n"
+ os.listdir(SRPMS)
+ SRPM=None
+ for fileName in os.listdir(SRPMS):
+ if re.search(srpmPattern, fileName):
+ print fileName
+ SRPM = fileName
+ if SRPM == None:
+ if package == "dogtag-pki":
+ srpmScript = scriptsDir + "/dogtag/build_" + package + "_srpm.sh"
+ print_debug("srpmScript: %s" % srpmScript)
+ shutil.copy(srpmScript, buildDir)
+ #cd ${BuildDir} && bash -x ./build_${PKG}_srpm.sh 2>&1 |tee ${BuildDir}/build_$PKG.srpm.log
+#TODO add logging to a file
+ os.chdir(buildDir)
+ for line in runProcess([srpmScript]):
+ print line,
+ else:
+ print "INFO: package name to convert = %s" % package
+ under = re.compile('-')
+ srpmScript = under.sub('_', package)
+ #print srpmScript #TODO add the -x for DEBUG
+ #cd ${BuildDir}/pki/scripts/ && bash -x ${BuildDir}/pki/scripts/compose_${PKG_SCRIPT}_packages hybrid_srpm
+ os.chdir(pkiScriptsDir)
+ srpmScript = pkiScriptsDir + "/compose_" + srpmScript + "_packages"
+ print srpmScript
+ for line in runProcess([srpmScript, 'hybrid_srpm']):
+ print line,
+ status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo)
+ print "Build status = %s for %s " % (status, package)
+ return status
+#end build_pki()
+
+##########################################################################################
+def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo):
+ def print_debug(msg):
+ if options_debug == True:
+ print "DEBUG: %s" % (msg)
+ print "package = %s " % package
+ #if not os.path.isdir(os.path.join(buildDir,package)):
+ # print "INFO: Creating directory %s " % os.path.join(buildDir,package)
+ # os.mkdir(os.path.join(buildDir,package))
+ print "+-----------------------------------------------------+"
+ print "Starting to build %s" % package
+ print "+-----------------------------------------------------+"
+ srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package
+ #SRPMS = buildDir + "/packages/SRPMS/"
+ SRPMS = os.path.join(buildDir,"packages/SRPMS")
+ print "SRPMS = %s" % SRPMS
+ if not os.path.exists(SRPMS):
+ os.makedirs(SRPMS)
+ os.chdir(SRPMS)
+ print "INFO: List of src.rpms:"
+ os.listdir(SRPMS)
+ SRPM=None
+ for fileName in os.listdir(SRPMS):
+ if re.search(srpmPattern, fileName):
+ print fileName
+ SRPM = fileName
+ if SRPM == None:
+ srpmScript = scriptsDir + "/build_" + package + ".sh"
+ print_debug("srpmScript = %s" % srpmScript)
+ shutil.copy(srpmScript, buildDir)
+ os.chdir(buildDir)
+ print os.getcwd()
+ srpmScript = buildDir + "/build_" + package + ".sh"
+ print "srpmScript = %s" % srpmScript
+ for line in runProcess([srpmScript]):
+ print line,
+
+ #check for exitence of .git directory
+ if os.path.isdir(os.path.join(buildDir, package, ".git")):
+ print ".git dir found %s" % os.path.join(buildDir, package, ".git")
+ os.chdir(os.path.join(buildDir, package))
+ print os.path.join(buildDir, package)
+ g = git.Git(os.path.join(buildDir,package))
+ gitrev = g.log('--pretty=%h','-1','--').split('\n')
+ gitrev2=gitrev[0]
+ gitrev=gitrev2
+ print "INFO: git rev = %s" % gitrev
+ else:
+ print ".git dir not found %s" % os.path.join(buildDir,package, ".git")
+ gitrev =""
+ #NOTE: the ipa package already has the gitrev embedded in the SRPM and doesn't need to be converted
+
+ status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo)
+ print "Build status = %s for %s " % (status, package)
+ return status
+#end build_generic_pkg()
+
+
+
+#TODO some of these 3 todo's are duplicates of those down at the symlink area
+#TODO add checking to see if the source checked out okay and that the pki/scripts dir exists
+#TODO put some code in here to verify that all the packages build succeded. if not, then we shouldn't even create the directories
+
+##########################################################################################
+def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, mailTo):
+ """ function for the repo directories
+ Currently this is called only when the build fails.
+
+ """
+ def print_debug(msg):
+ if options_debug == True:
+ print "DEBUG: %s" % (msg)
+ print_debug("function: createRepoDir")
+ print_debug("mock_cfg: %s" % mock_cfg)
+ print_debug("flavor: %s" % flavor)
+ print_debug("DT: %s" % DT)
+ print_debug("buildDir: %s" % buildDir)
+ print_debug("options_test: %s" % options_test)
+
+ workbase = os.environ['HOME']
+ scriptsDir = workbase + "/scripts"
+
+ dashes = "+" + 70 * "-" + "+"
+ longDashes = "+" + 80 * "-" + "+"
+ URL="http://mickey.dsdev.sjc.redhat.com"
+ print_debug("URL: %s" % URL)
+#TODO find a better name than repoPackage
+ repoPackage="pki"
+ print_debug("repoPackage: %s" % repoPackage)
+
+ [platform, rel, arch] = mock_cfg.split("-")
+
+ if arch == "i386":
+ arch = "i686"
+ print_debug("arch: %s" % arch)
+
+ if platform == "fedora":
+ dist="F" + rel
+ print_debug("rel: %s" % rel)
+
+ if platform == "rhel":
+ dist=platform + rel
+ print_debug("dist: %s" % dist)
+
+#TODO send an email the the build faild due to not being able to mount repos
+ if options_test:
+ repos = "/tmp/repos"
+ else:
+ repos = "/repos"
+ if not os.path.ismount(repos):
+ print "WARN: %s is not mounted" % repos
+ print "INFO: attempting to mount %s ... " % repos
+ for line in runProcess(["sudo", "/bin/mount", repos]):
+ print line
+ if not os.path.ismount(repos):
+ print "ERROR: unable to mount %s " % repos
+ sys.exit()
+
+ #TODO: handle other flavor names for future builds
+ if flavor == "DOGTAG 10":
+ #/repos/pki/dogtag/10
+ [flavorName, flavorGroup] = flavor.split()
+ flavorName=flavorName.lower()
+ repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
+ repoFull=os.path.join(repoBase, dist)
+ else:
+ flavorName="IPA"
+ flavorGroup="devel"
+ flavorName=flavorName.lower()
+ repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
+ repoFull=os.path.join(repoBase, dist)
+ print_debug("repoBase: %s" % repoBase)
+ print_debug("repoFull: %s" % repoFull)
+
+
+ #check for case when a new dist is released and create the dir
+ #################################################################
+ #create the repo tree
+ #################################################################
+ if os.path.exists(repoBase) and not os.path.exists(repoFull):
+ print "INFO: Creating dir", repoFull
+ os.makedirs(repoFull)
+ elif not os.path.exists(repoBase):
+ os.makedirs(repoBase)
+ #TODO add some logic here to exit if unable to create the directory
+ #print "ERROR: %s doesn't exist" % repoBase
+ #sys.exit(2)
+ if not os.path.exists(os.path.join(repoFull, DT)):
+ print "INFO: Creating dir %s" % os.path.join(repoFull, DT)
+ os.makedirs(os.path.join(repoFull, DT))
+ for dirType in (arch, "noarch", "src"):
+ if not os.path.exists(os.path.join(repoFull, DT, dirType)):
+ print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType)
+ os.makedirs(os.path.join(repoFull, DT, dirType))
+ if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")):
+ print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs")
+ os.makedirs(os.path.join(repoFull, DT, arch, "logs"))
+
+#def copyBuildLogs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo):
+ """still need a function for over the build logs only. in the case that one of the builds fails"""
+ sourceLogDir = os.path.join(buildDir,"build_logs", mock_cfg)
+ #repoBuildLogsDir = os.path.join(repoFull, DT, arch, "logs")
+ targetLogDir = os.path.join(repoFull, DT, arch, "logs", "build_logs", mock_cfg)
+ #destDir = os.path.join(repoFull, DT, rpmType)
+ #print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes))
+ #shutil.copy2(os.path.join(rpmsDir, RPM), destDir)
+ copyResult = distutils.dir_util.copy_tree(sourceLogDir, targetLogDir, preserve_mode=0, verbose=1)
+ print_debug("copyResult %s" % copyResult)
+ #print buildLogs
+ URL="http://mickey.dsdev.sjc.redhat.com"
+ #import json
+ test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".")
+ #message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch")
+ #print_debug("path =%s" % message_url_path)
+
+ failMessage= """
+You can find the build logs here:
+%s
+ %s%s
+ %s%s
+%s
+%s
+""" % (
+ longDashes,
+ URL, os.path.join(URL, repoFull, DT, arch),
+ URL, os.path.join(URL, repoFull, DT, "noarch"),
+ longDashes,
+ test_message,
+ )
+ buildStatus="FAILED"
+ message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, failMessage )
+ print(failMessage)
+ email(message, mailTo)
+#end createRepoDirs
+##########################################################################################
+
+def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo):
+ def print_debug(msg):
+ if options_debug == True:
+ print "DEBUG: %s" % (msg)
+
+ print_debug("mock_cfg: %s" % mock_cfg)
+ print_debug("flavor: %s" % flavor)
+ print_debug("DT: %s" % DT)
+ print_debug("buildDir: %s" % buildDir)
+ print_debug("options_test: %s" % options_test)
+
+ workbase = os.environ['HOME']
+ scriptsDir = workbase + "/scripts"
+
+ dashes = "+" + 70 * "-" + "+"
+ longDashes = "+" + 80 * "-" + "+"
+ URL="http://mickey.dsdev.sjc.redhat.com"
+ print_debug("URL: %s" % URL)
+#TODO find a better name than repoPackage
+ repoPackage="pki"
+ print_debug("repoPackage: %s" % repoPackage)
+
+ [platform, rel, arch] = mock_cfg.split("-")
+
+ if arch == "i386":
+ arch = "i686"
+ print_debug("arch: %s" % arch)
+
+ if platform == "fedora":
+ dist="F" + rel
+ print_debug("rel: %s" % rel)
+
+ if platform == "rhel":
+ dist=platform + rel
+ print_debug("dist: %s" % dist)
+
+#TODO send an email the the build faild due to not being able to mount repos
+ if options_test:
+ repos = "/tmp/repos"
+ else:
+ repos = "/repos"
+ if not os.path.ismount(repos):
+ print "WARN: %s is not mounted" % repos
+ print "INFO: attempting to mount %s ... " % repos
+ for line in runProcess(["sudo", "/bin/mount", repos]):
+ print line
+ if not os.path.ismount(repos):
+ print "ERROR: unable to mount %s " % repos
+ sys.exit()
+
+ #TODO: handle other flavor names for future builds
+ if flavor == "DOGTAG 10":
+ [flavorName, flavorGroup] = flavor.split()
+ flavorName=flavorName.lower()
+ repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
+ repoFull=os.path.join(repoBase, dist)
+ else:
+ flavorName="IPA"
+ flavorGroup="devel"
+ flavorName=flavorName.lower()
+ repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
+ repoFull=os.path.join(repoBase, dist)
+ print_debug("repoBase: %s" % repoBase)
+ print_debug("repoFull: %s" % repoFull)
+ if not options_skipWget:
+ #COPY over the download_builds.sh script
+ downloadScript=os.path.join(scriptsDir, flavorName, "download_builds.sh")
+ print_debug("downloadScript = %s" % downloadScript)
+ shutil.copy(downloadScript, buildDir)
+ os.chdir(buildDir)
+ downloadScript=os.path.join(buildDir, "download_builds.sh")
+ print_debug("downloadScript = %s" % downloadScript)
+ for line in runProcess([downloadScript, arch]):
+ print line,
+
+
+ #check for case when a new dist is released and create the dir
+ #################################################################
+ #create the repo tree
+ #################################################################
+ if os.path.exists(repoBase) and not os.path.exists(repoFull):
+ print "INFO: Creating dir", repoFull
+ os.makedirs(repoFull)
+ elif not os.path.exists(repoBase):
+ os.makedirs(repoBase)
+ #print "ERROR: %s doesn't exist" % repoBase
+ #sys.exit(2)
+#TODO add some logic here to exit if unable to create the directory
+ if not os.path.exists(os.path.join(repoFull, DT)):
+ print "INFO: Creating dir %s" % os.path.join(repoFull, DT)
+ os.makedirs(os.path.join(repoFull, DT))
+ for dirType in (arch, "noarch", "src"):
+ if not os.path.exists(os.path.join(repoFull, DT, dirType)):
+ print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType)
+ os.makedirs(os.path.join(repoFull, DT, dirType))
+ if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")):
+ print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs")
+ os.makedirs(os.path.join(repoFull, DT, arch, "logs"))
+ buildLogs = os.path.join(repoFull, DT, arch, "logs")
+ print buildLogs
+
+ os.chdir(repoFull)
+#TODO add some logic that if any of the builds fail,
+#Currently, if the build fails, I skip the entire creation of the repos--including the build logs
+# 1) do not create the symlink - DONE
+# 2) copy the build logs over - DONE
+# 3) do not copy the builds -DONE
+# 4) send an email with a url to the build logs - DONE (in other function)
+
+ archSymlink = "devel_" + arch
+
+ if os.path.exists("devel_" + arch):
+ os.remove("devel_" + arch)
+ print os.getcwd()
+ print "creating symlink %s_devel to %s " % (arch, DT)
+ os.symlink(DT, "devel_" + arch)
+
+ buildStatus="COMPLETED"
+
+ #################################################################
+ #copy the builds over to the newly created repo tree
+ #################################################################
+#TODO add some logic for the case where the only build is a noarch and there are no arch build (dogtag-pki)
+#currently, this creates an empty log file for fileList_i686
+# /tmp/repos/pki/dogtag/10/F17/20120913-150042/i686/logs/fileList_i686
+
+ rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg)
+ print dashes
+
+ for rpmType in (arch, "noarch", "src"):
+ filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ]
+ filelist.sort()
+ logFile = open(buildLogs + "/fileList_" + rpmType , "w")
+ destDir = os.path.join(repoFull, DT, rpmType)
+ for RPM in filelist:
+ print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes))
+ shutil.copy2(os.path.join(rpmsDir, RPM), destDir)
+ logFile.write(RPM+"\n")
+ logFile.close()
+#handle i386 rpm files that are part of the i686 repo
+ rpmType="i386"
+ filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ]
+ filelist.sort()
+ logFile = open(buildLogs + "/fileList_" + "i686" , "a")
+ destDir = os.path.join(repoFull, DT, "i686")
+ for RPM in filelist:
+ print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes))
+ shutil.copy2(os.path.join(rpmsDir, RPM), destDir)
+ logFile.write(RPM+"\n")
+ logFile.close()
+
+ #sort the file into a temp file then copy it back. this is needed for the email with the list of files
+ logFile = os.path.join(buildLogs, "fileList_" + "i686")
+ logFileTmp = os.path.join(buildLogs, "fileList_" + "i686.out")
+ inLogFile = open(logFile , "r")
+ outLogFile = open(logFileTmp , "w")
+ map(outLogFile.write, sorted(inLogFile.readlines()))
+ inLogFile.close()
+ outLogFile.close()
+ shutil.copy2(logFileTmp, logFile)
+ os.remove(logFileTmp)
+
+
+
+
+#when I need to create a repo:
+#add logic to not do this if build failed
+ for rpmType in (arch, "noarch", "src"):
+ destDir = os.path.join(repoFull, DT, rpmType)
+ logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "w")
+ print_debug("logFile = %s" % logFile)
+ cmd = "createrepo -vv --no-database --skip-stat " + destDir
+ output=run(cmd)
+ print output
+ logFile.write(output+"\n")
+ logFile.close()
+ numPkgDict={}
+ for rpmType in (arch, "noarch", "src"):
+ destDir = os.path.join(repoFull, DT, rpmType)
+ logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "r")
+ for line in logFile:
+ if "Spawning" in line:
+ #print line
+ pkgNum=line.split()[-2]
+ print_debug("pkgNum: %s" % pkgNum)
+ numPkgDict[rpmType]=pkgNum
+
+ print_debug("numPkgDict: %s" % numPkgDict)
+
+
+#TODO copy the log files over to the build_logs directory
+ rpmInRepoDict={}
+ for rpmType in (arch, "noarch", "src"):
+ rpmInRepoDict[rpmType] = open(buildLogs + "/fileList_" + rpmType, "r").read().split()
+ #" ".join(REPOLOG)
+ #print REPOLOG
+ #close(buildLogs + "/fileList_" + rpmType)
+
+ import json
+
+ #TODO add some logic here to check for test mode and alter the email accordingly
+ #NOTE this is partially implemented below:
+ test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".")
+ message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch")
+ print_debug("path =%s" % message_url_path)
+
+ releaseMessage= """
+You can find the build repos here:
+%s
+ %s%s
+ %s%s
+
+ %s%s
+ %s%s
+
+Summary of the number of RPMS that were built:
+%s
+%s
+
+Here are the RPMs that were built:
+%s
+%s
+%s
+ """ % (
+ longDashes,
+ URL, os.path.join(URL, repoFull, DT, arch),
+ URL, os.path.join(URL, repoFull, DT, "noarch"),
+ URL, os.path.join(URL, repoFull, "devel_" + arch, arch ),
+ URL, os.path.join(URL, repoFull, "devel_" + arch, "noarch"),
+ longDashes,
+ json.dumps(numPkgDict, indent=4),
+ longDashes,
+ json.dumps(rpmInRepoDict, indent=4),
+ test_message,
+ )
+ print releaseMessage
+
+ message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, releaseMessage )
+
+ email(message, mailTo)
+
+#end copyToRepo