diff options
Diffstat (limited to 'python-scripts/build_functions.py')
| -rwxr-xr-x | python-scripts/build_functions.py | 799 |
1 files changed, 799 insertions, 0 deletions
diff --git a/python-scripts/build_functions.py b/python-scripts/build_functions.py new file mode 100755 index 0000000..adad8f2 --- /dev/null +++ b/python-scripts/build_functions.py @@ -0,0 +1,799 @@ +#!/usr/bin/python + +"""porting build functions from bash script""" + +import re +import os +import subprocess +import commands +import shutil +import git +import sys +import distutils.dir_util + + +########################################################################################## +def email(message, mailTo): + import smtplib + import socket + mail_server='smtp.corp.redhat.com' + mail_server_port=25 + + receivers=mailTo + + #if options_mailMe: + # receivers = 'kwright@redhat.com' + #else: + # receivers = 'ipa-and-samba-team-automation@redhat.com' + + sender = 'kwright@redhat.com' + + #print socket.gethostname() + hostname = socket.gethostname() + dashes = "+" + 70 * "-" + "+" + header = "" + header = """\ +To: %s +From: %s +%s +%s +""" % (receivers, sender, message, header) + message = header + "\n\n\n" + dashes + "\nINFO: mailed from " + hostname + "\n" + dashes + "\n" + + try: + smtpObj = smtplib.SMTP('smtp.corp.redhat.com', 25) + #smtpObj = smtplib.SMTP('localhost', 25) + smtpObj.sendmail(sender, receivers, message) + print "INFO: Successfully sent email to %s" % receivers + except smtplib.SMTPException: + print "Error: unable to send email" + + +########################################################################################## +def tail2(f, n, offset=None): + """Reads a n lines from f with an offset of offset lines. The return + value is a tuple in the form ``(lines, has_more)`` where `has_more` is + an indicator that is `True` if there are more lines in the file. + """ + avg_line_length = 74 + to_read = n + (offset or 0) + + while 1: + try: + f.seek(-(avg_line_length * to_read), 2) + except IOError: + # woops. apparently file is smaller than what we want + # to step back, go to the beginning instead + f.seek(0) + pos = f.tell() + lines = f.read().splitlines() + #lines = f.read() + if len(lines) >= to_read or pos == 0: + return lines[-to_read:offset and -offset or None], len(lines) > to_read or pos > 0 + avg_line_length *= 1.3 + +########################################################################################## +def tail( f, window=20 ): + #BUFSIZ = 1024 + BUFSIZ = 2048 + f.seek(0, 2) + bytes = f.tell() + size = window + block = -1 + data = [] + while size > 0 and bytes > 0: + if (bytes - BUFSIZ > 0): + # Seek back one whole BUFSIZ + f.seek(block*BUFSIZ, 2) + # read BUFFER + data.append(f.read(BUFSIZ)) + else: + # file too small, start from begining + f.seek(0,0) + # only read what was not read + # kwright commented out the next line trying to fix a bug + #data.append(f.read(bytes)) + linesFound = data[-1].count('\n') + size -= linesFound + bytes -= BUFSIZ + block -= 1 + return '\n'.join(''.join(data).splitlines()[-window:]) + + +########################################################################################## +def runProcess(exe): + #print "runProcess" + p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + while(True): + retcode = p.poll() #returns None while subprocess is running + line = p.stdout.readline() + yield line + if(retcode is not None): + break + +########################################################################################## +def run(command): + #if options.verbose: + # print command + rv, output = commands.getstatusoutput(command) + if rv: + #raise RuntimeError, "error running %s, %s" % (command, rv) + pass + return output + +########################################################################################## +def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mailTo): + dashes = "+" + 70 * "-" + "+" + pounds = 70 * "#" + + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + + print_debug("package %s" % package) + print_debug("mock_cfg %s" % mock_cfg) + print_debug("group %s" % group) + if len(gitrev) > 0: + gitrev = "git" + gitrev + print_debug("gitrev %s" % gitrev) + print_debug("buildDir %s" % buildDir) + #if os.path.isdir(os.path.join(buildDir, package, ".git")): + #if os.path.isdir(resultsDir): + buildLogs = os.path.join(buildDir,"build_logs", mock_cfg) + print_debug("buildLogs: %s" % buildLogs) + if not os.path.exists(buildLogs): + os.makedirs(buildLogs) + os.chdir(buildLogs) + + print_debug("cwd %s" % os.getcwd()) + print pounds + print "INFO: build_package() -> building %s for %s" % (package, mock_cfg) + print pounds + + + SRPMS = buildDir + "/packages/SRPMS/" + if not os.path.exists(SRPMS): + os.makedirs(SRPMS) + os.chdir(SRPMS) + print_debug("cwd %s" % os.getcwd()) + srpmsdir = os.listdir(SRPMS) + srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package + print_debug("srpmsdir %s" % srpmsdir) + print_debug("package = %s" % package) + print_debug("SRPMS = %s" % SRPMS) + print_debug("srpmPattern = %s" % srpmPattern) + SRPM=None + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print_debug("fileName %s" % fileName) + SRPM = fileName + + if SRPM == None: + message = "Subject: %s ERROR: unable to build %s .\n\nThe src.rpm file for the package %s is not available. " % (group, package, package) + email(message, mailTo) + failure_message="ERROR: The src.rpm file for %s is not available!" % package + print_debug(failure_message) + print "INFO: %s " % dashes + print os.listdir(SRPMS) + print "INFO: %s " % dashes + #message = "Subject: %s The package %s build for %s failed\n\n%s" % (group, package, mock_cfg, failure_message ) + #email(message, mailTo) + status="FAILED" + return(status) + + else: + import datetime + now = datetime.datetime.now() + YEAR = now.strftime("%Y") + str(YEAR) + workbase = os.environ['HOME'] + scriptsDir = workbase + "/scripts" + + + if not re.search(YEAR, SRPM): + print "INFO: we need to redo this src.rpm file." + os.chdir(SRPMS) + print_debug("SRPM = %s, gitrev = %s" % (SRPM, gitrev)) + os.listdir(SRPMS) + script = "/srpm_timestamp.sh" + scriptPath = scriptsDir + script + + #for line in runProcess(['/home/kwright/scripts/srpm_timestamp.sh', SRPM, gitrev]): + for line in runProcess([scriptPath, SRPM, gitrev]): + print line, + os.remove(SRPM) + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print_debug("fileName %s" % fileName) + SRPM = fileName + else: + print "INFO: This srpm has already been converted.\n" + + #resultsDir= '/var/lib/mock/' + mock_cfg + '/result' + resultsDir= os.path.join('/var/lib/mock', mock_cfg, 'result') + print "%s\nINFO: cleaning up anything in %s\n%s\n" % (dashes, resultsDir, dashes) + if os.path.isdir(resultsDir): + fileList = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ] + print_debug("\nfileList %s" % fileList) + os.chdir(resultsDir) + for f in fileList: + print_debug("deleting %s" % f) + os.remove(f) + + os.chdir(SRPMS) + print_debug("cwd %s" % os.getcwd()) + print pounds + print "INFO: doing the mock build now of %s on %s" % (SRPM, mock_cfg) + print pounds + #logFilePath = buildLogs + "/build_" + package + ".log" + #buildLogs = os.path.join(buildDir,"build_logs", mock_cfg) + logFilePath = os.path.join(buildLogs, package + ".log") + logFile = open(logFilePath, "w") + for line in runProcess(['mock', '-v', '-r', mock_cfg, '--rebuild', SRPM]): + logFile.write(line,) + print line, + logFile.write("\n") + logFile.close() + + RPM=None + fileName=None + rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg) + RPMList = [] + builtRPMS="" + srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package + if not os.path.exists(rpmsDir): + os.makedirs(rpmsDir) + print "INFO: Creating dir %s" % rpmsDir + filelist = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ] + print "\n\n" + print_debug("filelist = %s" % filelist) + for RPM in filelist: + builtRPMS += RPM + "\n" + pathname = os.path.join(resultsDir, RPM) + if options_debug == True: + print_debug("copying:%s to %s" % (pathname, rpmsDir)) + shutil.copy2(pathname, rpmsDir) + + mockBuildLogs = os.path.join(buildDir,"build_logs", mock_cfg, package) + if not os.path.exists(mockBuildLogs): + print "INFO: Creating dir %s" % mockBuildLogs + os.makedirs(mockBuildLogs) + for basename in os.listdir(resultsDir): + if basename.endswith('.log'): + pathname = os.path.join(resultsDir, basename) + print_debug("Copying log file %s to %s" % (pathname, mockBuildLogs)) + if os.path.isfile(pathname): + shutil.copy2(pathname, mockBuildLogs) + + if builtRPMS != "": + print_debug("builtRPMS: %s" % builtRPMS.strip()) + message = "Subject: %s The package %s build for %s completed successfully\n\nINFO: Here are the rpms built:\n\n%s" % (group, package, mock_cfg, builtRPMS) + status="COMPLETED" + print "INFO: status = ", status + email(message, mailTo) + return(status) + else: + print pounds + print "RPM FILE NOT FOUND !" +#TODO move the generated build logs to this commented out line below + #not sure why I was trying to access the mock log file here. Since + #it's not currently being used, I'm commenting it out + #mockLogFilePath= os.path.join(buildLogs, package, "build.log") + + logTail = "" + print pounds + logTail=subprocess.Popen([r"tail","-n 40", logFilePath], stdout=subprocess.PIPE).communicate()[0] + print logTail + print pounds + + message = "Subject: %s The package %s build for %s failed\n\nINFO: Here are the last lines from the log file:\n%s\n%s\n" % (group, package, mock_cfg, dashes, logTail ) + email(message, mailTo) + status="FAILED" + print "INFO: status = ", status + return(status) +#end build_package() + +########################################################################################## +def build_pki(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo): + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + print "INFO: Building package %s in %s for %s: %s" % (package, buildDir, mock_cfg, flavor) + if not os.path.exists(buildDir): + os.makedirs(buildDir) + os.chdir(buildDir) + pkiScriptsDir = buildDir + "/pki/scripts" + if not os.path.exists(pkiScriptsDir): + print "INFO: Getting the sources for %s" % package + git.Git().clone("git://git.fedorahosted.org/git/pki.git") + #else: + pkiGitDir = buildDir + "/pki" + os.chdir(pkiGitDir) + g = git.Git(pkiGitDir) + print g.log('--pretty=%h','-1','--').split('\n') + gitrev = g.log('--pretty=%h','-1','--').split('\n')[0] + print_debug(gitrev) + print_debug(type(gitrev)) + #[gitrev, stuff1, stuff2] = g.log('--pretty=%h','-1','--').split('\n') + #gitrev2=gitrev[0] + #print type(gitrev2) + #gitrev=gitrev2 + #sys.exit() + print "INFO: git rev = %s" % gitrev + + srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package + SRPMS = buildDir + "/packages/SRPMS/" + if not os.path.exists(SRPMS): + os.makedirs(SRPMS) + os.chdir(SRPMS) + print "INFO: List of src.rpms:\n" + os.listdir(SRPMS) + SRPM=None + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print fileName + SRPM = fileName + if SRPM == None: + if package == "dogtag-pki": + srpmScript = scriptsDir + "/dogtag/build_" + package + "_srpm.sh" + print_debug("srpmScript: %s" % srpmScript) + shutil.copy(srpmScript, buildDir) + #cd ${BuildDir} && bash -x ./build_${PKG}_srpm.sh 2>&1 |tee ${BuildDir}/build_$PKG.srpm.log +#TODO add logging to a file + os.chdir(buildDir) + for line in runProcess([srpmScript]): + print line, + else: + print "INFO: package name to convert = %s" % package + under = re.compile('-') + srpmScript = under.sub('_', package) + #print srpmScript #TODO add the -x for DEBUG + #cd ${BuildDir}/pki/scripts/ && bash -x ${BuildDir}/pki/scripts/compose_${PKG_SCRIPT}_packages hybrid_srpm + os.chdir(pkiScriptsDir) + srpmScript = pkiScriptsDir + "/compose_" + srpmScript + "_packages" + print srpmScript + for line in runProcess([srpmScript, 'hybrid_srpm']): + print line, + status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo) + print "Build status = %s for %s " % (status, package) + return status +#end build_pki() + +########################################################################################## +def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo): + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + print "package = %s " % package + #if not os.path.isdir(os.path.join(buildDir,package)): + # print "INFO: Creating directory %s " % os.path.join(buildDir,package) + # os.mkdir(os.path.join(buildDir,package)) + print "+-----------------------------------------------------+" + print "Starting to build %s" % package + print "+-----------------------------------------------------+" + srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package + #SRPMS = buildDir + "/packages/SRPMS/" + SRPMS = os.path.join(buildDir,"packages/SRPMS") + print "SRPMS = %s" % SRPMS + if not os.path.exists(SRPMS): + os.makedirs(SRPMS) + os.chdir(SRPMS) + print "INFO: List of src.rpms:" + os.listdir(SRPMS) + SRPM=None + for fileName in os.listdir(SRPMS): + if re.search(srpmPattern, fileName): + print fileName + SRPM = fileName + if SRPM == None: + srpmScript = scriptsDir + "/build_" + package + ".sh" + print_debug("srpmScript = %s" % srpmScript) + shutil.copy(srpmScript, buildDir) + os.chdir(buildDir) + print os.getcwd() + srpmScript = buildDir + "/build_" + package + ".sh" + print "srpmScript = %s" % srpmScript + for line in runProcess([srpmScript]): + print line, + + #check for exitence of .git directory + if os.path.isdir(os.path.join(buildDir, package, ".git")): + print ".git dir found %s" % os.path.join(buildDir, package, ".git") + os.chdir(os.path.join(buildDir, package)) + print os.path.join(buildDir, package) + g = git.Git(os.path.join(buildDir,package)) + gitrev = g.log('--pretty=%h','-1','--').split('\n') + gitrev2=gitrev[0] + gitrev=gitrev2 + print "INFO: git rev = %s" % gitrev + else: + print ".git dir not found %s" % os.path.join(buildDir,package, ".git") + gitrev ="" + #NOTE: the ipa package already has the gitrev embedded in the SRPM and doesn't need to be converted + + status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo) + print "Build status = %s for %s " % (status, package) + return status +#end build_generic_pkg() + + + +#TODO some of these 3 todo's are duplicates of those down at the symlink area +#TODO add checking to see if the source checked out okay and that the pki/scripts dir exists +#TODO put some code in here to verify that all the packages build succeded. if not, then we shouldn't even create the directories + +########################################################################################## +def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, mailTo): + """ function for the repo directories + Currently this is called only when the build fails. + + """ + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + print_debug("function: createRepoDir") + print_debug("mock_cfg: %s" % mock_cfg) + print_debug("flavor: %s" % flavor) + print_debug("DT: %s" % DT) + print_debug("buildDir: %s" % buildDir) + print_debug("options_test: %s" % options_test) + + workbase = os.environ['HOME'] + scriptsDir = workbase + "/scripts" + + dashes = "+" + 70 * "-" + "+" + longDashes = "+" + 80 * "-" + "+" + URL="http://mickey.dsdev.sjc.redhat.com" + print_debug("URL: %s" % URL) +#TODO find a better name than repoPackage + repoPackage="pki" + print_debug("repoPackage: %s" % repoPackage) + + [platform, rel, arch] = mock_cfg.split("-") + + if arch == "i386": + arch = "i686" + print_debug("arch: %s" % arch) + + if platform == "fedora": + dist="F" + rel + print_debug("rel: %s" % rel) + + if platform == "rhel": + dist=platform + rel + print_debug("dist: %s" % dist) + +#TODO send an email the the build faild due to not being able to mount repos + if options_test: + repos = "/tmp/repos" + else: + repos = "/repos" + if not os.path.ismount(repos): + print "WARN: %s is not mounted" % repos + print "INFO: attempting to mount %s ... " % repos + for line in runProcess(["sudo", "/bin/mount", repos]): + print line + if not os.path.ismount(repos): + print "ERROR: unable to mount %s " % repos + sys.exit() + + #TODO: handle other flavor names for future builds + if flavor == "DOGTAG 10": + #/repos/pki/dogtag/10 + [flavorName, flavorGroup] = flavor.split() + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + else: + flavorName="IPA" + flavorGroup="devel" + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + print_debug("repoBase: %s" % repoBase) + print_debug("repoFull: %s" % repoFull) + + + #check for case when a new dist is released and create the dir + ################################################################# + #create the repo tree + ################################################################# + if os.path.exists(repoBase) and not os.path.exists(repoFull): + print "INFO: Creating dir", repoFull + os.makedirs(repoFull) + elif not os.path.exists(repoBase): + os.makedirs(repoBase) + #TODO add some logic here to exit if unable to create the directory + #print "ERROR: %s doesn't exist" % repoBase + #sys.exit(2) + if not os.path.exists(os.path.join(repoFull, DT)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT) + os.makedirs(os.path.join(repoFull, DT)) + for dirType in (arch, "noarch", "src"): + if not os.path.exists(os.path.join(repoFull, DT, dirType)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType) + os.makedirs(os.path.join(repoFull, DT, dirType)) + if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs") + os.makedirs(os.path.join(repoFull, DT, arch, "logs")) + +#def copyBuildLogs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo): + """still need a function for over the build logs only. in the case that one of the builds fails""" + sourceLogDir = os.path.join(buildDir,"build_logs", mock_cfg) + #repoBuildLogsDir = os.path.join(repoFull, DT, arch, "logs") + targetLogDir = os.path.join(repoFull, DT, arch, "logs", "build_logs", mock_cfg) + #destDir = os.path.join(repoFull, DT, rpmType) + #print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes)) + #shutil.copy2(os.path.join(rpmsDir, RPM), destDir) + copyResult = distutils.dir_util.copy_tree(sourceLogDir, targetLogDir, preserve_mode=0, verbose=1) + print_debug("copyResult %s" % copyResult) + #print buildLogs + URL="http://mickey.dsdev.sjc.redhat.com" + #import json + test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".") + #message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch") + #print_debug("path =%s" % message_url_path) + + failMessage= """ +You can find the build logs here: +%s + %s%s + %s%s +%s +%s +""" % ( + longDashes, + URL, os.path.join(URL, repoFull, DT, arch), + URL, os.path.join(URL, repoFull, DT, "noarch"), + longDashes, + test_message, + ) + buildStatus="FAILED" + message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, failMessage ) + print(failMessage) + email(message, mailTo) +#end createRepoDirs +########################################################################################## + +def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo): + def print_debug(msg): + if options_debug == True: + print "DEBUG: %s" % (msg) + + print_debug("mock_cfg: %s" % mock_cfg) + print_debug("flavor: %s" % flavor) + print_debug("DT: %s" % DT) + print_debug("buildDir: %s" % buildDir) + print_debug("options_test: %s" % options_test) + + workbase = os.environ['HOME'] + scriptsDir = workbase + "/scripts" + + dashes = "+" + 70 * "-" + "+" + longDashes = "+" + 80 * "-" + "+" + URL="http://mickey.dsdev.sjc.redhat.com" + print_debug("URL: %s" % URL) +#TODO find a better name than repoPackage + repoPackage="pki" + print_debug("repoPackage: %s" % repoPackage) + + [platform, rel, arch] = mock_cfg.split("-") + + if arch == "i386": + arch = "i686" + print_debug("arch: %s" % arch) + + if platform == "fedora": + dist="F" + rel + print_debug("rel: %s" % rel) + + if platform == "rhel": + dist=platform + rel + print_debug("dist: %s" % dist) + +#TODO send an email the the build faild due to not being able to mount repos + if options_test: + repos = "/tmp/repos" + else: + repos = "/repos" + if not os.path.ismount(repos): + print "WARN: %s is not mounted" % repos + print "INFO: attempting to mount %s ... " % repos + for line in runProcess(["sudo", "/bin/mount", repos]): + print line + if not os.path.ismount(repos): + print "ERROR: unable to mount %s " % repos + sys.exit() + + #TODO: handle other flavor names for future builds + if flavor == "DOGTAG 10": + [flavorName, flavorGroup] = flavor.split() + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + else: + flavorName="IPA" + flavorGroup="devel" + flavorName=flavorName.lower() + repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup) + repoFull=os.path.join(repoBase, dist) + print_debug("repoBase: %s" % repoBase) + print_debug("repoFull: %s" % repoFull) + if not options_skipWget: + #COPY over the download_builds.sh script + downloadScript=os.path.join(scriptsDir, flavorName, "download_builds.sh") + print_debug("downloadScript = %s" % downloadScript) + shutil.copy(downloadScript, buildDir) + os.chdir(buildDir) + downloadScript=os.path.join(buildDir, "download_builds.sh") + print_debug("downloadScript = %s" % downloadScript) + for line in runProcess([downloadScript, arch]): + print line, + + + #check for case when a new dist is released and create the dir + ################################################################# + #create the repo tree + ################################################################# + if os.path.exists(repoBase) and not os.path.exists(repoFull): + print "INFO: Creating dir", repoFull + os.makedirs(repoFull) + elif not os.path.exists(repoBase): + os.makedirs(repoBase) + #print "ERROR: %s doesn't exist" % repoBase + #sys.exit(2) +#TODO add some logic here to exit if unable to create the directory + if not os.path.exists(os.path.join(repoFull, DT)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT) + os.makedirs(os.path.join(repoFull, DT)) + for dirType in (arch, "noarch", "src"): + if not os.path.exists(os.path.join(repoFull, DT, dirType)): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType) + os.makedirs(os.path.join(repoFull, DT, dirType)) + if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")): + print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs") + os.makedirs(os.path.join(repoFull, DT, arch, "logs")) + buildLogs = os.path.join(repoFull, DT, arch, "logs") + print buildLogs + + os.chdir(repoFull) +#TODO add some logic that if any of the builds fail, +#Currently, if the build fails, I skip the entire creation of the repos--including the build logs +# 1) do not create the symlink - DONE +# 2) copy the build logs over - DONE +# 3) do not copy the builds -DONE +# 4) send an email with a url to the build logs - DONE (in other function) + + archSymlink = "devel_" + arch + + if os.path.exists("devel_" + arch): + os.remove("devel_" + arch) + print os.getcwd() + print "creating symlink %s_devel to %s " % (arch, DT) + os.symlink(DT, "devel_" + arch) + + buildStatus="COMPLETED" + + ################################################################# + #copy the builds over to the newly created repo tree + ################################################################# +#TODO add some logic for the case where the only build is a noarch and there are no arch build (dogtag-pki) +#currently, this creates an empty log file for fileList_i686 +# /tmp/repos/pki/dogtag/10/F17/20120913-150042/i686/logs/fileList_i686 + + rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg) + print dashes + + for rpmType in (arch, "noarch", "src"): + filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ] + filelist.sort() + logFile = open(buildLogs + "/fileList_" + rpmType , "w") + destDir = os.path.join(repoFull, DT, rpmType) + for RPM in filelist: + print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes)) + shutil.copy2(os.path.join(rpmsDir, RPM), destDir) + logFile.write(RPM+"\n") + logFile.close() +#handle i386 rpm files that are part of the i686 repo + rpmType="i386" + filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ] + filelist.sort() + logFile = open(buildLogs + "/fileList_" + "i686" , "a") + destDir = os.path.join(repoFull, DT, "i686") + for RPM in filelist: + print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir, dashes)) + shutil.copy2(os.path.join(rpmsDir, RPM), destDir) + logFile.write(RPM+"\n") + logFile.close() + + #sort the file into a temp file then copy it back. this is needed for the email with the list of files + logFile = os.path.join(buildLogs, "fileList_" + "i686") + logFileTmp = os.path.join(buildLogs, "fileList_" + "i686.out") + inLogFile = open(logFile , "r") + outLogFile = open(logFileTmp , "w") + map(outLogFile.write, sorted(inLogFile.readlines())) + inLogFile.close() + outLogFile.close() + shutil.copy2(logFileTmp, logFile) + os.remove(logFileTmp) + + + + +#when I need to create a repo: +#add logic to not do this if build failed + for rpmType in (arch, "noarch", "src"): + destDir = os.path.join(repoFull, DT, rpmType) + logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "w") + print_debug("logFile = %s" % logFile) + cmd = "createrepo -vv --no-database --skip-stat " + destDir + output=run(cmd) + print output + logFile.write(output+"\n") + logFile.close() + numPkgDict={} + for rpmType in (arch, "noarch", "src"): + destDir = os.path.join(repoFull, DT, rpmType) + logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "r") + for line in logFile: + if "Spawning" in line: + #print line + pkgNum=line.split()[-2] + print_debug("pkgNum: %s" % pkgNum) + numPkgDict[rpmType]=pkgNum + + print_debug("numPkgDict: %s" % numPkgDict) + + +#TODO copy the log files over to the build_logs directory + rpmInRepoDict={} + for rpmType in (arch, "noarch", "src"): + rpmInRepoDict[rpmType] = open(buildLogs + "/fileList_" + rpmType, "r").read().split() + #" ".join(REPOLOG) + #print REPOLOG + #close(buildLogs + "/fileList_" + rpmType) + + import json + + #TODO add some logic here to check for test mode and alter the email accordingly + #NOTE this is partially implemented below: + test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".") + message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch") + print_debug("path =%s" % message_url_path) + + releaseMessage= """ +You can find the build repos here: +%s + %s%s + %s%s + + %s%s + %s%s + +Summary of the number of RPMS that were built: +%s +%s + +Here are the RPMs that were built: +%s +%s +%s + """ % ( + longDashes, + URL, os.path.join(URL, repoFull, DT, arch), + URL, os.path.join(URL, repoFull, DT, "noarch"), + URL, os.path.join(URL, repoFull, "devel_" + arch, arch ), + URL, os.path.join(URL, repoFull, "devel_" + arch, "noarch"), + longDashes, + json.dumps(numPkgDict, indent=4), + longDashes, + json.dumps(rpmInRepoDict, indent=4), + test_message, + ) + print releaseMessage + + message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, releaseMessage ) + + email(message, mailTo) + +#end copyToRepo |