path: root/python-scripts/build_functions.py

#!/usr/bin/python

"""porting build functions from bash script"""

import re
import os
import subprocess
import commands
import shutil
import git 
import sys
import distutils.dir_util


##########################################################################################
def email(message, mailTo):  
    import smtplib
    import socket
    mail_server='smtp.corp.redhat.com'
    mail_server_port=25

    receivers=mailTo

    #if options_mailMe:
    #    receivers = 'kwright@redhat.com'
    #else:
    #    receivers = 'ipa-and-samba-team-automation@redhat.com'

    sender = 'kwright@redhat.com'

    #print socket.gethostname()
    hostname = socket.gethostname()
    dashes = "+" + 70 * "-" + "+"
    header = ""
    header = """\
To: %s
From: %s
%s
%s
""" % (receivers, sender, message, header)
    message = header + "\n\n\n" + dashes + "\nINFO: mailed from " + hostname + "\n" + dashes + "\n" 
    
    try:
        smtpObj = smtplib.SMTP('smtp.corp.redhat.com', 25)
        #smtpObj = smtplib.SMTP('localhost', 25)
        smtpObj.sendmail(sender, receivers, message)
        print "INFO: Successfully sent email to %s" % receivers
    except smtplib.SMTPException:
        print "Error: unable to send email"


##########################################################################################
def tail2(f, n, offset=None):
    """Reads a n lines from f with an offset of offset lines.  The return
    value is a tuple in the form ``(lines, has_more)`` where `has_more` is
    an indicator that is `True` if there are more lines in the file.
    """
    avg_line_length = 74
    to_read = n + (offset or 0)

    while 1:
        try:
            f.seek(-(avg_line_length * to_read), 2)
        except IOError:
            # woops.  apparently file is smaller than what we want
            # to step back, go to the beginning instead
            f.seek(0)
        pos = f.tell()
        lines = f.read().splitlines()
        #lines = f.read()
        if len(lines) >= to_read or pos == 0:
            return lines[-to_read:offset and -offset or None],  len(lines) > to_read or pos > 0
        avg_line_length *= 1.3
   
##########################################################################################
def tail( f, window=20 ):
    #BUFSIZ = 1024
    BUFSIZ = 2048
    f.seek(0, 2)
    bytes = f.tell()
    size = window
    block = -1
    data = []
    while size > 0 and bytes > 0:
        if (bytes - BUFSIZ > 0):
            # Seek back one whole BUFSIZ
            f.seek(block*BUFSIZ, 2)
            # read BUFFER
            data.append(f.read(BUFSIZ))
        else:
            # file too small, start from begining
            f.seek(0,0)
            # only read what was not read
            # kwright commented out the next line trying to fix a bug
            #data.append(f.read(bytes))
        linesFound = data[-1].count('\n')
        size -= linesFound
        bytes -= BUFSIZ
        block -= 1
    return '\n'.join(''.join(data).splitlines()[-window:])


##########################################################################################
def runProcess(exe):    
    #print "runProcess"
    p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    while(True):
        retcode = p.poll() #returns None while subprocess is running
        line = p.stdout.readline()
        yield line
        if(retcode is not None):
            break

##########################################################################################
def run(command):
    #if options.verbose:
    #    print command
    rv, output = commands.getstatusoutput(command)
    if rv:
        #raise RuntimeError, "error running %s, %s" % (command, rv)
        pass
    return output
    
##########################################################################################
def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mailTo):
    dashes = "+" + 70 * "-" + "+"
    pounds = 70 * "#"

    def print_debug(msg):
        if options_debug == True:
            print "DEBUG: %s" % (msg)

    print_debug("package %s" % package)
    print_debug("mock_cfg %s" % mock_cfg)
    print_debug("group %s" % group)
    if len(gitrev) > 0:
        gitrev = "git" + gitrev
        print_debug("gitrev %s" % gitrev)
    print_debug("buildDir %s" % buildDir)
    #if os.path.isdir(os.path.join(buildDir, package, ".git")):
    #if os.path.isdir(resultsDir):
    buildLogs = os.path.join(buildDir,"build_logs", mock_cfg)
    print_debug("buildLogs: %s" % buildLogs)
    if not os.path.exists(buildLogs):
        os.makedirs(buildLogs)
    os.chdir(buildLogs)
    
    print_debug("cwd %s" % os.getcwd())
    print pounds
    print "INFO: build_package() -> building %s for %s" % (package, mock_cfg)
    print pounds


    SRPMS = buildDir + "/packages/SRPMS/" 
    if not os.path.exists(SRPMS):
        os.makedirs(SRPMS)
    os.chdir(SRPMS)
    print_debug("cwd %s" % os.getcwd())
    srpmsdir = os.listdir(SRPMS)
    srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package
    print_debug("srpmsdir  %s" % srpmsdir)
    print_debug("package = %s" %  package)
    print_debug("SRPMS  = %s" %  SRPMS)
    print_debug("srpmPattern  = %s" %  srpmPattern)
    SRPM=None
    for fileName in os.listdir(SRPMS):
        if re.search(srpmPattern, fileName):
            print_debug("fileName %s" % fileName)
            SRPM = fileName
  
    if SRPM == None:
        message = "Subject: %s ERROR: unable to build %s .\n\nThe src.rpm file for the package %s is not available. " % (group, package,  package)
        email(message, mailTo)
        failure_message="ERROR: The src.rpm file for %s is not available!" % package
        print_debug(failure_message)
        print "INFO: %s " % dashes
        print os.listdir(SRPMS)
        print "INFO: %s " % dashes
        #message = "Subject: %s The package %s build for %s failed\n\n%s" % (group, package, mock_cfg,  failure_message )
        #email(message, mailTo)
        status="FAILED"
        return(status)
    
    else:
        import datetime
        now = datetime.datetime.now()
        YEAR = now.strftime("%Y")
        str(YEAR)
        workbase =  os.environ['HOME']
        scriptsDir = workbase + "/scripts"
        
        
        if not re.search(YEAR, SRPM):
            print "INFO: we need to redo this src.rpm file."
            os.chdir(SRPMS)
            print_debug("SRPM = %s, gitrev = %s" % (SRPM, gitrev))
            os.listdir(SRPMS)
            script = "/srpm_timestamp.sh"
            scriptPath = scriptsDir + script
            
            #for line in runProcess(['/home/kwright/scripts/srpm_timestamp.sh', SRPM, gitrev]):
            for line in runProcess([scriptPath, SRPM, gitrev]):
                print line,
            os.remove(SRPM)
            for fileName in os.listdir(SRPMS):
                if re.search(srpmPattern, fileName):
                    print_debug("fileName %s" % fileName)
                    SRPM = fileName
        else:
            print "INFO: This srpm has already been converted.\n"
        
        #resultsDir= '/var/lib/mock/' + mock_cfg + '/result'
        resultsDir= os.path.join('/var/lib/mock', mock_cfg, 'result')
        print "%s\nINFO: cleaning up anything in %s\n%s\n" % (dashes, resultsDir, dashes)
        if os.path.isdir(resultsDir):
            fileList = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ]
            print_debug("\nfileList %s" % fileList)
            os.chdir(resultsDir)
            for f in fileList:
                print_debug("deleting %s" % f)
                os.remove(f)
                
        os.chdir(SRPMS)
        print_debug("cwd %s" % os.getcwd())
        print pounds
        print "INFO: doing the mock build now of %s on %s" % (SRPM, mock_cfg)
        print pounds
        #logFilePath = buildLogs +  "/build_" + package + ".log"
        #buildLogs = os.path.join(buildDir,"build_logs", mock_cfg)
        logFilePath = os.path.join(buildLogs, package + ".log")
        logFile = open(logFilePath, "w") 
        for line in runProcess(['mock', '-v', '-r', mock_cfg, '--rebuild', SRPM]):
            logFile.write(line,)
            print line,
        logFile.write("\n")
        logFile.close()

        RPM=None
        fileName=None
        rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg)
        RPMList = []
        builtRPMS=""
        srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package
        if not os.path.exists(rpmsDir):
            os.makedirs(rpmsDir)
            print "INFO: Creating dir %s" % rpmsDir
        filelist = [ f for f in os.listdir(resultsDir) if f.endswith(".rpm") ]
        print "\n\n"
        print_debug("filelist = %s" % filelist)
        for RPM in filelist:
            builtRPMS += RPM + "\n"
            pathname = os.path.join(resultsDir, RPM)
            if options_debug == True:
                print_debug("copying:%s to %s" % (pathname, rpmsDir))
            shutil.copy2(pathname, rpmsDir)  
                
        mockBuildLogs = os.path.join(buildDir,"build_logs", mock_cfg, package)
        if not os.path.exists(mockBuildLogs):
             print "INFO: Creating dir %s" % mockBuildLogs
             os.makedirs(mockBuildLogs)
        for basename in os.listdir(resultsDir):
             if basename.endswith('.log'):
                  pathname = os.path.join(resultsDir, basename)
                  print_debug("Copying log file %s to %s" % (pathname, mockBuildLogs))
                  if os.path.isfile(pathname):
                       shutil.copy2(pathname, mockBuildLogs)  

        if builtRPMS != "":
             print_debug("builtRPMS: %s" % builtRPMS.strip())
             message = "Subject: %s  The package %s build for %s completed successfully\n\nINFO: Here are the rpms built:\n\n%s" % (group, package, mock_cfg, builtRPMS)
             status="COMPLETED"
             print "INFO: status = ", status
             email(message, mailTo)
             return(status)
        else:
             print pounds
             print "RPM FILE NOT FOUND !" 
#TODO move the generated build logs to this commented out line below
            #not sure why I was trying to access the mock log file here. Since
            #it's not currently being used, I'm commenting it out
            #mockLogFilePath= os.path.join(buildLogs, package,  "build.log")

             logTail = ""
             print pounds
             logTail=subprocess.Popen([r"tail","-n 40", logFilePath], stdout=subprocess.PIPE).communicate()[0]
             print logTail
             print pounds

             message = "Subject: %s The package %s build for %s failed\n\nINFO: Here are the last lines from the log file:\n%s\n%s\n" % (group, package, mock_cfg, dashes,  logTail )
             email(message, mailTo)
             status="FAILED"
             print "INFO: status = ", status
             return(status)
#end build_package()

##########################################################################################
def build_pki(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo):
    def print_debug(msg):
        if options_debug == True:
            print "DEBUG: %s" % (msg)
    print "INFO: Building package %s in %s for %s: %s" % (package, buildDir, mock_cfg, flavor)
    if not os.path.exists(buildDir):
        os.makedirs(buildDir)
    os.chdir(buildDir)
    pkiScriptsDir = buildDir + "/pki/scripts"
    if not os.path.exists(pkiScriptsDir):
        print "INFO: Getting the sources for %s" % package
        git.Git().clone("git://git.fedorahosted.org/git/pki.git")
    #else:
    pkiGitDir = buildDir + "/pki"
    os.chdir(pkiGitDir)
    g  = git.Git(pkiGitDir) 
    print g.log('--pretty=%h','-1','--').split('\n')
    gitrev = g.log('--pretty=%h','-1','--').split('\n')[0]
    print_debug(gitrev)
    print_debug(type(gitrev))
    #[gitrev, stuff1, stuff2] = g.log('--pretty=%h','-1','--').split('\n') 
    #gitrev2=gitrev[0]
    #print type(gitrev2)
    #gitrev=gitrev2
    #sys.exit()
    print "INFO: git rev = %s" % gitrev

    srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package
    SRPMS = buildDir + "/packages/SRPMS/" 
    if not os.path.exists(SRPMS):
        os.makedirs(SRPMS)
    os.chdir(SRPMS)
    print "INFO: List of src.rpms:\n"
    os.listdir(SRPMS)
    SRPM=None
    for fileName in os.listdir(SRPMS):
        if re.search(srpmPattern, fileName):
            print fileName
            SRPM = fileName
    if SRPM == None:
        if package == "dogtag-pki": 
            srpmScript = scriptsDir + "/dogtag/build_" + package + "_srpm.sh"
            print_debug("srpmScript: %s" % srpmScript)
            shutil.copy(srpmScript, buildDir)
            #cd ${BuildDir} && bash -x ./build_${PKG}_srpm.sh 2>&1 |tee ${BuildDir}/build_$PKG.srpm.log 
#TODO add logging to a file
            os.chdir(buildDir)
            for line in runProcess([srpmScript]):
                print line,
        else:
            print "INFO: package name to convert = %s" % package
            under = re.compile('-')
            srpmScript = under.sub('_', package)
            #print srpmScript #TODO add the -x for DEBUG
            #cd ${BuildDir}/pki/scripts/ && bash -x ${BuildDir}/pki/scripts/compose_${PKG_SCRIPT}_packages hybrid_srpm
            os.chdir(pkiScriptsDir)
            srpmScript = pkiScriptsDir + "/compose_" + srpmScript + "_packages"
            print srpmScript
            for line in runProcess([srpmScript, 'hybrid_srpm']):
                print line,
    status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo)
    print "Build status = %s for %s " %  (status, package)
    return status
#end build_pki()

##########################################################################################
def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo):
    def print_debug(msg):
        if options_debug == True:
            print "DEBUG: %s" % (msg)
    print "package = %s " % package
    #if not os.path.isdir(os.path.join(buildDir,package)):
    #    print "INFO: Creating directory %s " % os.path.join(buildDir,package)
    #    os.mkdir(os.path.join(buildDir,package))
    print "+-----------------------------------------------------+"
    print "Starting to build %s" % package 
    print "+-----------------------------------------------------+"
    srpmPattern = r'^%s\-\d+?\S*?\.*?\.rpm$' % package
    #SRPMS = buildDir + "/packages/SRPMS/"
    SRPMS = os.path.join(buildDir,"packages/SRPMS")
    print "SRPMS = %s" % SRPMS
    if not os.path.exists(SRPMS):
        os.makedirs(SRPMS)
    os.chdir(SRPMS)
    print "INFO: List of src.rpms:"
    os.listdir(SRPMS)
    SRPM=None
    for fileName in os.listdir(SRPMS):
        if re.search(srpmPattern, fileName):
            print fileName
            SRPM = fileName
    if SRPM == None:
        srpmScript = scriptsDir + "/build_" + package + ".sh"
        print_debug("srpmScript = %s" % srpmScript)
        shutil.copy(srpmScript, buildDir)
        os.chdir(buildDir)
        print os.getcwd()
        srpmScript = buildDir + "/build_" + package + ".sh"
        print "srpmScript = %s" % srpmScript
        for line in runProcess([srpmScript]):
            print line,

    #check for exitence of .git directory
    if os.path.isdir(os.path.join(buildDir, package, ".git")):
        print ".git dir found %s" % os.path.join(buildDir, package, ".git")
        os.chdir(os.path.join(buildDir, package))
        print os.path.join(buildDir, package)
        g  = git.Git(os.path.join(buildDir,package))
        gitrev = g.log('--pretty=%h','-1','--').split('\n')
        gitrev2=gitrev[0]
        gitrev=gitrev2
        print "INFO: git rev = %s" % gitrev
    else:
        print ".git dir not found %s" % os.path.join(buildDir,package, ".git")
        gitrev =""
        #NOTE: the ipa package already has the gitrev embedded in the SRPM and doesn't need to be converted

    status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo)
    print "Build status = %s for %s " %  (status, package)
    return status
#end build_generic_pkg()



#TODO some of these 3 todo's are duplicates of those down at the symlink area
#TODO add checking to see if the source checked out okay and that the pki/scripts dir exists
#TODO put some code in here to verify that all the packages build succeded.  if not, then we shouldn't even create the directories

##########################################################################################
def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, mailTo):
    """ function for the repo directories
    Currently this is called only when the build fails.

    """
    def print_debug(msg):
        if options_debug == True:
            print "DEBUG: %s" % (msg)
    print_debug("function: createRepoDir")
    print_debug("mock_cfg: %s" % mock_cfg)
    print_debug("flavor: %s" % flavor)
    print_debug("DT: %s" % DT)
    print_debug("buildDir: %s" % buildDir)
    print_debug("options_test: %s" % options_test)

    workbase =  os.environ['HOME']
    scriptsDir = workbase + "/scripts"

    dashes = "+" + 70 * "-" + "+"
    longDashes = "+" + 80 * "-" + "+"
    URL="http://mickey.dsdev.sjc.redhat.com"
    print_debug("URL: %s" % URL)
#TODO find a better name than repoPackage
    repoPackage="pki"
    print_debug("repoPackage: %s" % repoPackage)

    [platform, rel, arch] = mock_cfg.split("-")

    if arch  == "i386": 
        arch = "i686"
        print_debug("arch: %s" % arch)

    if platform == "fedora":
        dist="F" + rel
        print_debug("rel: %s" % rel)
        
    if platform == "rhel": 
        dist=platform + rel 
        print_debug("dist: %s" % dist)

#TODO send an email the the build faild due to not being able to mount repos
    if options_test: 
        repos = "/tmp/repos"
    else:
        repos = "/repos"
        if not os.path.ismount(repos): 
            print "WARN: %s is not mounted" % repos
            print "INFO: attempting to mount %s ... " % repos
            for line in runProcess(["sudo", "/bin/mount", repos]):
                print line
        if not os.path.ismount(repos): 
            print "ERROR: unable to mount %s " % repos
            sys.exit()

    #TODO: handle other flavor names for future builds
    if flavor == "DOGTAG 10":
        #/repos/pki/dogtag/10
        [flavorName, flavorGroup] = flavor.split()
        flavorName=flavorName.lower()
        repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
        repoFull=os.path.join(repoBase, dist)
    else:
        flavorName="IPA"
        flavorGroup="devel"
        flavorName=flavorName.lower()
        repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
        repoFull=os.path.join(repoBase, dist)
        print_debug("repoBase: %s" % repoBase)
        print_debug("repoFull: %s" % repoFull)


    #check for case when a new dist is released and create the dir
    #################################################################
    #create the repo tree 
    #################################################################
    if os.path.exists(repoBase) and not os.path.exists(repoFull):
        print "INFO: Creating dir", repoFull
        os.makedirs(repoFull)
    elif not os.path.exists(repoBase):
        os.makedirs(repoBase)
        #TODO add some logic here to exit if unable to create the directory
        #print "ERROR: %s doesn't exist" % repoBase
        #sys.exit(2)
    if not os.path.exists(os.path.join(repoFull, DT)):
        print "INFO: Creating dir %s" % os.path.join(repoFull, DT)
        os.makedirs(os.path.join(repoFull, DT))
    for dirType in (arch, "noarch", "src"):
        if not os.path.exists(os.path.join(repoFull, DT, dirType)):
            print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType)
            os.makedirs(os.path.join(repoFull, DT, dirType))
    if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")):
        print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs")
        os.makedirs(os.path.join(repoFull, DT, arch, "logs"))

#def copyBuildLogs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo):
    """still need a function for  over the build logs only. in the case that one of the builds fails"""
    sourceLogDir = os.path.join(buildDir,"build_logs", mock_cfg)
    #repoBuildLogsDir = os.path.join(repoFull, DT, arch, "logs")
    targetLogDir = os.path.join(repoFull, DT, arch, "logs", "build_logs", mock_cfg)
    #destDir = os.path.join(repoFull, DT, rpmType)
    #print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir,  dashes))
    #shutil.copy2(os.path.join(rpmsDir, RPM), destDir)
    copyResult = distutils.dir_util.copy_tree(sourceLogDir, targetLogDir, preserve_mode=0, verbose=1)
    print_debug("copyResult %s" %  copyResult)
    #print buildLogs
    URL="http://mickey.dsdev.sjc.redhat.com"
    #import json
    test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".")
    #message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch")
    #print_debug("path =%s" % message_url_path)

    failMessage= """
You can find the build logs here:
%s
  %s%s 
  %s%s 
%s
%s
""" % (
          longDashes,
          URL, os.path.join(URL, repoFull, DT, arch),
          URL, os.path.join(URL, repoFull, DT, "noarch"),
          longDashes,
          test_message,
          )
    buildStatus="FAILED"
    message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, failMessage )
    print(failMessage)
    email(message, mailTo)
#end createRepoDirs
##########################################################################################

def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo):
    def print_debug(msg):
        if options_debug == True:
            print "DEBUG: %s" % (msg)

    print_debug("mock_cfg: %s" % mock_cfg)
    print_debug("flavor: %s" % flavor)
    print_debug("DT: %s" % DT)
    print_debug("buildDir: %s" % buildDir)
    print_debug("options_test: %s" % options_test)

    workbase =  os.environ['HOME']
    scriptsDir = workbase + "/scripts"

    dashes = "+" + 70 * "-" + "+"
    longDashes = "+" + 80 * "-" + "+"
    URL="http://mickey.dsdev.sjc.redhat.com"
    print_debug("URL: %s" % URL)
#TODO find a better name than repoPackage
    repoPackage="pki"
    print_debug("repoPackage: %s" % repoPackage)

    [platform, rel, arch] = mock_cfg.split("-")

    if arch  == "i386": 
        arch = "i686"
        print_debug("arch: %s" % arch)

    if platform == "fedora":
        dist="F" + rel
        print_debug("rel: %s" % rel)
        
    if platform == "rhel": 
        dist=platform + rel 
        print_debug("dist: %s" % dist)

#TODO send an email the the build faild due to not being able to mount repos
    if options_test: 
        repos = "/tmp/repos"
    else:
        repos = "/repos"
        if not os.path.ismount(repos): 
            print "WARN: %s is not mounted" % repos
            print "INFO: attempting to mount %s ... " % repos
            for line in runProcess(["sudo", "/bin/mount", repos]):
                print line
        if not os.path.ismount(repos): 
            print "ERROR: unable to mount %s " % repos
            sys.exit()

    #TODO: handle other flavor names for future builds
    if flavor == "DOGTAG 10":
        [flavorName, flavorGroup] = flavor.split()
        flavorName=flavorName.lower()
        repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
        repoFull=os.path.join(repoBase, dist)
    else:
        flavorName="IPA"
        flavorGroup="devel"
        flavorName=flavorName.lower()
        repoBase=os.path.join(repos, repoPackage, flavorName, flavorGroup)
        repoFull=os.path.join(repoBase, dist)
        print_debug("repoBase: %s" % repoBase)
        print_debug("repoFull: %s" % repoFull)
        if not options_skipWget:
            #COPY over the download_builds.sh script
            downloadScript=os.path.join(scriptsDir, flavorName, "download_builds.sh")
            print_debug("downloadScript = %s" % downloadScript) 
            shutil.copy(downloadScript, buildDir)
            os.chdir(buildDir)
            downloadScript=os.path.join(buildDir, "download_builds.sh")
            print_debug("downloadScript = %s" % downloadScript)
            for line in runProcess([downloadScript, arch]):
                print line,


    #check for case when a new dist is released and create the dir
    #################################################################
    #create the repo tree 
    #################################################################
    if os.path.exists(repoBase) and not os.path.exists(repoFull):
        print "INFO: Creating dir", repoFull
        os.makedirs(repoFull)
    elif not os.path.exists(repoBase):
        os.makedirs(repoBase)
        #print "ERROR: %s doesn't exist" % repoBase
        #sys.exit(2)
#TODO add some logic here to exit if unable to create the directory
    if not os.path.exists(os.path.join(repoFull, DT)):
        print "INFO: Creating dir %s" % os.path.join(repoFull, DT)
        os.makedirs(os.path.join(repoFull, DT))
    for dirType in (arch, "noarch", "src"):
        if not os.path.exists(os.path.join(repoFull, DT, dirType)):
            print "INFO: Creating dir %s" % os.path.join(repoFull, DT, dirType)
            os.makedirs(os.path.join(repoFull, DT, dirType))
    if not os.path.exists(os.path.join(repoFull, DT, arch, "logs")):
        print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs")
        os.makedirs(os.path.join(repoFull, DT, arch, "logs"))
    buildLogs = os.path.join(repoFull, DT, arch, "logs")
    print buildLogs
        
    os.chdir(repoFull)
#TODO add some logic that if any of the builds fail,
#Currently, if the build fails, I skip the entire creation of the repos--including the build logs
# 1) do not create the symlink - DONE
# 2) copy the build logs over - DONE
# 3) do not copy the builds -DONE
# 4) send an email with a url to the build logs - DONE (in other function)

    archSymlink = "devel_" + arch

    if os.path.exists("devel_" + arch):
        os.remove("devel_" + arch)
    print os.getcwd()
    print "creating symlink %s_devel to %s " % (arch, DT)
    os.symlink(DT, "devel_" + arch)
    
    buildStatus="COMPLETED"

    #################################################################
    #copy the builds over to the newly created repo tree
    #################################################################
#TODO add some logic for the case where the only build is a noarch and there are no arch build (dogtag-pki)
#currently, this creates an empty log file for fileList_i686
#  /tmp/repos/pki/dogtag/10/F17/20120913-150042/i686/logs/fileList_i686

    rpmsDir = os.path.join(buildDir,  "packages/RPMS/" + mock_cfg)
    print dashes

    for rpmType in (arch, "noarch", "src"):
        filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ]
        filelist.sort()
        logFile = open(buildLogs + "/fileList_" + rpmType , "w") 
        destDir = os.path.join(repoFull, DT, rpmType)
        for RPM in filelist:
            print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir,  dashes))
            shutil.copy2(os.path.join(rpmsDir, RPM), destDir)
            logFile.write(RPM+"\n")
        logFile.close()
#handle i386 rpm files that are part of the i686 repo
    rpmType="i386"
    filelist = [ f for f in os.listdir(rpmsDir) if f.endswith(rpmType + ".rpm") ]
    filelist.sort()
    logFile = open(buildLogs + "/fileList_" + "i686" , "a") 
    destDir = os.path.join(repoFull, DT, "i686")
    for RPM in filelist:
        print_debug("copying:(source: dest:)\n%s/%s\n%s\n%s" % (rpmsDir, RPM, destDir,  dashes))
        shutil.copy2(os.path.join(rpmsDir, RPM), destDir)
        logFile.write(RPM+"\n")
    logFile.close()

    #sort the file into a temp file then copy it back. this is needed for the email with the list of files
    logFile = os.path.join(buildLogs, "fileList_" + "i686") 
    logFileTmp = os.path.join(buildLogs,  "fileList_" + "i686.out")
    inLogFile = open(logFile , "r")
    outLogFile = open(logFileTmp , "w")
    map(outLogFile.write, sorted(inLogFile.readlines()))
    inLogFile.close()
    outLogFile.close()
    shutil.copy2(logFileTmp, logFile)
    os.remove(logFileTmp)

    


#when I need to create a repo:
#add logic to not do this if build failed
    for rpmType in (arch, "noarch", "src"):
        destDir = os.path.join(repoFull, DT, rpmType)
        logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "w") 
        print_debug("logFile = %s" % logFile)
        cmd = "createrepo -vv --no-database --skip-stat " + destDir
        output=run(cmd)
        print output
        logFile.write(output+"\n")
        logFile.close()
    numPkgDict={}
    for rpmType in (arch, "noarch", "src"):
        destDir = os.path.join(repoFull, DT, rpmType)
        logFile = open(buildLogs + "/createrepo_" + rpmType + ".log", "r") 
        for line in logFile:
            if "Spawning" in line:
                #print line
                pkgNum=line.split()[-2]
                print_debug("pkgNum: %s" % pkgNum)
                numPkgDict[rpmType]=pkgNum

    print_debug("numPkgDict: %s" % numPkgDict)


#TODO copy the log files over to the build_logs directory
    rpmInRepoDict={}
    for rpmType in (arch, "noarch", "src"): 
        rpmInRepoDict[rpmType] =  open(buildLogs + "/fileList_" + rpmType, "r").read().split()
        #" ".join(REPOLOG)
        #print REPOLOG
        #close(buildLogs + "/fileList_" + rpmType)

    import json

    #TODO add some logic here to check for test mode and alter the email accordingly
    #NOTE this is partially implemented below:
    test_message = "{test_build}".format(test_build="****this is a TEST BUILD****" if options_test else ".")
    message_url_path = "{url_path}".format(url_path=os.path.join(URL, repoFull, DT, arch)) if options_test else URL + os.path.join(URL, repoFull, DT, "noarch")
    print_debug("path =%s" % message_url_path)

    releaseMessage= """
You can find the build repos here:
%s
  %s%s 
  %s%s 

  %s%s 
  %s%s 

Summary of the number of RPMS that were built:
%s
%s

Here are the RPMs that were built:
%s
%s
%s
    """ % (
          longDashes,
          URL, os.path.join(URL, repoFull, DT, arch), 
          URL, os.path.join(URL, repoFull, DT, "noarch"),
          URL, os.path.join(URL, repoFull, "devel_" + arch, arch ),
          URL, os.path.join(URL, repoFull, "devel_" + arch, "noarch"),
          longDashes,
          json.dumps(numPkgDict, indent=4),
          longDashes,
          json.dumps(rpmInRepoDict, indent=4),
          test_message,
          )
    print releaseMessage

    message = "Subject: %s %s build %s \n\n\n%s\n" % (flavor, mock_cfg, buildStatus, releaseMessage )

    email(message, mailTo)
    
#end copyToRepo