Add support for also building client packages

5 files changed, 148 insertions, 147 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..53c8bba
--- /dev/null
+++ b/README
@@ -0,0 +1,7 @@
+Python scripts do do local mock builds of nss packages and optionally cleinent packages of nss.
+
+Example:
+python ./build-nss.py -d -t -r rawhide -a i386 -sc -sw -c evolution-data-server
+
+will build nspr, nss-util, nss-softoken, and nss by default and also the evolution-data-server
+
diff --git a/build-nss.py b/build-nss.py
index 7e7a9ae..5cb463d 100755
--- a/build-nss.py
+++ b/build-nss.py
@@ -1,4 +1,5 @@
-#!/usr/bin/python 
+#!/usr/bin/python
+ 
 import sys
 import os
 import argparse
@@ -7,111 +8,88 @@ import build_functions
 print os.environ.get("USER")
 #######################################################
 
-parser = argparse.ArgumentParser(description='Program to automate nss builds using mock.', 
+parser = argparse.ArgumentParser(
+    description='Automate NSS builds using mock.', 
     version = '0.9 beta', 
     add_help=True, 
     conflict_handler='resolve', 
-    epilog='This program is designed to be used both on the command line or by cron') 
-
-#parser = argparse.ArgumentParser(description='Program to automate nss builds using mock.', version = '0.9 beta')#, add_help=True)
+    epilog='Program designed to be used both on the command line or by cron') 
 
 
-parser.add_argument("-d", 
-                    "--debug", 
-                    action="store_true", 
-                    default=True, 
+parser.add_argument("-d", "--debug", 
+                    action="store_true", default=True, 
                     help="show debug output")
 
-parser.add_argument("-t", 
-                    "--test", 
-                    action="store_true", 
-                    default=False,
+
+parser.add_argument("-t", "--test", 
+                    action="store_true", default=False,
                     help="test mode (for debugging)")
-parser.add_argument(
-                    '-f', 
-                    '--flavor', 
-                    action="store", 
-                    dest="flavor", 
-                    default='nss', 
+
+parser.add_argument('-f', '--flavor', 
+                    action="store", dest="flavor", default='nss', 
                     help='specify the flavor of the build. [default=nss]')
-#parser.add_argument(
-#                    '-p',
+
+#parser.add_argument('-p',
 #                    '--platform', 
 #                    action="store", 
 #                    dest="platform", 
 #                    default=None, 
 #                    help='specify a platform to use for mock config.  [default=None] NOT IMPLEMENTED')
-parser.add_argument(
-                    '-r', 
-                    '--rel',
-                    action="store", 
-                    dest="rel", 
-                    default=None,
-                    choices=("rawhide", "f18" ),
+
+parser.add_argument('-r', '--rel',
+                    action="store", dest="rel", 
+                    default=None, choices=("rawhide", "f18" ),
                     help='specify a release to use for mock config.  [default=None]')
-parser.add_argument(
-                    '-a', 
-                    '--arch', 
-                    action="store", 
-                    dest="arch", 
-                    default=None,
-                    choices=("i386", "x86_64"),
+
+parser.add_argument('-a', '--arch', 
+                    action="store", dest="arch", 
+                    default=None,  choices=("i386", "x86_64"),
                     help='specify an arch to use for mock config.  [default=None]')
-parser.add_argument(
-                    '-l', 
-                    '--local', 
-                    action="store_true", 
-                    dest="localBuild", 
-                    default=False, 
+
+parser.add_argument('-l', '--local', 
+                    action="store_true", dest="localBuild", 
+                    default=True, 
                     help='perform a build in the current working diretory. [default=False]')
-parser.add_argument(
-                    '-mm', 
-                    '--mailme', 
-                    action="store_true", 
-                    dest="mailMe", 
-                    #default=True,  
+
+parser.add_argument('-mm', '--mailme', 
+                    action="store_true", dest="mailMe", 
                     default=False,
-                    help='send email to kwright@redhat.com. [default=False')
-parser.add_argument(
-                    '-sb', 
-                    '--skipbuild', 
-                    action="store_true", 
-                    dest="skipBuild", 
+                    help='send email to emaldona@redhat.com. [default=False')
+
+parser.add_argument('-sb', '--skipbuild', 
+                    action="store_true", dest="skipBuild", 
                     default=False, 
                     help='skip the build portion of the program. [default=False] ')
-parser.add_argument(
-                    '-sc', 
-                    '--skipcopy', 
-                    action="store_true", 
-                    dest="skipCopy", 
-                    default=False, 
-                    help='skip the copy portion of the program. [default=False] ')
-parser.add_argument(
-                    '-sw', 
-                    '--skipwget', 
+
+parser.add_argument('-sc', '--skipcopy', 
+                    action="store_true", dest="skipCopy", 
+                    default=True, 
+                    help='skip the copy portion of the program. [default=True] ')
+
+parser.add_argument('-sw', '--skipwget', 
                     action="store_true", 
                     dest="skipWget", 
-                    default=False, 
-                    help='skip the wget portion of the program. [default=False] ')
-parser.add_argument(
-                    '-b', 
-                    '--build', 
-                    action='store',
-                    dest='nssPackageList',
-                    nargs='+',
-                   #default=['nspr', 'nss-util', 'nss-softokn', 'nss' ],
+                    default=True, 
+                    help='skip the wget portion of the program. [default=True] ')
+
+parser.add_argument('-b', '--build', 
+                    action='store', dest='nssPackageList', nargs='+',
                     default=['nspr', 'nss-util', 'nss-softokn', 'nss', 'curl', 'xulrunner' ],
                     help='specify a list of one of more builds to perform. ',
                     choices=('nspr', 'nss-util', 'nss-softokn', 'nss'))
-parser.add_argument(
-                    '-m', 
-                    '--mock_cfg', 
-                    action="store", 
-                    dest="mockCfgList", 
-                    nargs='+', 
+
+parser.add_argument('-c', '--clients', 
+                    action='store', dest='clientPackageList', nargs='+',
+                    default=[ 'xulrunner' ],
+                    help='specify a list of one of more builds to perform. ',
+                    choices=('curl', 'evolution-data-server', 'mod_nss', 'mod_revocator', 'openswan', 'xulrunner'))
+
+parser.add_argument('-m', '--mock_cfg', 
+                    action="store", dest="mockCfgList", nargs='+', 
                     default=["fedora-rawhide-i386", "fedora-rawhide-x86_64"],
                     help='specify a list of one or more mock configs to use.  [default = fedora-rawhide-i386", "fedora-rawhide-x86_64]',
                     choices=("fedora-rawhide-i386", "fedora-rawhide-x86_64"))
+
 options  = parser.parse_args()
 #if args.servername is None and not args.dry_run:
 
@@ -150,6 +128,7 @@ print_debug("skipBuild = %s " % options.skipBuild)
 print_debug("skipCopy = %s " % options.skipCopy)
 print_debug("skipWget = %s " % options.skipWget)
 print_debug("nssPackageList = %s " % options.nssPackageList)
+print_debug("clientPackageList = %s " % options.clientPackageList)
 print_debug("mockCfgList = %s " % options.mockCfgList)
 #######################################################
 mockCfgList=[]
@@ -206,6 +185,7 @@ else:
 print_debug("buildDir = %s" % buildDir)
 
 nssPackageList = options.nssPackageList
+clientPackageList = options.clientPackageList
 flavor = options.flavor
 #######################################################
 progname = os.path.basename(__file__)
@@ -231,10 +211,6 @@ buildStatusDict={}
 if options.skipBuild is False:
     for mock_cfg in mockCfgList:
         [platform, rel, arch] = mock_cfg.split("-")
-        if rel == "18":
-            nssPackageList.append("xulrunner")
-            print_debug("Adding xulrunner to list of packages to build for %s %s" % (platform, rel))
-            print_debug("nssPackageList = %s" % nssPackageList)
         for package in nssPackageList:
             status = build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options.debug, mailTo)
             buildStatusDict[(package,mock_cfg)]=status
@@ -244,7 +220,11 @@ if options.skipBuild is False:
 ####################
             print_debug("package %s completed with status %s for %s" % (package, status, mock_cfg))
 
-
+    if len(clientPackageList) > 0:
+        for client in clientPackageList:
+            status = build_generic_pkg(client, buildDir, mock_cfg, flavor, scriptsDir, options.debug, mailTo)
+            print_debug("%s" % json.dumps(str(buildStatusDict), sort_keys=True, indent=4))
+            print_debug("package %s completed with status %s for %s" % (package, status, mock_cfg))
 
 # TODO fix issue with failed build not stopping creating the repo
 # NOTE the problem appears to be if the x86_64 bit builds all complete
@@ -257,11 +237,12 @@ print_debug("%s" % json.dumps(str(buildStatusDict), sort_keys=True, indent=4))
 #This way, I can easily tell if anything failed.
 
 ################################################################################
-#NOTE: handle whether or not to copy the files over and create the rpm repo. 
-#The logic is as follows: 
-#if one of the packages failed to build, skipCopy is enalbed, don't copy
-#if the commandline option skipCopy is enabled, don't copy
-#else skipCopy is disabled and the files get copied then the yum repos are created.
+# NOTE: handle whether or not to copy the files over and create the rpm repo. 
+# The logic is as follows: 
+# if any package fails to build, skipCopy gets enabled, don't copy
+# if the command line option skipCopy is enabled, don't copy
+# else if either skipCopy is disabled and the files get copied then the
+# yum repos are created.
 ################################################################################
 skipCopy=False
 for mock_cfg in mockCfgList:
@@ -281,7 +262,7 @@ for mock_cfg in mockCfgList:
         for (package, status) in buildStatusDict:
             status_message += "build status of package: %s on %s: %s\n" % (package, status, buildStatusDict[package,mock_cfg])
         message = "Subject: ERROR: %s skipping copying to and/or creating the repo.\n\nERROR: The repo was not created due to a failure of one or more packages.\n\nHere's the list of builds and their status:\n\n%s\n" % (flavor, status_message)
-#TODO add code to copy the log files over somewhere when the build failes.
+#TODO add code to copy the log files over somewhere when the build fails.
         createRepoDirs(mock_cfg, flavor, DT, buildDir, options.test, options.debug, mailTo)
 
         print_debug("%s" %  message)
@@ -289,7 +270,8 @@ for mock_cfg in mockCfgList:
     elif skipCopy is False:
         #for mock_cfg in mockCfgList:
         print_debug("copy the files to the repo for %s" % mock_cfg)
-        copyToRepo(mock_cfg, flavor, DT, buildDir, options.test, options.debug, options.skipWget,  mailTo)
+        copyToRepo(mock_cfg, flavor, DT, buildDir, 
+                   options.test, options.debug, options.skipWget, mailTo)
 
 sys.exit()
 
diff --git a/build_functions.py b/build_functions.py
index 66efc57..d746957 100755
--- a/build_functions.py
+++ b/build_functions.py
@@ -16,8 +16,8 @@ import distutils.dir_util
 def email(message, mailTo):  
     import smtplib
     import socket
-    mail_server='smtp.corp.redhat.com'
-    mail_server_port=25
+    # mail_server='smtp.corp.redhat.com'
+    # mail_server_port=25
 
     receivers=mailTo
 
@@ -77,25 +77,25 @@ def tail( f, window=20 ):
     #BUFSIZ = 1024
     BUFSIZ = 2048
     f.seek(0, 2)
-    bytes = f.tell()
+    byteCount = f.tell()
     size = window
     block = -1
     data = []
-    while size > 0 and bytes > 0:
-        if (bytes - BUFSIZ > 0):
+    while size > 0 and byteCount > 0:
+        if (byteCount - BUFSIZ > 0):
             # Seek back one whole BUFSIZ
             f.seek(block*BUFSIZ, 2)
             # read BUFFER
             data.append(f.read(BUFSIZ))
         else:
-            # file too small, start from begining
+            # file too small, start from beginning
             f.seek(0,0)
             # only read what was not read
             # kwright commented out the next line trying to fix a bug
-            #data.append(f.read(bytes))
+            #data.append(f.read(byteCount))
         linesFound = data[-1].count('\n')
         size -= linesFound
-        bytes -= BUFSIZ
+        byteCount -= BUFSIZ
         block -= 1
     return '\n'.join(''.join(data).splitlines()[-window:])
 
@@ -122,12 +122,12 @@ def run(command):
     return output
     
 ##########################################################################################
-def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mailTo):
+def build_package(package, mock_cfg, gitrev, buildDir, group, optionsDebug, mailTo):
     dashes = "+" + 70 * "-" + "+"
     pounds = 70 * "#"
 
     def print_debug(msg):
-        if options_debug == True:
+        if optionsDebug == True:
             print "DEBUG: %s" % (msg)
 
     print_debug("package %s" % package)
@@ -186,8 +186,8 @@ def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mai
         now = datetime.datetime.now()
         YEAR = now.strftime("%Y")
         str(YEAR)
-        workbase =  os.environ['HOME']
-        scriptsDir = workbase + "/scripts"
+#        workbase =  os.environ['HOME']
+#        scriptsDir = workbase + "/scripts"
         
         
 #        if not re.search(YEAR, SRPM):
@@ -198,8 +198,8 @@ def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mai
 #            script = "/srpm_timestamp.sh"
 #            scriptPath = scriptsDir + script
             
-     #for line in runProcess(['/home/kwright/scripts/srpm_timestamp.sh', SRPM, gitrev]):
-#            for line in runProcess([scriptPath, SRPM, gitrev]):
+#    for line in runProcess(['/home/kwright/scripts/srpm_timestamp.sh', SRPM, gitrev]):
+#         for line in runProcess([scriptPath, SRPM, gitrev]):
 #                print line,
 #            os.remove(SRPM)
 #            for fileName in os.listdir(SRPMS):
@@ -238,7 +238,7 @@ def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mai
         RPM=None
         fileName=None
         rpmsDir = os.path.join(buildDir, "packages/RPMS/" + mock_cfg)
-        RPMList = []
+        
         builtRPMS=""
         srpmPattern = r'^%s\-\d*?\S*?\.*?\.rpm$' % package
         if not os.path.exists(rpmsDir):
@@ -250,55 +250,64 @@ def build_package(package, mock_cfg, gitrev, buildDir, group, options_debug, mai
         for RPM in filelist:
             builtRPMS += RPM + "\n"
             pathname = os.path.join(resultsDir, RPM)
-            if options_debug == True:
+            if optionsDebug == True:
                 print_debug("copying:%s to %s" % (pathname, rpmsDir))
             shutil.copy2(pathname, rpmsDir)  
                 
         mockBuildLogs = os.path.join(buildDir,"build_logs", mock_cfg, package)
+        
         if not os.path.exists(mockBuildLogs):
-             print "INFO: Creating dir %s" % mockBuildLogs
-             os.makedirs(mockBuildLogs)
+            print "INFO: Creating dir %s" % mockBuildLogs
+        os.makedirs(mockBuildLogs)
+             
         for basename in os.listdir(resultsDir):
-             if basename.endswith('.log'):
-                  pathname = os.path.join(resultsDir, basename)
-                  print_debug("Copying log file %s to %s" % (pathname, mockBuildLogs))
-                  if os.path.isfile(pathname):
-                       shutil.copy2(pathname, mockBuildLogs)  
+            if basename.endswith('.log'):
+                pathname = os.path.join(resultsDir, basename)
+                print_debug("Copying log file %s to %s" % (pathname, mockBuildLogs))
+                if os.path.isfile(pathname):
+                    shutil.copy2(pathname, mockBuildLogs)  
 
         if builtRPMS != "":
-             print_debug("builtRPMS: %s" % builtRPMS.strip())
-             message = "Subject: %s  The package %s build for %s completed successfully\n\nINFO: Here are the rpms built:\n\n%s" % (group, package, mock_cfg, builtRPMS)
-             status="COMPLETED"
-             print "INFO: status = ", status
-             email(message, mailTo)
-             return(status)
+            print_debug("builtRPMS: %s" % builtRPMS.strip())
+            message = "Subject: %s  The package %s build for %s completed successfully\n\nINFO: Here are the rpms built:\n\n%s" % (group, package, mock_cfg, builtRPMS)
+            status="COMPLETED"
+            print "INFO: status = ", status
+            email(message, mailTo)
+            return(status)
         else:
-             print pounds
-             print "RPM FILE NOT FOUND !" 
+            print pounds
+            print "RPM FILE NOT FOUND !" 
 #TODO move the generated build logs to this commented out line below
             #not sure why I was trying to access the mock log file here. Since
             #it's not currently being used, I'm commenting it out
             #mockLogFilePath= os.path.join(buildLogs, package,  "build.log")
 
-             logTail = ""
-             print pounds
-             logTail=subprocess.Popen([r"tail","-n 40", logFilePath], stdout=subprocess.PIPE).communicate()[0]
-             print logTail
-             print pounds
-
-             message = "Subject: %s The package %s build for %s failed\n\nINFO: Here are the last lines from the log file:\n%s\n%s\n" % (group, package, mock_cfg, dashes,  logTail )
-             email(message, mailTo)
-             status="FAILED"
-             print "INFO: status = ", status
-             return(status)
+            logTail = ""
+            print pounds
+            logTail=subprocess.Popen([r"tail","-n 40", logFilePath], stdout=subprocess.PIPE).communicate()[0]
+            print logTail
+            print pounds
+
+            message = "Subject: %s The package %s build for %s failed\n\nINFO: Here are the last lines from the log file:\n%s\n%s\n" % (group, package, mock_cfg, dashes,  logTail )
+            email(message, mailTo)
+            status="FAILED"
+            print "INFO: status = ", status
+            return(status)
 #end build_package()
 
 ##########################################################################################
 
 ##########################################################################################
-def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_debug, mailTo):
+def build_generic_pkg(
+    package, 
+    buildDir, 
+    mock_cfg, 
+    flavor, 
+    scriptsDir, 
+    optionsDebug, 
+    mailTo):
     def print_debug(msg):
-        if options_debug == True:
+        if optionsDebug == True:
             print "DEBUG: %s" % (msg)
     print "package = %s " % package
     #if not os.path.isdir(os.path.join(buildDir,package)):
@@ -347,7 +356,7 @@ def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_d
         gitrev =""
         #NOTE: the ipa package already has the gitrev embedded in the SRPM and doesn't need to be converted
 
-    status = build_package(package, mock_cfg, gitrev, buildDir, flavor, options_debug, mailTo)
+    status = build_package(package, mock_cfg, gitrev, buildDir, flavor, optionsDebug, mailTo)
     print "Build status = %s for %s " %  (status, package)
     return status
 #end build_generic_pkg()
@@ -359,13 +368,13 @@ def build_generic_pkg(package, buildDir, mock_cfg, flavor, scriptsDir, options_d
 #TODO put some code in here to verify that all the packages build succeded.  if not, then we shouldn't even create the directories
 
 ##########################################################################################
-def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, mailTo):
+def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, optionsDebug, mailTo):
     """ function for the repo directories
     Currently this is called only when the build fails.
 
     """
     def print_debug(msg):
-        if options_debug == True:
+        if optionsDebug == True:
             print "DEBUG: %s" % (msg)
     print_debug("function: createRepoDir")
     print_debug("mock_cfg: %s" % mock_cfg)
@@ -374,10 +383,12 @@ def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug,
     print_debug("buildDir: %s" % buildDir)
     print_debug("options_test: %s" % options_test)
 
-    workbase =  os.environ['HOME']
-    scriptsDir = workbase + "/scripts"
+    #workbase =  os.environ['HOME']
 
-    dashes = "+" + 70 * "-" + "+"
+    #scriptsDir = os.environ['HOME'] + "/scripts"
+
+    #dashes = "+" + 70 * "-" + "+"
+    
     longDashes = "+" + 80 * "-" + "+"
     URL="http://pkgs.fedoraproject.org"
     print_debug("URL: %s" % URL)
@@ -453,7 +464,7 @@ def createRepoDirs(mock_cfg, flavor, DT, buildDir, options_test, options_debug,
         print "INFO: Creating dir %s" % os.path.join(repoFull, DT, arch, "logs")
         os.makedirs(os.path.join(repoFull, DT, arch, "logs"))
 
-#def copyBuildLogs(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo):
+#def copyBuildLogs(mock_cfg, flavor, DT, buildDir, options_test, optionsDebug, options_skipWget, mailTo):
     """still need a function for  over the build logs only. in the case that one of the builds fails"""
     sourceLogDir = os.path.join(buildDir,"build_logs", mock_cfg)
     #repoBuildLogsDir = os.path.join(repoFull, DT, arch, "logs")
@@ -491,9 +502,9 @@ You can find the build logs here:
 #end createRepoDirs
 ##########################################################################################
 
-def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, options_debug, options_skipWget, mailTo):
+def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, optionsDebug, options_skipWget, mailTo):
     def print_debug(msg):
-        if options_debug == True:
+        if optionsDebug == True:
             print "DEBUG: %s" % (msg)
 
     print_debug("mock_cfg: %s" % mock_cfg)
@@ -594,13 +605,14 @@ def copyToRepo(mock_cfg, flavor, DT, buildDir, options_test, options_debug, opti
         
     os.chdir(repoFull)
 #TODO add some logic that if any of the builds fail,
-#Currently, if the build fails, I skip the entire creation of the repos--including the build logs
+# Currently, if the build fails, I skip the entire creation 
+# of the repos--including the build logs
 # 1) do not create the symlink - DONE
 # 2) copy the build logs over - DONE
 # 3) do not copy the builds -DONE
 # 4) send an email with a url to the build logs - DONE (in other function)
 
-    archSymlink = "devel_" + arch
+#    archSymlink = "devel_" + arch
 
     if os.path.exists("devel_" + arch):
         os.remove("devel_" + arch)
diff --git a/shell-scripts/clean-all-packages.sh b/shell-scripts/clean-all-packages.sh
index 5a0be7b..bd8dee9 100755
--- a/shell-scripts/clean-all-packages.sh
+++ b/shell-scripts/clean-all-packages.sh
@@ -12,7 +12,7 @@ remove_pkgs() {
 #----------------------------------------------------------
 
 nssPkgs="nspr nss-util nss-softokn nss"
-clientPkgs="crypto-utils curl xulrunner"
+clientPkgs="crypto-utils curl evolution-data-server mod_nss mod_revocator nss_compat_ossl openswan xulrunner"
 
 # Defaults
 TARGET=Rawhide
diff --git a/shell-scripts/mock-build-prepare.sh b/shell-scripts/mock-build-prepare.sh
index d3b0d60..ddfa2a7 100755
--- a/shell-scripts/mock-build-prepare.sh
+++ b/shell-scripts/mock-build-prepare.sh
@@ -92,7 +92,7 @@ for pkg in $nssPkgs; do
     # mock -r fedora-${dist}-${arch} --installdeps ${pkg}-*.fc${target}.src.rpm
     mock -r fedora-${dist}-${arch} --copyin ${pkg}-*.fc${target}.src.rpm /tmp
   fi
-done-
+done
 
 # copy the build script into mock
 mock -r fedora-${dist}-${arch} --copyin ./mock-build-inside.sh /usr/bin