Make the shell functions a bit more modular for scripting

14 files changed, 417 insertions, 0 deletions

diff --git a/rebuild-scripts/CVS/Entries b/rebuild-scripts/CVS/Entries
new file mode 100644
index 0000000..d2d53df
--- /dev/null
+++ b/rebuild-scripts/CVS/Entries
@@ -0,0 +1,12 @@
+/add_owner/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/buildcheck/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/buildcheck-extras-development.repo/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/bumpspecfile.py/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/check_rebuild/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/check_rebuild2/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/check_release/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/gen_nr_2/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/gen_rebuild_lst/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/rm_noarch/1.1.1.1/Fri Oct  6 22:22:02 2006//
+/semi_auto_rebuild.txt/1.1.1.1/Fri Oct  6 22:22:02 2006//
+D
diff --git a/rebuild-scripts/CVS/Repository b/rebuild-scripts/CVS/Repository
new file mode 100644
index 0000000..2014fa1
--- /dev/null
+++ b/rebuild-scripts/CVS/Repository
@@ -0,0 +1 @@
+rebuild-scripts
diff --git a/rebuild-scripts/CVS/Root b/rebuild-scripts/CVS/Root
new file mode 100644
index 0000000..7c4b192
--- /dev/null
+++ b/rebuild-scripts/CVS/Root
@@ -0,0 +1 @@
+:ext:cvs.fedora.redhat.com:/cvs/fedora
diff --git a/rebuild-scripts/add_owner b/rebuild-scripts/add_owner
new file mode 100755
index 0000000..e419d42
--- /dev/null
+++ b/rebuild-scripts/add_owner
@@ -0,0 +1,28 @@
+#!/usr/bin/env perl
+# $Id: add_owner,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+local *IN;
+my %OWN;
+open IN, "/home/chris/src/isrec/fedora/owners/owners.list"
+  or die "Couldn't open owners.list: $!";
+while ( <IN> ) {
+  next if /^#/;
+  s/\s+$//;
+  my @F = split /\|/;
+  next if $F[1] eq "";
+  $OWN{$F[1]} = \@F;
+}
+close IN;
+while ( <> ) {
+  s/\s+$//;
+  my @F = split;
+  my $name = $F[5];
+  my $e = $OWN{$name};
+  if (defined $e) {
+    push @F, $$e[3];
+  } else {
+    push @F, "Unknown";
+  }
+  print join(" ", @F), "\n";
+}
+exit 0;
diff --git a/rebuild-scripts/buildcheck b/rebuild-scripts/buildcheck
new file mode 100755
index 0000000..be2bba4
--- /dev/null
+++ b/rebuild-scripts/buildcheck
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# if this one is rebuild again it will confuse the script:
+PACKAGESUPTO=recode
+byebye()
+{
+	[[ -e "${TMPDIR}" ]] && rm -f "${TMPDIR}/owners.list" "${TMPDIR}/repoquery" "${TMPDIR}/repoquery-noarch" "${TMPDIR}/result"
+	[[ -e "${TMPDIR}" ]] && rmdir "${TMPDIR}"
+	local return=${1}
+	shift
+	[[ "${1}" ]] && echo $@
+	exit ${return}
+}
+
+
+
+if ! TMPDIR="$(mktemp -d)"
+then
+	byebye 1 Could not create TMPDIR
+fi
+echo "TMPDIR is $TMPDIR"
+
+if ! wget --quiet -O ${TMPDIR}/owners.list http://cvs.fedora.redhat.com/viewcvs/*checkout*/owners/owners.list?root=extras
+then
+	byebye 2 Failed to get owners.list	
+fi
+
+if ! repoquery -a --repoid=buildcheck-extras-development-sources --repoid=buildcheck-extras-development-buildsys  --qf='%{buildtime} %{name} %{version}-%{release}' *.src | sort | grep -B 9999 ${PACKAGESUPTO} | head -n -1  > ${TMPDIR}/repoquery
+then
+	byebye 3 Repoquery failed
+fi
+
+if ! repoquery -a --repoid=buildcheck-extras-development-x86 --repoid=buildcheck-extras-development-x64 --repoid=buildcheck-extras-development-ppc --repoid=buildcheck-extras-development-buildsys  --qf='%{buildtime} %{name} %{arch} %{version}-%{release}' | sort | grep ' noarch ' | awk '{print $2}' > ${TMPDIR}/repoquery-noarch
+then
+	byebye 3 Repoquery failed
+fi
+
+
+
+while read buildtime name verrel
+do
+	if ownerstring="$(grep \|${name}\| ${TMPDIR}/owners.list)"
+	then
+		echo "${ownerstring}" | while IFS='|' read ignoredone ignoredtwo ignoredthree emailone emailtwo emailthree emailfour alsoignored 
+		do
+			echo ${emailone} ${name} ${verrel} | sed 's|@|_AT_|' >> ${TMPDIR}/result
+		done
+	else
+		echo No owner found for "${name}" >&2
+	fi
+done < ${TMPDIR}/repoquery
+
+sort < ${TMPDIR}/result | uniq | column -t > extras-tobuild
+sort < ${TMPDIR}/result | uniq | column -t | grep -f ${TMPDIR}/repoquery-noarch > extras-tobuild-noarch
+sort < ${TMPDIR}/result | uniq | column -t | grep -v -f ${TMPDIR}/repoquery-noarch > extras-tobuild-arch
+
+
+byebye 0
diff --git a/rebuild-scripts/buildcheck-extras-development.repo b/rebuild-scripts/buildcheck-extras-development.repo
new file mode 100644
index 0000000..0e6ffc6
--- /dev/null
+++ b/rebuild-scripts/buildcheck-extras-development.repo
@@ -0,0 +1,12 @@
+[buildcheck-extras-development-source]
+name=Fedora Extras - Development - Source
+baseurl=http://fedoraproject.org/extras/development/SRPMS/
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-extras
+gpgcheck=1
+
+[buildcheck-extras-buildsys-source]
+name=Fedora Extras - Development - Source
+baseurl=http://buildsys.fedoraproject.org/plague-results/fedora-development-extras/
+enabled=0
+gpgcheck=1
diff --git a/rebuild-scripts/bumpspecfile.py b/rebuild-scripts/bumpspecfile.py
new file mode 100755
index 0000000..9054266
--- /dev/null
+++ b/rebuild-scripts/bumpspecfile.py
@@ -0,0 +1,116 @@
+#!/usr/bin/python -t
+# -*- mode: Python; indent-tabs-mode: nil; -*-
+
+import fcntl
+import os, sys
+import rpmUtils
+import shutil
+import string
+import re
+import time
+
+EXIT_ON_WARN = False
+
+class SpecFile:
+    def __init__(self,filename):
+        self.filename = filename
+        file=open(filename,"r")
+        self.lines=file.readlines()
+        file.close()
+
+    def bumpRelease(self):
+        bump_patterns=[(re.compile(r"^Release:(\s*)(\d+.*)",re.I), self.increase),
+                       (re.compile(r"^%define\s+rel\s+(\d+.*)"), self.increase2),
+                       (re.compile(r"^%define\s+release\s+(\d+.*)"), self.increase3),
+                       (re.compile(r"^%define\s+RELEASE\s+(\d+.*)"), self.increase4),
+                       (re.compile(r"^Release:\s+%release_func\s+(\d+.*)"), self.increase5)
+                       ]
+        skip_pattern=re.compile(r"\$Revision:")
+        for i in range(len(self.lines)):
+            if skip_pattern.search(self.lines[i]): continue
+            for bumpit, bumpit_func in bump_patterns:
+                self.lines[i]=bumpit.sub(bumpit_func,self.lines[i])
+
+    def addChangelogEntry(self,entry,email):
+        versionre=re.compile(r"^Version:\s*(\S+)")
+        changematch=re.compile(r"^%changelog")
+        date=time.strftime("%a %b %d %Y",   time.localtime(time.time()))
+        for i in range(len(self.lines)):
+            versionmatch=versionre.search(self.lines[i])
+            if(versionmatch):
+                version=versionmatch.group(1)
+            if(changematch.match(self.lines[i])):
+                newchangelogentry="%changelog\n* "+date+" "+email+" "+version+"-"+self.newrelease+"\n"+entry+"\n\n"
+                self.lines[i]=newchangelogentry
+                break
+
+    def increaseMain(self,release):
+        relre = re.compile(r'(?P<pre>0\.)?(?P<rel>\d+)(?P<post>.*)',re.I)
+        relmatch = relre.search(release)
+
+        pre = relmatch.group('pre')
+        value = int(relmatch.group('rel'))
+        self.newrelease = `value+1`
+        post = relmatch.group('post')
+
+        old = ''
+        if pre != None:
+            old += pre
+        old += relmatch.group('rel')+post
+
+        if pre == None:
+            if post.find('rc')>=0:
+                print 'CRUCIAL WARNING: Bad pre-release versioning scheme!'
+                print self.filename
+                if EXIT_ON_WARN:
+                    sys.exit(1)
+            new = `value+1`+post
+            if True or post != '%{?dist}' and len(post):
+                self.debugdiff(old,new)
+        else:
+            if value == None or value > 10000:
+                print 'CRUCIAL WARNING: Bad pre-release versioning scheme!'
+                print self.filename
+                if EXIT_ON_WARN:
+                    sys.exit(1)
+            new = '0.'+`value+1`+post
+            self.debugdiff(old,new)
+        return new
+
+    def increase(self,match):
+        return 'Release:' + match.group(1) + self.increaseMain(match.group(2))
+
+    def increase2(self,match):
+        return '%define rel ' + self.increaseMain(match.group(1))
+
+    def increase3(self,match):
+        return '%define release ' + self.increaseMain(match.group(1))
+
+    def increase4(self,match):
+        return '%define RELEASE ' + self.increaseMain(match.group(1))
+
+    def increase5(self,match):
+        return 'Release: %release_func ' + self.increaseMain(match.group(1))
+
+    def writeFile(self,filename):
+        file=open(filename,"w")
+        file.writelines(self.lines)
+        file.close()
+
+    def debugdiff(self,old,new):
+        print '-%s' % old
+        print '+%s\n' % new
+
+if __name__=="__main__":
+    if len(sys.argv) < 2:
+        print 'SYNTAX: %s <specfile> [specfile]...' % sys.argv[0]
+        sys.exit(22)
+
+    userstring = "You <Your.Address@your.domain>"
+    for aspec in sys.argv[1:]:
+        s=SpecFile(aspec)
+        s.bumpRelease()
+        s.addChangelogEntry(" - rebuilt for unwind info generation, broken in gcc-4.1.1-21", userstring)
+        s.writeFile(aspec)
+
+sys.exit(0)
diff --git a/rebuild-scripts/check_rebuild b/rebuild-scripts/check_rebuild
new file mode 100755
index 0000000..863bbf3
--- /dev/null
+++ b/rebuild-scripts/check_rebuild
@@ -0,0 +1,17 @@
+#!/usr/bin/env perl
+# $Id: check_rebuild,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+my %B;
+my $base_time = 1156678565;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[1];
+  $name =~ s/-[^-]+-[^-]+\.src\.rpm$//;
+  if ($F[0] >= $base_time) {
+    $B{$name} = 1;
+    next;
+  }
+  next if defined $B{$name};
+  print;
+}
+exit 0;
diff --git a/rebuild-scripts/check_rebuild2 b/rebuild-scripts/check_rebuild2
new file mode 100755
index 0000000..c71c25a
--- /dev/null
+++ b/rebuild-scripts/check_rebuild2
@@ -0,0 +1,20 @@
+#!/usr/bin/env perl
+# $Id: check_rebuild2,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+my %B;
+my $cur = <>;
+while ( $cur ) {
+  my @F = split /\s+/, $cur;
+  last if $F[1] eq "Sep" and $F[2] == 26;
+  $B{$F[5]} = 1;
+  $cur = <>;
+}
+while ( $cur ) {
+  my @F = split /\s+/, $cur;
+  last if $F[1] eq "Sep" and $F[2] == 17;
+  unless (defined $B{$F[5]}) {
+    print $cur;
+  }
+  $cur = <>;
+}
+exit 0;
diff --git a/rebuild-scripts/check_release b/rebuild-scripts/check_release
new file mode 100755
index 0000000..74b9999
--- /dev/null
+++ b/rebuild-scripts/check_release
@@ -0,0 +1,12 @@
+#!/usr/bin/env perl
+# $Id: check_release,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+while ( <> ) {
+  s/\s+$//;
+  my @F = split;
+  my ($r) = $F[6] =~ /-([^-]+)$/;
+  $r =~ s/\.fc6//;
+  next if $r =~ /^\d+$/;
+  print "$_\n";
+}
+exit 0;
diff --git a/rebuild-scripts/gen_nr_2 b/rebuild-scripts/gen_nr_2
new file mode 100755
index 0000000..343b968
--- /dev/null
+++ b/rebuild-scripts/gen_nr_2
@@ -0,0 +1,26 @@
+#!/usr/bin/env perl
+# $Id: gen_nr_2,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+local *IN;
+my %OWN;
+open IN, "/home/chris/src/isrec/fedora/owners/owners.list"
+  or die "Couldn't open owners.list: $!";
+while ( <IN> ) {
+  next if /^#/;
+  s/\s+$//;
+  my @F = split /\|/;
+  next if $F[1] eq "";
+  $OWN{$F[1]} = \@F;
+}
+close IN;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[0];
+  my $e = $OWN{$name};
+  my $o = "UNKNOWN";
+  if (defined $e) {
+    $o = $$e[3];
+  }
+  print "$o ", join(" ", @F), "\n";
+}
+exit 0;
diff --git a/rebuild-scripts/gen_rebuild_lst b/rebuild-scripts/gen_rebuild_lst
new file mode 100755
index 0000000..c3fab75
--- /dev/null
+++ b/rebuild-scripts/gen_rebuild_lst
@@ -0,0 +1,26 @@
+#!/usr/bin/env perl
+# $Id: gen_rebuild_lst,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+local *IN;
+my %OWN;
+open IN, "/home/chris/src/isrec/fedora/owners/owners.list"
+  or die "Couldn't open owners.list: $!";
+while ( <IN> ) {
+  next if /^#/;
+  s/\s+$//;
+  my @F = split /\|/;
+  next if $F[1] eq "";
+  $OWN{$F[1]} = \@F;
+}
+close IN;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[1];
+  my $e = $OWN{$name};
+  if (defined $e) {
+    print "$$e[3]|$name\n";
+    next;
+  }
+  print STDERR "No owner for $name\n";
+}
+exit 0;
diff --git a/rebuild-scripts/rm_noarch b/rebuild-scripts/rm_noarch
new file mode 100755
index 0000000..317c6fe
--- /dev/null
+++ b/rebuild-scripts/rm_noarch
@@ -0,0 +1,21 @@
+#!/usr/bin/env perl
+# $Id: rm_noarch,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+my %B;
+local *IN;
+open IN, "pkg_noarch.txt" or die "Couldn't open pkg_noarch.txt: $!";
+while ( <IN> ) {
+  chop;
+  my @F = split;
+  my $name = $F[4];
+  $name =~ s/-[^-]+-[^-]+\.src\.rpm$//;
+  $B{$name} = 1;
+}
+close IN;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[5];
+  next if defined $B{$name};
+  print;
+}
+exit 0;
diff --git a/rebuild-scripts/semi_auto_rebuild.txt b/rebuild-scripts/semi_auto_rebuild.txt
new file mode 100644
index 0000000..e258cb8
--- /dev/null
+++ b/rebuild-scripts/semi_auto_rebuild.txt
@@ -0,0 +1,67 @@
+# Log of things done for the last semi-mass rebuild
+# $Id: semi_auto_rebuild.txt,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+
+
+# Grab list of noarch packages
+repoquery -a --repoid=extras-development \
+  --qf='%{buildtime} %{name} %{arch} %{version}-%{release} %{sourcerpm}' \
+  | sort | grep ' noarch ' > pkg_noarch.txt
+
+# Grab list of source packages in the repo, with their build time
+repoquery -a --repoid=buildcheck-extras-development-source \
+  --qf='%{buildtime} %{name} %{version}-%{release}' \*.src \
+  | sort -nr > lst.txt
+
+# Turn the time in a readable form
+perl -ane '$F[0] = localtime($F[0]); print join(" ", @F), "\n"' \
+  lst.txt > lst2.txt
+
+# Apply script to determine which ones need a rebuild
+./check_rebuild2 lst2.txt > lst3.txt
+
+# Remove the noarch packages from the list
+./rm_noarch lst3.txt >lst4.txt
+
+# Produce a list of package names to include in a mail
+perl -ane 'print "$F[5]\n"' lst4.txt|sort >~/to_rebuild.txt
+
+# Add package owner email field in the list of packages to rebuild
+./add_owner lst4.txt >lst5.txt
+
+# Edit list to hand-remove packages that other people said they'd handle
+# themselves
+vi lst5.txt
+
+# Mark out packages that do not have a simple numeric release field
+./check_release lst5.txt > lst_noauto.txt
+
+# Extract the list of simple packages
+diff lst5.txt lst_noauto.txt |grep '^<'|sed 's/^..//' >lst_auto.txt
+
+# Make sure we have the latest version of all packages in our CVS checkout
+pushd /export/scratch/extras/
+cvs update
+popd
+
+# Do the automated mass rebuild for the simple packages
+for f in `awk '{print $6}' lst_auto.txt`; do
+  echo $f
+  ./bumpspecfile.py /export/scratch/extras/$f/devel/$f.spec
+  pushd /export/scratch/extras/$f/devel
+  cvs commit -m 'automated bump and rebuild' && make tag && make build
+  popd
+done
+
+# Now do the rest of the packages (repeat for each package...)
+  for f in `head -1 lst_noauto.txt | awk '{print $6}'`; do
+    echo $f
+    ./bumpspecfile.py /export/scratch/extras/$f/devel/$f.spec
+    pushd /export/scratch/extras/$f/devel
+    cvs diff
+  done
+  vi <package>.spec
+  cvs diff
+  cvs commit -m 'automated bump and rebuild' && make tag && make build
+  popd
+  vi lst_noauto.txt # To remove the first line
+# Done...