From d491ac3a2199cb1f3cdcd4f2f6b3d877b3a65a65 Mon Sep 17 00:00:00 2001 From: Yaakov Nemoy Date: Sun, 28 Sep 2008 10:29:00 -0400 Subject: Make the shell functions a bit more modular for scripting --- rebuild-scripts/CVS/Entries | 12 +++ rebuild-scripts/CVS/Repository | 1 + rebuild-scripts/CVS/Root | 1 + rebuild-scripts/add_owner | 28 +++++ rebuild-scripts/buildcheck | 58 +++++++++++ rebuild-scripts/buildcheck-extras-development.repo | 12 +++ rebuild-scripts/bumpspecfile.py | 116 +++++++++++++++++++++ rebuild-scripts/check_rebuild | 17 +++ rebuild-scripts/check_rebuild2 | 20 ++++ rebuild-scripts/check_release | 12 +++ rebuild-scripts/gen_nr_2 | 26 +++++ rebuild-scripts/gen_rebuild_lst | 26 +++++ rebuild-scripts/rm_noarch | 21 ++++ rebuild-scripts/semi_auto_rebuild.txt | 67 ++++++++++++ 14 files changed, 417 insertions(+) create mode 100644 rebuild-scripts/CVS/Entries create mode 100644 rebuild-scripts/CVS/Repository create mode 100644 rebuild-scripts/CVS/Root create mode 100755 rebuild-scripts/add_owner create mode 100755 rebuild-scripts/buildcheck create mode 100644 rebuild-scripts/buildcheck-extras-development.repo create mode 100755 rebuild-scripts/bumpspecfile.py create mode 100755 rebuild-scripts/check_rebuild create mode 100755 rebuild-scripts/check_rebuild2 create mode 100755 rebuild-scripts/check_release create mode 100755 rebuild-scripts/gen_nr_2 create mode 100755 rebuild-scripts/gen_rebuild_lst create mode 100755 rebuild-scripts/rm_noarch create mode 100644 rebuild-scripts/semi_auto_rebuild.txt (limited to 'rebuild-scripts') diff --git a/rebuild-scripts/CVS/Entries b/rebuild-scripts/CVS/Entries new file mode 100644 index 0000000..d2d53df --- /dev/null +++ b/rebuild-scripts/CVS/Entries @@ -0,0 +1,12 @@ +/add_owner/1.1.1.1/Fri Oct 6 22:22:02 2006// +/buildcheck/1.1.1.1/Fri Oct 6 22:22:02 2006// +/buildcheck-extras-development.repo/1.1.1.1/Fri Oct 6 22:22:02 2006// +/bumpspecfile.py/1.1.1.1/Fri Oct 6 22:22:02 2006// +/check_rebuild/1.1.1.1/Fri Oct 6 22:22:02 2006// +/check_rebuild2/1.1.1.1/Fri Oct 6 22:22:02 2006// +/check_release/1.1.1.1/Fri Oct 6 22:22:02 2006// +/gen_nr_2/1.1.1.1/Fri Oct 6 22:22:02 2006// +/gen_rebuild_lst/1.1.1.1/Fri Oct 6 22:22:02 2006// +/rm_noarch/1.1.1.1/Fri Oct 6 22:22:02 2006// +/semi_auto_rebuild.txt/1.1.1.1/Fri Oct 6 22:22:02 2006// +D diff --git a/rebuild-scripts/CVS/Repository b/rebuild-scripts/CVS/Repository new file mode 100644 index 0000000..2014fa1 --- /dev/null +++ b/rebuild-scripts/CVS/Repository @@ -0,0 +1 @@ +rebuild-scripts diff --git a/rebuild-scripts/CVS/Root b/rebuild-scripts/CVS/Root new file mode 100644 index 0000000..7c4b192 --- /dev/null +++ b/rebuild-scripts/CVS/Root @@ -0,0 +1 @@ +:ext:cvs.fedora.redhat.com:/cvs/fedora diff --git a/rebuild-scripts/add_owner b/rebuild-scripts/add_owner new file mode 100755 index 0000000..e419d42 --- /dev/null +++ b/rebuild-scripts/add_owner @@ -0,0 +1,28 @@ +#!/usr/bin/env perl +# $Id: add_owner,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $ +use strict; +local *IN; +my %OWN; +open IN, "/home/chris/src/isrec/fedora/owners/owners.list" + or die "Couldn't open owners.list: $!"; +while ( ) { + next if /^#/; + s/\s+$//; + my @F = split /\|/; + next if $F[1] eq ""; + $OWN{$F[1]} = \@F; +} +close IN; +while ( <> ) { + s/\s+$//; + my @F = split; + my $name = $F[5]; + my $e = $OWN{$name}; + if (defined $e) { + push @F, $$e[3]; + } else { + push @F, "Unknown"; + } + print join(" ", @F), "\n"; +} +exit 0; diff --git a/rebuild-scripts/buildcheck b/rebuild-scripts/buildcheck new file mode 100755 index 0000000..be2bba4 --- /dev/null +++ b/rebuild-scripts/buildcheck @@ -0,0 +1,58 @@ +#!/bin/bash + +# if this one is rebuild again it will confuse the script: +PACKAGESUPTO=recode +byebye() +{ + [[ -e "${TMPDIR}" ]] && rm -f "${TMPDIR}/owners.list" "${TMPDIR}/repoquery" "${TMPDIR}/repoquery-noarch" "${TMPDIR}/result" + [[ -e "${TMPDIR}" ]] && rmdir "${TMPDIR}" + local return=${1} + shift + [[ "${1}" ]] && echo $@ + exit ${return} +} + + + +if ! TMPDIR="$(mktemp -d)" +then + byebye 1 Could not create TMPDIR +fi +echo "TMPDIR is $TMPDIR" + +if ! wget --quiet -O ${TMPDIR}/owners.list http://cvs.fedora.redhat.com/viewcvs/*checkout*/owners/owners.list?root=extras +then + byebye 2 Failed to get owners.list +fi + +if ! repoquery -a --repoid=buildcheck-extras-development-sources --repoid=buildcheck-extras-development-buildsys --qf='%{buildtime} %{name} %{version}-%{release}' *.src | sort | grep -B 9999 ${PACKAGESUPTO} | head -n -1 > ${TMPDIR}/repoquery +then + byebye 3 Repoquery failed +fi + +if ! repoquery -a --repoid=buildcheck-extras-development-x86 --repoid=buildcheck-extras-development-x64 --repoid=buildcheck-extras-development-ppc --repoid=buildcheck-extras-development-buildsys --qf='%{buildtime} %{name} %{arch} %{version}-%{release}' | sort | grep ' noarch ' | awk '{print $2}' > ${TMPDIR}/repoquery-noarch +then + byebye 3 Repoquery failed +fi + + + +while read buildtime name verrel +do + if ownerstring="$(grep \|${name}\| ${TMPDIR}/owners.list)" + then + echo "${ownerstring}" | while IFS='|' read ignoredone ignoredtwo ignoredthree emailone emailtwo emailthree emailfour alsoignored + do + echo ${emailone} ${name} ${verrel} | sed 's|@|_AT_|' >> ${TMPDIR}/result + done + else + echo No owner found for "${name}" >&2 + fi +done < ${TMPDIR}/repoquery + +sort < ${TMPDIR}/result | uniq | column -t > extras-tobuild +sort < ${TMPDIR}/result | uniq | column -t | grep -f ${TMPDIR}/repoquery-noarch > extras-tobuild-noarch +sort < ${TMPDIR}/result | uniq | column -t | grep -v -f ${TMPDIR}/repoquery-noarch > extras-tobuild-arch + + +byebye 0 diff --git a/rebuild-scripts/buildcheck-extras-development.repo b/rebuild-scripts/buildcheck-extras-development.repo new file mode 100644 index 0000000..0e6ffc6 --- /dev/null +++ b/rebuild-scripts/buildcheck-extras-development.repo @@ -0,0 +1,12 @@ +[buildcheck-extras-development-source] +name=Fedora Extras - Development - Source +baseurl=http://fedoraproject.org/extras/development/SRPMS/ +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-extras +gpgcheck=1 + +[buildcheck-extras-buildsys-source] +name=Fedora Extras - Development - Source +baseurl=http://buildsys.fedoraproject.org/plague-results/fedora-development-extras/ +enabled=0 +gpgcheck=1 diff --git a/rebuild-scripts/bumpspecfile.py b/rebuild-scripts/bumpspecfile.py new file mode 100755 index 0000000..9054266 --- /dev/null +++ b/rebuild-scripts/bumpspecfile.py @@ -0,0 +1,116 @@ +#!/usr/bin/python -t +# -*- mode: Python; indent-tabs-mode: nil; -*- + +import fcntl +import os, sys +import rpmUtils +import shutil +import string +import re +import time + +EXIT_ON_WARN = False + +class SpecFile: + def __init__(self,filename): + self.filename = filename + file=open(filename,"r") + self.lines=file.readlines() + file.close() + + def bumpRelease(self): + bump_patterns=[(re.compile(r"^Release:(\s*)(\d+.*)",re.I), self.increase), + (re.compile(r"^%define\s+rel\s+(\d+.*)"), self.increase2), + (re.compile(r"^%define\s+release\s+(\d+.*)"), self.increase3), + (re.compile(r"^%define\s+RELEASE\s+(\d+.*)"), self.increase4), + (re.compile(r"^Release:\s+%release_func\s+(\d+.*)"), self.increase5) + ] + skip_pattern=re.compile(r"\$Revision:") + for i in range(len(self.lines)): + if skip_pattern.search(self.lines[i]): continue + for bumpit, bumpit_func in bump_patterns: + self.lines[i]=bumpit.sub(bumpit_func,self.lines[i]) + + def addChangelogEntry(self,entry,email): + versionre=re.compile(r"^Version:\s*(\S+)") + changematch=re.compile(r"^%changelog") + date=time.strftime("%a %b %d %Y", time.localtime(time.time())) + for i in range(len(self.lines)): + versionmatch=versionre.search(self.lines[i]) + if(versionmatch): + version=versionmatch.group(1) + if(changematch.match(self.lines[i])): + newchangelogentry="%changelog\n* "+date+" "+email+" "+version+"-"+self.newrelease+"\n"+entry+"\n\n" + self.lines[i]=newchangelogentry + break + + def increaseMain(self,release): + relre = re.compile(r'(?P
0\.)?(?P\d+)(?P.*)',re.I)
+        relmatch = relre.search(release)
+
+        pre = relmatch.group('pre')
+        value = int(relmatch.group('rel'))
+        self.newrelease = `value+1`
+        post = relmatch.group('post')
+
+        old = ''
+        if pre != None:
+            old += pre
+        old += relmatch.group('rel')+post
+
+        if pre == None:
+            if post.find('rc')>=0:
+                print 'CRUCIAL WARNING: Bad pre-release versioning scheme!'
+                print self.filename
+                if EXIT_ON_WARN:
+                    sys.exit(1)
+            new = `value+1`+post
+            if True or post != '%{?dist}' and len(post):
+                self.debugdiff(old,new)
+        else:
+            if value == None or value > 10000:
+                print 'CRUCIAL WARNING: Bad pre-release versioning scheme!'
+                print self.filename
+                if EXIT_ON_WARN:
+                    sys.exit(1)
+            new = '0.'+`value+1`+post
+            self.debugdiff(old,new)
+        return new
+
+    def increase(self,match):
+        return 'Release:' + match.group(1) + self.increaseMain(match.group(2))
+
+    def increase2(self,match):
+        return '%define rel ' + self.increaseMain(match.group(1))
+
+    def increase3(self,match):
+        return '%define release ' + self.increaseMain(match.group(1))
+
+    def increase4(self,match):
+        return '%define RELEASE ' + self.increaseMain(match.group(1))
+
+    def increase5(self,match):
+        return 'Release: %release_func ' + self.increaseMain(match.group(1))
+
+    def writeFile(self,filename):
+        file=open(filename,"w")
+        file.writelines(self.lines)
+        file.close()
+
+    def debugdiff(self,old,new):
+        print '-%s' % old
+        print '+%s\n' % new
+
+if __name__=="__main__":
+    if len(sys.argv) < 2:
+        print 'SYNTAX: %s  [specfile]...' % sys.argv[0]
+        sys.exit(22)
+
+    userstring = "You "
+    for aspec in sys.argv[1:]:
+        s=SpecFile(aspec)
+        s.bumpRelease()
+        s.addChangelogEntry(" - rebuilt for unwind info generation, broken in gcc-4.1.1-21", userstring)
+        s.writeFile(aspec)
+
+sys.exit(0)
diff --git a/rebuild-scripts/check_rebuild b/rebuild-scripts/check_rebuild
new file mode 100755
index 0000000..863bbf3
--- /dev/null
+++ b/rebuild-scripts/check_rebuild
@@ -0,0 +1,17 @@
+#!/usr/bin/env perl
+# $Id: check_rebuild,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+my %B;
+my $base_time = 1156678565;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[1];
+  $name =~ s/-[^-]+-[^-]+\.src\.rpm$//;
+  if ($F[0] >= $base_time) {
+    $B{$name} = 1;
+    next;
+  }
+  next if defined $B{$name};
+  print;
+}
+exit 0;
diff --git a/rebuild-scripts/check_rebuild2 b/rebuild-scripts/check_rebuild2
new file mode 100755
index 0000000..c71c25a
--- /dev/null
+++ b/rebuild-scripts/check_rebuild2
@@ -0,0 +1,20 @@
+#!/usr/bin/env perl
+# $Id: check_rebuild2,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+my %B;
+my $cur = <>;
+while ( $cur ) {
+  my @F = split /\s+/, $cur;
+  last if $F[1] eq "Sep" and $F[2] == 26;
+  $B{$F[5]} = 1;
+  $cur = <>;
+}
+while ( $cur ) {
+  my @F = split /\s+/, $cur;
+  last if $F[1] eq "Sep" and $F[2] == 17;
+  unless (defined $B{$F[5]}) {
+    print $cur;
+  }
+  $cur = <>;
+}
+exit 0;
diff --git a/rebuild-scripts/check_release b/rebuild-scripts/check_release
new file mode 100755
index 0000000..74b9999
--- /dev/null
+++ b/rebuild-scripts/check_release
@@ -0,0 +1,12 @@
+#!/usr/bin/env perl
+# $Id: check_release,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+while ( <> ) {
+  s/\s+$//;
+  my @F = split;
+  my ($r) = $F[6] =~ /-([^-]+)$/;
+  $r =~ s/\.fc6//;
+  next if $r =~ /^\d+$/;
+  print "$_\n";
+}
+exit 0;
diff --git a/rebuild-scripts/gen_nr_2 b/rebuild-scripts/gen_nr_2
new file mode 100755
index 0000000..343b968
--- /dev/null
+++ b/rebuild-scripts/gen_nr_2
@@ -0,0 +1,26 @@
+#!/usr/bin/env perl
+# $Id: gen_nr_2,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+local *IN;
+my %OWN;
+open IN, "/home/chris/src/isrec/fedora/owners/owners.list"
+  or die "Couldn't open owners.list: $!";
+while (  ) {
+  next if /^#/;
+  s/\s+$//;
+  my @F = split /\|/;
+  next if $F[1] eq "";
+  $OWN{$F[1]} = \@F;
+}
+close IN;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[0];
+  my $e = $OWN{$name};
+  my $o = "UNKNOWN";
+  if (defined $e) {
+    $o = $$e[3];
+  }
+  print "$o ", join(" ", @F), "\n";
+}
+exit 0;
diff --git a/rebuild-scripts/gen_rebuild_lst b/rebuild-scripts/gen_rebuild_lst
new file mode 100755
index 0000000..c3fab75
--- /dev/null
+++ b/rebuild-scripts/gen_rebuild_lst
@@ -0,0 +1,26 @@
+#!/usr/bin/env perl
+# $Id: gen_rebuild_lst,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+local *IN;
+my %OWN;
+open IN, "/home/chris/src/isrec/fedora/owners/owners.list"
+  or die "Couldn't open owners.list: $!";
+while (  ) {
+  next if /^#/;
+  s/\s+$//;
+  my @F = split /\|/;
+  next if $F[1] eq "";
+  $OWN{$F[1]} = \@F;
+}
+close IN;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[1];
+  my $e = $OWN{$name};
+  if (defined $e) {
+    print "$$e[3]|$name\n";
+    next;
+  }
+  print STDERR "No owner for $name\n";
+}
+exit 0;
diff --git a/rebuild-scripts/rm_noarch b/rebuild-scripts/rm_noarch
new file mode 100755
index 0000000..317c6fe
--- /dev/null
+++ b/rebuild-scripts/rm_noarch
@@ -0,0 +1,21 @@
+#!/usr/bin/env perl
+# $Id: rm_noarch,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+use strict;
+my %B;
+local *IN;
+open IN, "pkg_noarch.txt" or die "Couldn't open pkg_noarch.txt: $!";
+while (  ) {
+  chop;
+  my @F = split;
+  my $name = $F[4];
+  $name =~ s/-[^-]+-[^-]+\.src\.rpm$//;
+  $B{$name} = 1;
+}
+close IN;
+while ( <> ) {
+  my @F = split;
+  my $name = $F[5];
+  next if defined $B{$name};
+  print;
+}
+exit 0;
diff --git a/rebuild-scripts/semi_auto_rebuild.txt b/rebuild-scripts/semi_auto_rebuild.txt
new file mode 100644
index 0000000..e258cb8
--- /dev/null
+++ b/rebuild-scripts/semi_auto_rebuild.txt
@@ -0,0 +1,67 @@
+# Log of things done for the last semi-mass rebuild
+# $Id: semi_auto_rebuild.txt,v 1.1.1.1 2006/10/06 22:22:02 c4chris Exp $
+
+
+# Grab list of noarch packages
+repoquery -a --repoid=extras-development \
+  --qf='%{buildtime} %{name} %{arch} %{version}-%{release} %{sourcerpm}' \
+  | sort | grep ' noarch ' > pkg_noarch.txt
+
+# Grab list of source packages in the repo, with their build time
+repoquery -a --repoid=buildcheck-extras-development-source \
+  --qf='%{buildtime} %{name} %{version}-%{release}' \*.src \
+  | sort -nr > lst.txt
+
+# Turn the time in a readable form
+perl -ane '$F[0] = localtime($F[0]); print join(" ", @F), "\n"' \
+  lst.txt > lst2.txt
+
+# Apply script to determine which ones need a rebuild
+./check_rebuild2 lst2.txt > lst3.txt
+
+# Remove the noarch packages from the list
+./rm_noarch lst3.txt >lst4.txt
+
+# Produce a list of package names to include in a mail
+perl -ane 'print "$F[5]\n"' lst4.txt|sort >~/to_rebuild.txt
+
+# Add package owner email field in the list of packages to rebuild
+./add_owner lst4.txt >lst5.txt
+
+# Edit list to hand-remove packages that other people said they'd handle
+# themselves
+vi lst5.txt
+
+# Mark out packages that do not have a simple numeric release field
+./check_release lst5.txt > lst_noauto.txt
+
+# Extract the list of simple packages
+diff lst5.txt lst_noauto.txt |grep '^<'|sed 's/^..//' >lst_auto.txt
+
+# Make sure we have the latest version of all packages in our CVS checkout
+pushd /export/scratch/extras/
+cvs update
+popd
+
+# Do the automated mass rebuild for the simple packages
+for f in `awk '{print $6}' lst_auto.txt`; do
+  echo $f
+  ./bumpspecfile.py /export/scratch/extras/$f/devel/$f.spec
+  pushd /export/scratch/extras/$f/devel
+  cvs commit -m 'automated bump and rebuild' && make tag && make build
+  popd
+done
+
+# Now do the rest of the packages (repeat for each package...)
+  for f in `head -1 lst_noauto.txt | awk '{print $6}'`; do
+    echo $f
+    ./bumpspecfile.py /export/scratch/extras/$f/devel/$f.spec
+    pushd /export/scratch/extras/$f/devel
+    cvs diff
+  done
+  vi .spec
+  cvs diff
+  cvs commit -m 'automated bump and rebuild' && make tag && make build
+  popd
+  vi lst_noauto.txt # To remove the first line
+# Done...
-- 
cgit