r24878: Support specifying a reason for skips/expected failures

(This used to be commit 84c94b37fed837c7a45f59746de80d20063d93e8)

5 files changed, 63 insertions, 40 deletions

diff --git a/source4/samba4-skip b/source4/samba4-skip
index d9159a4199..9d9a6a675f 100644
--- a/source4/samba4-skip
+++ b/source4/samba4-skip
@@ -7,7 +7,7 @@ BASE-IOMETER
 BASE-SAMBA3ERROR
 BASE-CASETABLE
 BASE-NTTRANS
-BASE-BENCH-HOLDCON
+.*BASE-BENCH-HOLDCON.*				# Very slow
 BASE-SCAN-MAXFID
 RAW-BENCH-OPLOCK
 RAW-HOLD-OPLOCK
@@ -24,7 +24,6 @@ ntvfs/cifs BASE-DELAYWRITE
 ntvfs/cifs BASE-IOMETER
 ntvfs/cifs BASE-CASETABLE
 ntvfs/cifs BASE-NTTRANS
-ntvfs/cifs BASE-BENCH-HOLDCON
 ntvfs/cifs BASE-SCAN-MAXFID
 ntvfs/cifs BASE-UTABLE
 ntvfs/cifs BASE-SMB
@@ -38,5 +37,5 @@ ntvfs/cifs RAW-QFILEINFO-IPC
 RPC-DSSYNC
 RPC-SAMSYNC
 LDAP-UPTODATEVECTOR
-RPC-SCANNER
-.*SAMBA3.*
+RPC-SCANNER							# Very slow
+.*SAMBA3.*							# Samba3-specific test
diff --git a/source4/selftest/output/buildfarm.pm b/source4/selftest/output/buildfarm.pm
index d35fe9f250..97c2ec0c40 100644
--- a/source4/selftest/output/buildfarm.pm
+++ b/source4/selftest/output/buildfarm.pm
@@ -99,11 +99,15 @@ sub missing_env($$$)
 	print "FAIL: $name (ENV[$envname] not available!)\n";
 }
 
-sub skip_testsuite($$$)
+sub skip_testsuite($$$$)
 {
-	my ($self, $envname, $name) = @_;
+	my ($self, $envname, $name, $reason) = @_;
 
-	print "SKIPPED: $name\n";
+	if ($reason) {
+		print "SKIPPED: $name [$reason]\n";
+	} else {
+		print "SKIPPED: $name\n";
+	}
 }
 
 1;
diff --git a/source4/selftest/output/html.pm b/source4/selftest/output/html.pm
index 44846dc9dd..2a2a12b63d 100644
--- a/source4/selftest/output/html.pm
+++ b/source4/selftest/output/html.pm
@@ -315,14 +315,18 @@ sub missing_env($$$)
 	print INDEX "</tr>\n";
 }
 
-sub skip_testsuite($$$)
+sub skip_testsuite($$$$)
 {
-	my ($self, $envname, $name) = @_;
+	my ($self, $envname, $name, $reason) = @_;
 
 	print INDEX "<tr>\n";
 	print INDEX "  <td class=\"testSuite\">$name</td>\n";
 	print INDEX "  <td class=\"environment\">$envname</td>\n";
-	print INDEX "  <td class=\"resultSkipped\">SKIPPED</td>\n";
+	if ($reason) {
+		print INDEX "  <td class=\"resultSkipped\">SKIPPED - $reason</td>\n";
+	} else {
+		print INDEX "  <td class=\"resultSkipped\">SKIPPED</td>\n";
+	}
 	print INDEX "</tr>\n";
 }
 
diff --git a/source4/selftest/output/plain.pm b/source4/selftest/output/plain.pm
index 736a138b07..bbf0f8f414 100644
--- a/source4/selftest/output/plain.pm
+++ b/source4/selftest/output/plain.pm
@@ -113,9 +113,13 @@ sub missing_env($$$)
 
 sub skip_testsuite($$$)
 {
-	my ($self, $envname, $name) = @_;
+	my ($self, $envname, $name, $reason) = @_;
 
-	print "SKIPPED: $name\n";
+	if ($reason) {
+		print "SKIPPED: $name [$reason]\n";
+	} else {
+		print "SKIPPED: $name\n";
+	}
 }
 
 1;
diff --git a/source4/selftest/selftest.pl b/source4/selftest/selftest.pl
index c407fbd329..39755e692d 100755
--- a/source4/selftest/selftest.pl
+++ b/source4/selftest/selftest.pl
@@ -161,26 +161,30 @@ my $statistics = {
 	TESTS_SKIP => 0,
 };
 
-sub expecting_failure($)
+sub find_in_list($$)
 {
-	my $fullname = shift;
+	my ($list, $fullname) = @_;
 
-	foreach (@expected_failures) {
-		return 1 if ($fullname =~ /$_/);
+	foreach (@$list) {
+		if ($fullname =~ /$$_[0]/) {
+			 return ($$_[1]) if ($$_[1]);
+			 return "NO REASON SPECIFIED";
+		}
 	}
 
-	return 0;
+	return undef;
 }
 
-sub skip($)
+sub expecting_failure($)
 {
-	my $fullname = shift;
-
-	foreach (@skips) {
-		return 1 if ($fullname =~ /$_/);
-	}
+	my ($name) = @_;
+	return find_in_list(\@expected_failures, $name);
+}
 
-	return 0;
+sub skip($)
+{
+	my ($name) = @_;
+	return find_in_list(\@skips, $name);
 }
 
 sub getlog_env($);
@@ -420,22 +424,30 @@ if ($opt_target eq "samba4") {
 	$target = new Windows();
 }
 
-if (defined($opt_expected_failures)) {
-	open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
-	while (<KNOWN>) { 
+sub read_test_regexes($)
+{
+	my ($name) = @_;
+	my @ret = ();
+	open(LF, "<$name") or die("unable to read $name: $!");
+	while (<LF>) { 
 		chomp; 
-		s/([ \t]+)\#(.*)$//;
-		push (@expected_failures, $_); }
-	close(KNOWN);
+		if (/^(.*?)([ \t]+)\#(.*)$/) {
+			push (@ret, [$1, $3]);
+		} else {
+			s/^(.*?)([ \t]+)\#(.*)$//;
+			push (@ret, [$_, undef]); 
+		}
+	}
+	close(LF);
+	return @ret;
+}
+
+if (defined($opt_expected_failures)) {
+	@expected_failures = read_test_regexes($opt_expected_failures);
 }
 
 if (defined($opt_skip)) {
-	open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
-	while (<SKIP>) { 
-		chomp; 
-		s/([ \t]+)\#(.*)$//;
-		push (@skips, $_); }
-	close(SKIP);
+	@skips = read_test_regexes($opt_skip);
 }
 
 my $interfaces = join(',', ("127.0.0.6/8", 
@@ -697,16 +709,16 @@ $envvarstr
 		my $name = $$_[0];
 		my $envname = $$_[1];
 		
-		if (skip($name)) {
-			$msg_ops->skip_testsuite($envname, $name);
+		my $skipreason = skip($name);
+		if ($skipreason) {
+			$msg_ops->skip_testsuite($envname, $name, $skipreason);
 			$statistics->{SUITES_SKIPPED}++;
 			next;
 		}
 
 		my $envvars = setup_env($envname);
 		if (not defined($envvars)) {
-			$statistics->{SUITES_FAIL}++;
-			$statistics->{TESTS_ERROR}++;
+			$statistics->{SUITES_SKIPPED}++;
 			$msg_ops->missing_env($name, $envname);
 			next;
 		}