diff options
| -rwxr-xr-x | lib/subunit/tap2subunit | 1 | ||||
| -rw-r--r-- | selftest/Subunit.pm | 90 | ||||
| -rwxr-xr-x | selftest/filter-subunit.pl | 29 | ||||
| -rw-r--r-- | selftest/output/plain.pm | 30 | ||||
| -rwxr-xr-x | selftest/selftest.pl | 23 | ||||
| -rw-r--r-- | source4/selftest/config.mk | 5 |
6 files changed, 145 insertions, 33 deletions
diff --git a/lib/subunit/tap2subunit b/lib/subunit/tap2subunit index 45f515540b0..9e335168f52 100755 --- a/lib/subunit/tap2subunit +++ b/lib/subunit/tap2subunit @@ -33,4 +33,3 @@ while(<STDIN>) { } } exit $error; - diff --git a/selftest/Subunit.pm b/selftest/Subunit.pm index 5ee8743c6c5..4fddeec8b13 100644 --- a/selftest/Subunit.pm +++ b/selftest/Subunit.pm @@ -35,11 +35,11 @@ sub parse_results($$$$) while(<$fh>) { if (/^test: (.+)\n/) { $msg_ops->control_msg($_); - $msg_ops->start_test($open_tests, $1); + $msg_ops->start_test($1); push (@$open_tests, $1); } elsif (/^time: (\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)Z\n/) { $msg_ops->report_time(mktime($6, $5, $4, $3, $2, $1)); - } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail): (.*?)( \[)?([ \t]*)\n/) { + } elsif (/^(success|successful|failure|fail|skip|knownfail|error|xfail|skip-testsuite|testsuite-failure|testsuite-success|testsuite-error): (.*?)( \[)?([ \t]*)\n/) { $msg_ops->control_msg($_); my $result = $1; my $testname = $2; @@ -55,7 +55,7 @@ sub parse_results($$$$) unless ($terminated) { $statistics->{TESTS_ERROR}++; - $msg_ops->end_test($testname, $result, 1, "reason interrupted"); + $msg_ops->end_test($testname, "error", 1, "reason ($result) interrupted"); return 1; } } @@ -85,14 +85,26 @@ sub parse_results($$$$) pop(@$open_tests); #FIXME: Check that popped value == $testname $msg_ops->end_test($testname, $result, 1, $reason); $unexpected_err++; - } + } elsif ($result eq "skip-testsuite") { + $msg_ops->skip_testsuite($testname); + } elsif ($result eq "testsuite-success") { + $msg_ops->end_testsuite($testname, "success", $reason); + } elsif ($result eq "testsuite-failure") { + $msg_ops->end_testsuite($testname, "failure", $reason); + } elsif ($result eq "testsuite-error") { + $msg_ops->end_testsuite($testname, "error", $reason); + } + } elsif (/^testsuite: (.*)\n/) { + $msg_ops->start_testsuite($1); + } elsif (/^testsuite-count: (\d+)\n/) { + $msg_ops->testsuite_count($1); } else { $msg_ops->output_msg($_); } } while ($#$open_tests > $orig_open_len) { - $msg_ops->end_test($open_tests, pop(@$open_tests), "error", 1, + $msg_ops->end_test(pop(@$open_tests), "error", 1, "was started but never finished!"); $statistics->{TESTS_ERROR}++; $unexpected_err++; @@ -118,7 +130,7 @@ sub end_test($$;$) my $result = shift; my $reason = shift; if ($reason) { - print "$result: $name ["; + print "$result: $name [\n"; print "$reason"; print "]\n"; } else { @@ -126,6 +138,34 @@ sub end_test($$;$) } } +sub skip_test($;$) +{ + my $name = shift; + my $reason = shift; + end_test($name, "skip", $reason); +} + +sub fail_test($;$) +{ + my $name = shift; + my $reason = shift; + end_test($name, "fail", $reason); +} + +sub success_test($;$) +{ + my $name = shift; + my $reason = shift; + end_test($name, "success", $reason); +} + +sub xfail_test($;$) +{ + my $name = shift; + my $reason = shift; + end_test($name, "xfail", $reason); +} + sub report_time($) { my ($time) = @_; @@ -133,4 +173,42 @@ sub report_time($) printf "time: %04d-%02d-%02d %02d:%02d:%02dZ\n", $year+1900, $mon, $mday, $hour, $min, $sec; } +# The following are Samba extensions: + +sub start_testsuite($) +{ + my ($name) = @_; + print "testsuite: $name\n"; +} + +sub skip_testsuite($;$) +{ + my ($name, $reason) = @_; + if ($reason) { + print "skip-testsuite: $name [$reason]\n"; + } else { + print "skip-testsuite: $name\n"; + } +} + +sub end_testsuite($$;$) +{ + my $name = shift; + my $result = shift; + my $reason = shift; + if ($reason) { + print "testsuite-$result: $name ["; + print "$reason"; + print "]\n"; + } else { + print "$result: $name\n"; + } +} + +sub testsuite_count($) +{ + my ($count) = @_; + print "testsuite-count: $count\n"; +} + 1; diff --git a/selftest/filter-subunit.pl b/selftest/filter-subunit.pl index 9a2c6f556c7..b7a72217f3b 100755 --- a/selftest/filter-subunit.pl +++ b/selftest/filter-subunit.pl @@ -147,9 +147,9 @@ sub output_msg($$) print $msg; } -sub start_test($$$) +sub start_test($$) { - my ($self, $parents, $testname) = @_; + my ($self, $testname) = @_; if (defined($opt_prefix)) { $testname = $opt_prefix.$testname; @@ -160,7 +160,7 @@ sub start_test($$$) sub end_test($$$$$) { - my ($self, $parents, $testname, $result, $unexpected, $reason) = @_; + my ($self, $testname, $result, $unexpected, $reason) = @_; if (defined($opt_prefix)) { $testname = $opt_prefix.$testname; @@ -174,6 +174,29 @@ sub end_test($$$$$) Subunit::end_test($testname, $result, $reason); } +sub skip_testsuite($;$) +{ + Subunit::skip_testsuite(@_); +} + +sub start_testsuite($;$) +{ + my ($self, $name) = @_; + Subunit::start_testsuite($name); +} + +sub end_testsuite($$;$) +{ + my ($self, $name, $result, $reason) = @_; + Subunit::end_testsuite($name, $result, $reason); +} + +sub testsuite_count($$) +{ + my ($self, $count) = @_; + Subunit::testsuite_count($count); +} + my $msg_ops = {}; bless $msg_ops; diff --git a/selftest/output/plain.pm b/selftest/output/plain.pm index 2427136073f..bfdced29364 100644 --- a/selftest/output/plain.pm +++ b/selftest/output/plain.pm @@ -1,6 +1,6 @@ #!/usr/bin/perl # Plain text output for selftest -# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org> +# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,6 +41,12 @@ sub new($$$$$$$) { bless($self, $class); } +sub testsuite_count($$) +{ + my ($self, $count) = @_; + $self->{totalsuites} = $count; +} + sub report_time($$) { my ($self, $time) = @_; @@ -101,23 +107,21 @@ sub control_msg($$) #$self->output_msg($output); } -sub end_testsuite($$$$$) +sub end_testsuite($$$$) { - my ($self, $name, $result, $unexpected, $reason) = @_; + my ($self, $name, $result, $reason) = @_; my $out = ""; + my $unexpected = 0; - if ($unexpected) { - if ($result eq "success" and not defined($reason)) { - $reason = "Expected negative exit code, got positive exit code"; - } + if ($result eq "success" or $result eq "xfail") { + $self->{suites_ok}++; + } else { $self->output_msg("ERROR: $reason\n"); push (@{$self->{suitesfailed}}, $name); - } else { - $self->{suites_ok}++; - } - - if ($unexpected and $self->{immediate} and not $self->{verbose}) { - $out .= $self->{test_output}->{$name}; + if ($self->{immediate} and not $self->{verbose}) { + $out .= $self->{test_output}->{$name}; + } + $unexpected = 1; } if (not $self->{immediate}) { diff --git a/selftest/selftest.pl b/selftest/selftest.pl index d98c5ae370b..da259439574 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -166,7 +166,7 @@ sub find_in_list($$) foreach (@$list) { if ($fullname =~ /$$_[0]/) { return ($$_[1]) if ($$_[1]); - return "NO REASON SPECIFIED"; + return ""; } } @@ -217,11 +217,15 @@ sub run_testsuite($$$$$) my $pcap_file = setup_pcap($name); Subunit::report_time(time()); - Subunit::start_test($name); + Subunit::start_testsuite($name); - my $ret = system("$cmd | $RealBin/filter-subunit.pl --prefix \"$name.\" 2>&1"); + system("$cmd 2>&1 | $RealBin/filter-subunit.pl --prefix \"$name.\""); + my $ret = $?; if ($ret == -1) { - Subunit::end_test($name, "error", "Unable to run $cmd: $!"); + Subunit::end_testsuite($name, "error", "Unable to run $cmd: $!"); + return 0; + } elsif ($ret & 127) { + Subunit::end_testsuite($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ? "with": "without")); return 0; } my $envlog = getlog_env($envname); @@ -241,7 +245,7 @@ sub run_testsuite($$$$$) } else { $result = "failure"; } - Subunit::end_test($name, $result, $reason); + Subunit::end_testsuite($name, $result, $reason); cleanup_pcap($pcap_file, $exitcode); @@ -616,18 +620,19 @@ my @available = (); foreach my $fn (@testlists) { foreach (read_testlist($fn)) { my $name = $$_[0]; - next if (@includes and not find_in_list(\@includes, $name)); + next if (@includes and not defined(find_in_list(\@includes, $name))); push (@available, $_); } } +Subunit::testsuite_count($#available+1); Subunit::report_time(time()); foreach (@available) { my $name = $$_[0]; my $skipreason = skip($name); - if ($skipreason) { - Subunit::end_test($name, "skip", $skipreason); + if (defined($skipreason)) { + Subunit::skip_testsuite($name, $skipreason); } else { push(@todo, $_); } @@ -815,7 +820,7 @@ $envvarstr my $envvars = setup_env($envname); if (not defined($envvars)) { - Subunit::end_test($name, "skip", + Subunit::skip_testsuite($name, "unable to set up environment $envname"); next; } diff --git a/source4/selftest/config.mk b/source4/selftest/config.mk index 3271e703112..328ae093308 100644 --- a/source4/selftest/config.mk +++ b/source4/selftest/config.mk @@ -11,7 +11,7 @@ SELFTEST_QUICK_OPTS = $(SELFTEST_NOSLOW_OPTS) --quick --include=$(srcdir)/selfte FILTER_XFAIL = $(PERL) $(selftestdir)/filter-subunit.pl --expected-failures=$(srcdir)/selftest/knownfail FORMAT_TEST_OUTPUT = $(FILTER_XFAIL) | $(PERL) $(selftestdir)/format-subunit.pl --format=$(TEST_FORMAT) -subunittest:: everything +test-subunit:: everything $(SELFTEST) --socket-wrapper $(TESTS) slowtest:: everything @@ -46,6 +46,9 @@ test-noswrap:: everything quicktest:: all $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) | $(FORMAT_TEST_OUTPUT) --immediate +quicktest-subunit:: all + $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper $(TESTS) + quicktestone:: all $(SELFTEST) $(SELFTEST_QUICK_OPTS) --socket-wrapper --one $(TESTS) | $(FORMAT_TEST_OUTPUT) |