tests: Replace 'make extra-tests' with individual tests.

'make extra-tests' was a monolithic set of tests that did all sorts of
things: valgrind, tests over local guests, tests with upstream qemu,
tests with upstream libvirt, tests with the appliance attach method.

This made it hard to perform individual tests, eg. just valgrind
testing.  It was also hard to maintain because the tests were not
located in the same directories as the programs and sometimes
duplicated tests that were run elsewhere.

This commit splits up 'make extra-tests' into 5 separate targets:

 make check-valgrind              # run a subset of tests under valgrind
 make check-valgrind-local-guests # test under valgrind with local guests
 make check-with-appliance        # test with attach-method == appliance
 make check-with-upstream-qemu    # test with an alternate/upstream qemu
 make check-with-upstream-libvirt # test with an alternate/upstream libvirt

(You can also still run 'make extra-tests' which is now simply
a rule that runs the above 5 targets in order).

This replaces everything that was in the tests/extra directory,
so that has now gone.

7 files changed, 6 insertions, 403 deletions

diff --git a/tests/c-api/Makefile.am b/tests/c-api/Makefile.am
index 7adedeb7..5f684f76 100644
--- a/tests/c-api/Makefile.am
+++ b/tests/c-api/Makefile.am
@@ -169,3 +169,6 @@ test_environment_LDADD = \
 #	$(top_builddir)/src/libguestfs.la $(LIBVIRT_LIBS) \
 #	$(LTLIBTHREAD) $(top_builddir)/gnulib/lib/libgnu.la
 #endif
+
+check-valgrind:
+	$(MAKE) VG="$(top_builddir)/run @VG@" check
diff --git a/tests/extra/Makefile.am b/tests/extra/Makefile.am
deleted file mode 100644
index 41cc54c1..00000000
--- a/tests/extra/Makefile.am
+++ /dev/null
@@ -1,229 +0,0 @@
-# libguestfs
-# Copyright (C) 2009-2012 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Extra tests.
-#
-# Requires:
-#   - some local libvirt guests are available
-#   - the disks of these guests are accessible by the current user
-#     (tip: add yourself to the 'disk' group)
-#   - valgrind
-#   - libtool
-# Optional:
-#   - upstream qemu in $QEMUDIR
-#   - upstream libvirt in $LIBVIRTDIR
-#
-# These tests may fail for reasons which aren't necessarily real problems.
-#
-# XXX Not tested:
-#
-# ../edit/virt-edit
-#
-# Perl bindings
-# ../edit/virt-edit -e
-# ../tools/virt-win-reg (Perl)
-# ../tools/virt-make-fs (Perl)
-#   - hard because Perl doesn't cooperate by freeing memory
-#
-# Python bindings
-#   - as for Perl
-#
-# Ruby bindings
-#   - these look feasible to test using valgrind
-#
-# Java bindings
-#
-# ../fuse/guestmount
-#   - hard to test because guestmount forks into the background, and
-#     if valgrind reports errors it doesn't stop the test shell script
-
-# Old RHEL 5 autoconf doesn't have abs_srcdir.
-abs_srcdir ?= $(shell cd $(top_srcdir)/tests/extra && pwd)
-
-# Old RHEL 5 autoconf doesn't have abs_builddir.
-abs_builddir ?= $(shell pwd)
-
-EXTRA_DIST = \
-	pick-guests.pl \
-	test-with-launch-appliance.sh \
-	test-with-upstream-libvirt.sh \
-	test-with-upstream-qemu.sh
-
-VG = @VG@
-RUN_VG = $(abs_top_builddir)/run $(VG)
-
-# Choose guests at random on each run.
-random_guests := \
-	$(shell \
-	  LIBVIRT_DEFAULT_URI="$(libvirt_ro_uri)" \
-	    $(abs_top_builddir)/run $(srcdir)/pick-guests.pl 5)
-
-extra-tests-non-recursive: \
-	test-guests \
-	test-prerequisites \
-	test-c-api \
-	test-tools-null \
-	test-tools-internal \
-	test-df-real \
-	test-filesystems-real \
-	test-inspector-real \
-	test-sysprep-real \
-	test-ocaml \
-	test-resize \
-	test-sparsify \
-	test-mount-local
-
-extra-tests: \
-	extra-tests-non-recursive \
-	extra-tests-with-upstream-qemu \
-	extra-tests-with-upstream-libvirt \
-	extra-tests-with-launch-appliance
-
-test-guests:
-	$(MAKE) -C ../guests check
-
-test-prerequisites:
-	@libtool --help >/dev/null 2>&1 || { \
-	  echo "extra-tests: libtool is not installed"; \
-	  exit 1; \
-	}
-	@valgrind --help >/dev/null 2>&1 || { \
-	  echo "extra-tests: valgrind is not installed"; \
-	  exit 1; \
-	}
-
-# Null invocations of the basic tools shouldn't leak memory.
-test-tools-null:
-	$(RUN_VG) ../../fish/guestfish -N part exit
-	$(RUN_VG) ../../align/virt-alignment-scan -a test1.img >/dev/null
-	$(RUN_VG) ../../format/virt-format -a test1.img >/dev/null
-	rm test1.img
-	$(RUN_VG) ../../cat/virt-filesystems -a /dev/null >/dev/null
-	$(RUN_VG) ../../cat/virt-filesystems -a /dev/null --all --long -h --uuid >/dev/null
-	$(RUN_VG) ../../df/virt-df -a /dev/null >/dev/null
-	$(RUN_VG) ../../fish/guestfish -a /dev/null run
-	$(RUN_VG) ../../inspector/virt-inspector -a /dev/null >/dev/null
-	$(RUN_VG) ../../test-tool/libguestfs-test-tool >/dev/null 2>&1
-
-# Invocations of tools on internal images shouldn't leak memory.
-test-tools-internal:
-	$(RUN_VG) ../../cat/virt-cat -a ../guests/fedora.img /etc/fstab >/dev/null
-	$(RUN_VG) ../../cat/virt-filesystems -a ../guests/fedora.img >/dev/null
-	$(RUN_VG) ../../cat/virt-filesystems -a ../guests/fedora.img --all --long -h --uuid >/dev/null
-	$(RUN_VG) ../../cat/virt-ls -a ../guests/fedora.img / >/dev/null
-	$(RUN_VG) ../../cat/virt-ls -a ../guests/fedora.img -l / >/dev/null
-	$(RUN_VG) ../../cat/virt-ls -a ../guests/fedora.img -R / >/dev/null
-	$(RUN_VG) ../../cat/virt-ls -a ../guests/fedora.img -lR / >/dev/null
-	$(RUN_VG) ../../df/virt-df -a ../guests/fedora.img >/dev/null
-	$(RUN_VG) ../../df/virt-df -a ../guests/fedora.img -h >/dev/null
-	$(RUN_VG) ../../fish/guestfish -a ../guests/fedora.img run
-	$(RUN_VG) ../../fish/guestfish -a ../guests/fedora.img -i exit
-	$(RUN_VG) ../../inspector/virt-inspector -a ../guests/debian.img >/dev/null
-	$(RUN_VG) ../../inspector/virt-inspector -a ../guests/fedora.img >/dev/null
-	$(RUN_VG) ../../inspector/virt-inspector -a ../guests/fedora-md1.img -a ../guests/fedora-md2.img >/dev/null
-	$(RUN_VG) ../../inspector/virt-inspector -a ../guests/ubuntu.img >/dev/null
-	$(RUN_VG) ../../inspector/virt-inspector -a ../guests/windows.img >/dev/null
-	$(RUN_VG) ../../sysprep/virt-sysprep -n -a ../guests/debian.img
-	$(RUN_VG) ../../sysprep/virt-sysprep -n -a ../guests/fedora.img
-	$(RUN_VG) ../../sysprep/virt-sysprep -n -a ../guests/fedora-md1.img -a ../guests/fedora-md2.img
-	$(RUN_VG) ../../sysprep/virt-sysprep -n -a ../guests/ubuntu.img
-	$(RUN_VG) ../../sysprep/virt-sysprep -n -a ../guests/windows.img
-
-# Invocations of tools on real images shouldn't leak memory.
-test-df-real:
-	$(RUN_VG) ../../df/virt-df -c "$(libvirt_ro_uri)" >/dev/null
-	$(RUN_VG) ../../df/virt-df -c "$(libvirt_ro_uri)" -h >/dev/null
-
-test-filesystems-real:
-	@for g in $(random_guests); do \
-	  echo $(RUN_VG) ../../cat/virt-filesystems -c "$(libvirt_ro_uri)" -d $$g --all --long -h --uuid; \
-	  $(RUN_VG) ../../cat/virt-filesystems -c "$(libvirt_ro_uri)" -d $$g --all --long -h --uuid > /dev/null; \
-	  r=$$?; \
-	  if [ $$r -ne 0 ]; then exit $$r; fi; \
-	done
-
-test-inspector-real:
-	@for g in $(random_guests); do \
-	  echo $(RUN_VG) ../../inspector/virt-inspector -c "$(libvirt_ro_uri)" -d $$g; \
-	  $(RUN_VG) ../../inspector/virt-inspector -c "$(libvirt_ro_uri)" -d $$g > /dev/null; \
-	  r=$$?; \
-	  if [ $$r -ne 0 ]; then exit $$r; fi; \
-	done
-
-test-sysprep-real:
-	@for g in $(random_guests); do \
-	  echo $(RUN_VG) ../../sysprep/virt-sysprep -n -c "$(libvirt_ro_uri)" -d $$g; \
-	  $(RUN_VG) ../../sysprep/virt-sysprep -n -c "$(libvirt_ro_uri)" -d $$g > /dev/null; \
-	  r=$$?; \
-	  if [ $$r -ne 0 ]; then exit $$r; fi; \
-	done
-
-# Run virt-resize tests under valgrind.
-test-resize:
-	$(MAKE) -C ../../resize VG="$(VG)" check
-
-# Run virt-sparsify tests under valgrind.
-test-sparsify:
-	$(MAKE) -C ../../sparsify VG="$(VG)" check
-
-# Run c-api tests under valgrind.
-test-c-api:
-	$(MAKE) -C ../c-api VG="$(RUN_VG)" check
-
-# Run parallel mount-local test under valgrind.
-test-mount-local:
-	$(MAKE) -C ../mount-local VG="$(VG)" check
-
-# Test OCaml bindings under valgrind.
-test-ocaml:
-	$(MAKE) -C ../../ocaml VG="$(VG)" check
-
-# When running the extra tests, if directory $QEMUDIR exists then we
-# will rerun all the extra-tests using the version of qemu from that
-# directory.  eg. That might be qemu from git.
-#
-# The directory name can be overridden using:
-#   make extra-tests QEMUDIR=/path/to/other/qemu
-
-QEMUDIR = $(HOME)/d/qemu
-export QEMUDIR
-export abs_srcdir
-export MAKE
-
-extra-tests-with-upstream-qemu:
-	./test-with-upstream-qemu.sh
-
-# When running the extra tests and ./configure
-# --with-default-attach-method=libvirt, if directory $LIBVIRTDIR
-# exists, we will rerun all the extra-tests using the version of
-# libvirt in that directory.  eg. That might be upstream libvirt from
-# git.
-#
-# The directory name can be overridden using:
-#   make extra-tests LIBVIRTDIR=/path/to/other/libvirt
-
-LIBVIRTDIR = $(HOME)/d/libvirt
-export LIBVIRTDIR
-
-extra-tests-with-upstream-libvirt:
-	./test-with-upstream-libvirt.sh
-
-# If libvirt is being used as the backend, rerun all the extra-tests
-# again using the regular appliance backend.
-
-extra-tests-with-launch-appliance:
-	./test-with-launch-appliance.sh
diff --git a/tests/extra/pick-guests.pl b/tests/extra/pick-guests.pl
deleted file mode 100755
index d2e4a225..00000000
--- a/tests/extra/pick-guests.pl
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/perl -w
-# libguestfs
-# Copyright (C) 2009-2012 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# Pick guests at random on the local machine which are accessible.
-# Note that the Makefile sets $LIBVIRT_DEFAULT_URI.
-
-use strict;
-
-use Sys::Guestfs;
-use Sys::Virt;
-use List::Util qw(shuffle);
-
-die "$0 nr-guests\n" unless @ARGV == 1;
-my $n = $ARGV[0];
-
-my $vmm = Sys::Virt->new;
-my @domains = ($vmm->list_domains, $vmm->list_defined_domains);
-
-# Only guests which are accessible by the current (non-root) user.  On
-# the machine where I run these tests, I have added my user account to
-# the 'disk' group, so that most guests are accessible.  However
-# because libvirt changes the permissions on guest disks, a guest
-# which has been run on the machine becomes inaccessible, hence the
-# need for this code - RWMJ.
-my @accessible;
-foreach my $dom (@domains) {
-    my $name = $dom->get_name;
-    my $g = Sys::Guestfs->new;
-    eval {
-        $g->add_domain ($name, readonly => 1);
-        # $g->launch (); - don't actually need to do this
-    };
-    push @accessible, $name unless $@;
-}
-
-# Randomize the list of guests.
-@accessible = shuffle (@accessible);
-
-$n = @accessible if @accessible < $n;
-
-# Return the first n guests from the list.
-for (my $i = 0; $i < $n; ++$i) {
-    print $accessible[$i], "\n";
-}
diff --git a/tests/extra/test-with-launch-appliance.sh b/tests/extra/test-with-launch-appliance.sh
deleted file mode 100755
index 0a4f51b7..00000000
--- a/tests/extra/test-with-launch-appliance.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/sh -
-# Copyright (C) 2012 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -e
-#set -x
-
-method="$(../../fish/guestfish get-attach-method)"
-if [ "$method" = "appliance" ]; then
-    echo "$0: test skipped because attach-method is already 'appliance'"
-    exit 77
-fi
-
-exec $MAKE extra-tests-non-recursive LIBGUESTFS_ATTACH_METHOD=appliance
diff --git a/tests/extra/test-with-upstream-libvirt.sh b/tests/extra/test-with-upstream-libvirt.sh
deleted file mode 100755
index 31b50e8f..00000000
--- a/tests/extra/test-with-upstream-libvirt.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/sh -
-# Copyright (C) 2012 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -e
-#set -x
-
-method="$(../../fish/guestfish get-attach-method)"
-if [ "$method" != "libvirt" ]; then
-    echo "$0: test skipped because attach-method is not 'libvirt'"
-    exit 77
-fi
-
-if [ ! -d "$LIBVIRTDIR" ]; then
-    echo "$0: \$LIBVIRTDIR not a directory, tests against upstream libvirt skipped"
-    exit 77
-fi
-
-libvirt_run="$LIBVIRTDIR/run"
-if [ ! -x "$libvirt_run" ]; then
-    echo "$0: $libvirt_run not executable, tests against upstream libvirt skipped"
-    exit 77
-fi
-
-exec "$libvirt_run" $MAKE extra-tests-non-recursive
diff --git a/tests/extra/test-with-upstream-qemu.sh b/tests/extra/test-with-upstream-qemu.sh
deleted file mode 100755
index 7c883288..00000000
--- a/tests/extra/test-with-upstream-qemu.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/sh -
-# Copyright (C) 2012 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -e
-#set -x
-
-if [ ! -d "$QEMUDIR" ]; then
-    echo "$0: \$QEMUDIR not a directory, tests against upstream qemu skipped"
-    exit 77
-fi
-
-upstream_qemu="$QEMUDIR/x86_64-softmmu/qemu-system-x86_64"
-if ! "$upstream_qemu" --help >/dev/null 2>&1; then
-    echo "$0: $upstream_qemu not executable, tests against upstream qemu skipped"
-    exit 77
-fi
-
-"$upstream_qemu" --version
-
-# Write a custom wrapper script for this invocation.  (Since multiple
-# parallel 'make extra-tests' should not be running in the same
-# libguestfs directory, this should be safe).
-LIBGUESTFS_QEMU="$abs_srcdir/qemu-wrapper.sh"
-rm -f "$LIBGUESTFS_QEMU" "$LIBGUESTFS_QEMU"-t
-cat > "$LIBGUESTFS_QEMU"-t <<EOF
-#!/bin/sh -
-# Wrapper generated by '$0'.
-# Do not edit this file directly.
-exec "$upstream_qemu" -L "$QEMUDIR/pc-bios" "\$@"
-EOF
-chmod +x "$LIBGUESTFS_QEMU"-t
-mv "$LIBGUESTFS_QEMU"-t "$LIBGUESTFS_QEMU"
-
-export LIBGUESTFS_QEMU
-
-exec $MAKE extra-tests-non-recursive
diff --git a/tests/mount-local/Makefile.am b/tests/mount-local/Makefile.am
index ec892213..aa910140 100644
--- a/tests/mount-local/Makefile.am
+++ b/tests/mount-local/Makefile.am
@@ -38,4 +38,7 @@ test_parallel_mount_local_LDADD = \
 	$(top_builddir)/src/libguestfs.la \
 	$(top_builddir)/gnulib/lib/libgnu.la
 
+check-valgrind:
+	$(MAKE) VG="$(top_builddir)/run @VG@" check
+
 endif