New tool: virt-make-fs for creating filesystems on devices.

3 files changed, 623 insertions, 0 deletions

diff --git a/tools/Makefile.am b/tools/Makefile.am
index 1d7c0d1c..9cc61399 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -24,6 +24,7 @@ tools = \
 	list-filesystems \
 	list-partitions \
 	ls \
+	make-fs \
 	rescue \
 	resize \
 	tar \
@@ -84,6 +85,7 @@ TESTS = test-virt-cat.sh \
 	test-virt-df.sh \
 	test-virt-list-filesystems.sh \
 	test-virt-ls.sh \
+	test-virt-make-fs.sh \
 	test-virt-tar.sh
 
 endif
diff --git a/tools/test-virt-make-fs.sh b/tools/test-virt-make-fs.sh
new file mode 100755
index 00000000..e2465064
--- /dev/null
+++ b/tools/test-virt-make-fs.sh
@@ -0,0 +1,50 @@
+#!/bin/bash -
+
+export LANG=C
+set -e
+
+# Engage in some montecarlo testing of virt-make-fs.
+case $((RANDOM % 4)) in
+    0) type="--type=ext2" ;;
+    1) type="--type=ext3" ;;
+    2) type="--type=ext4" ;;
+    3) type="--type=ntfs" ;;
+    # Can't test vfat because we cannot create a tar archive
+    # where files are owned by UID:GID 0:0.  As a result, tar
+    # in the appliance fails when trying to change the UID of
+    # the files to some non-zero value (not supported by FAT).
+    # 4) type="--type=vfat" ;;
+esac
+
+case $((RANDOM % 2)) in
+    0) format="--format=raw" ;;
+    1) format="--format=qcow2" ;;
+esac
+
+case $((RANDOM % 3)) in
+    0) partition="--partition" ;;
+    1) partition="--partition=gpt" ;;
+    2) ;;
+esac
+
+case $((RANDOM % 2)) in
+    0) ;;
+    1) size="--size=+1M" ;;
+esac
+
+if [ -n "$LIBGUESTFS_DEBUG" ]; then debug=--debug; fi
+
+params="$type $format $partition $size $debug"
+echo "test-virt-make-fs: parameters: $params"
+
+rm -f test.file test.tar output.img
+
+tarsize=$((RANDOM & 8191))
+echo "test-virt-make-fs: size of test file: $tarsize KB"
+dd if=/dev/zero of=test.file bs=1024 count=$tarsize
+tar -c -f test.tar test.file
+rm test.file
+
+./virt-make-fs $params -- test.tar output.img
+
+rm test.tar output.img
diff --git a/tools/virt-make-fs b/tools/virt-make-fs
new file mode 100755
index 00000000..54c5a1dc
--- /dev/null
+++ b/tools/virt-make-fs
@@ -0,0 +1,571 @@
+#!/usr/bin/perl -w
+# virt-make-fs
+# Copyright (C) 2010 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+use warnings;
+use strict;
+
+use Sys::Guestfs;
+
+use Pod::Usage;
+use Getopt::Long;
+use File::Temp qw(tempdir);
+use POSIX qw(mkfifo floor);
+use Data::Dumper;
+use String::ShellQuote qw(shell_quote);
+use Locale::TextDomain 'libguestfs';
+
+=encoding utf8
+
+=head1 NAME
+
+virt-make-fs - Make a filesystem from a tar archive or files
+
+=head1 SYNOPSIS
+
+ virt-make-fs [--options] input.tar output.img
+
+ virt-make-fs [--options] input.tar.gz output.img
+
+ virt-make-fs [--options] directory output.img
+
+=head1 DESCRIPTION
+
+Virt-make-fs is a command line tool for creating a filesystem from a
+tar archive or some files in a directory.  It is similar to tools like
+L<mkisofs(1)>, L<genisoimage(1)> and L<mksquashfs(1)>.  Unlike those
+tools, it can create common filesystem types like ext2/3 or NTFS,
+which can be useful if you want to attach these filesystems to
+existing virtual machines (eg. to import large amounts of read-only
+data to a VM).
+
+Basic usage is:
+
+ virt-make-fs input output
+
+where C<input> is either a directory containing files that you want to
+add, or a tar archive (either uncompressed tar or gzip-compressed
+tar); and C<output> is a disk image.  The input type is detected
+automatically.  The output disk image defaults to a raw ext2 image
+unless you specify extra flags (see L</OPTIONS> below).
+
+=head2 EXTRA SPACE
+
+Unlike formats such as tar and squashfs, a filesystem does not "just
+fit" the files that it contains, but might have extra space.
+Depending on how you are going to use the output, you might think this
+extra space is wasted and want to minimize it, or you might want to
+leave space so that more files can be added later.  Virt-make-fs
+defaults to minimizing the extra space, but you can use the C<--size>
+flag to leave space in the filesystem if you want it.
+
+An alternative way to leave extra space but not make the output image
+any bigger is to use an alternative disk image format (instead of the
+default "raw" format).  Using C<--format=qcow2> will use the native
+QEmu/KVM qcow2 image format (check your hypervisor supports this
+before using it).  This allows you to choose a large C<--size> but the
+extra space won't actually be allocated in the image until you try to
+store something in it.
+
+Don't forget that you can also use local commands including
+L<resize2fs(8)> and L<virt-resize(1)> to resize existing filesystems,
+or rerun virt-make-resize to build another image from scratch.
+
+=head3 EXAMPLE
+
+ virt-make-fs --format=qcow2 --size=+200M input output.img
+
+=head2 FILESYSTEM TYPE
+
+The default filesystem type is C<ext2>.  Just about any filesystem
+type that libguestfs supports can be used (but I<not> read-only
+formats like ISO9660).  Here are some of the more common choices:
+
+=over 4
+
+=item I<ext3>
+
+Note that ext3 filesystems contain a journal, typically 1-32 MB in size.
+If you are not going to use the filesystem in a way that requires the
+journal, then this is just wasted overhead.
+
+=item I<ntfs> or I<vfat>
+
+Useful if exporting data to a Windows guest.
+
+I<Note for vfat>: The tar archive or local directory must only contain
+files which are owned by root (ie. UID:GID = 0:0).  The reason is that
+the tar program running within libguestfs is unable to change the
+ownership of non-root files, since vfat itself does not support this.
+
+=item I<minix>
+
+Lower overhead than C<ext2>, but certain limitations on filename
+length and total filesystem size.
+
+=back
+
+=head3 EXAMPLE
+
+ virt-make-fs --type=minix input minixfs.img
+
+=head2 TO PARTITION OR NOT TO PARTITION
+
+Optionally virt-make-fs can add a partition table to the output disk.
+
+Adding a partition can make the disk image more compatible with
+certain virtualized operating systems which don't expect to see a
+filesystem directly located on a block device (Linux doesn't care and
+will happily handle both types).
+
+On the other hand, if you have a partition table then the output image
+is no longer a straight filesystem.  For example you cannot run
+L<fsck(8)> directly on a partitioned disk image.  (However libguestfs
+tools such as L<guestfish(1)> and L<virt-resize(1)> can still be
+used).
+
+=head3 EXAMPLE
+
+Add an MBR partition:
+
+ virt-make-fs --partition -- input disk.img
+
+If the output disk image could be terabyte-sized or larger, it's
+better to use an EFI/GPT-compatible partition table:
+
+ virt-make-fs --partition=gpt --size=+4T --format=qcow2 input disk.img
+
+=head1 OPTIONS
+
+=over 4
+
+=cut
+
+my $help;
+
+=item B<--help>
+
+Display brief help.
+
+=cut
+
+my $version;
+
+=item B<--version>
+
+Display version number and exit.
+
+=cut
+
+my $debug;
+
+=item B<--debug>
+
+Enable debugging information.
+
+=cut
+
+my $size;
+
+=item B<--size=E<lt>NE<gt>>
+
+=item B<--size=+E<lt>NE<gt>>
+
+=item B<-s E<lt>NE<gt>>
+
+=item B<-s +E<lt>NE<gt>>
+
+Use the C<--size> (or C<-s>) option to choose the size of the output
+image.
+
+If this option is I<not> given, then the output image will be just
+large enough to contain all the files, with not much wasted space.
+
+To choose a fixed size output disk, specify an absolute number
+followed by b/K/M/G/T/P/E to mean bytes, Kilobytes, Megabytes,
+Gigabytes, Terabytes, Petabytes or Exabytes.  This must be large
+enough to contain all the input files, else you will get an error.
+
+To leave extra space, specify C<+> (plus sign) and a number followed
+by b/K/M/G/T/P/E to mean bytes, Kilobytes, Megabytes, Gigabytes,
+Terabytes, Petabytes or Exabytes.  For example: C<--size=+200M> means
+enough space for the input files, and (approximately) an extra 200 MB
+free space.
+
+Note that virt-make-fs estimates free space, and therefore will not
+produce filesystems containing precisely the free space requested.
+(It is much more expensive and time-consuming to produce a filesystem
+which has precisely the desired free space).
+
+=cut
+
+my $format = "raw";
+
+=item B<--format=E<lt>fmtE<gt>>
+
+=item B<-F E<lt>fmtE<gt>>
+
+Choose the output disk image format.
+
+The default is C<raw> (raw disk image).
+
+For other choices, see the L<qemu-img(1)> manpage.  The only other
+choice that would really make sense here is C<qcow2>.
+
+=cut
+
+my $type = "ext2";
+
+=item B<--type=E<lt>fsE<gt>>
+
+=item B<-t E<lt>fsE<gt>>
+
+Choose the output filesystem type.
+
+The default is C<ext2>.
+
+Any filesystem which is supported read-write by libguestfs can be used
+here.
+
+=cut
+
+my $partition;
+
+=item B<--partition>
+
+=item B<--partition=E<lt>parttypeE<gt>>
+
+If specified, this flag adds an MBR partition table to the output disk
+image.
+
+You can change the partition table type, eg. C<--partition=gpt> for
+large disks.
+
+Note that if you just use a lonesome C<--partition>, the Perl option
+parser might consider the next parameter to be the partition type.
+For example:
+
+ virt-make-fs --partition input.tar ...
+
+would cause virt-make-fs to think you wanted to use a partition type
+of C<input.tar> which is completely wrong.  To avoid this, use C<-->
+(a double dash) between options and the input file argument:
+
+ virt-make-fs --partition -- input.tar ...
+
+=back
+
+=cut
+
+GetOptions ("help|?" => \$help,
+            "version" => \$version,
+            "debug" => \$debug,
+            "s|size=s" => \$size,
+            "F|format=s" => \$format,
+            "t|type=s" => \$type,
+            "partition:s" => \$partition,
+    ) or pod2usage (2);
+pod2usage (1) if $help;
+if ($version) {
+    my $g = Sys::Guestfs->new ();
+    my %h = $g->version ();
+    print "$h{major}.$h{minor}.$h{release}$h{extra}\n";
+    exit
+}
+
+die __"virt-make-fs input output\n" if @ARGV != 2;
+
+my $input = $ARGV[0];
+my $output = $ARGV[1];
+
+# Input.  What is it?  Estimate how much space it will need.
+#
+# Estimation is a Hard Problem.  Some factors which make it hard:
+#
+#   - Superblocks, block free bitmaps, FAT and other fixed overhead
+#   - Indirect blocks (ext2, ext3), and extents
+#   - Journal size
+#   - Internal fragmentation of files
+#
+# What we could also do is try shrinking the filesystem after creating
+# and populating it, but that is complex given partitions.
+
+my $estimate;     # Estimated size required (in bytes).
+my $ifmt;         # Input format.
+
+if (-d $input) {
+    $ifmt = "directory";
+
+    my @cmd = ("du", "--apparent-size", "-b", "-s", $input);
+    open PIPE, "-|", @cmd or die "du $input: $!";
+
+    $_ = <PIPE>;
+    if (/^(\d+)/) {
+        $estimate = $1;
+    } else {
+        die __"unexpected output from 'du' command";
+    }
+} else {
+    local $ENV{LANG} = "C";
+    my @cmd = ("file", "-bsLz", $input);
+    open PIPE, "-|", @cmd or die "file $input: $!";
+
+    $ifmt = <PIPE>;
+    chomp $ifmt;
+    close PIPE;
+
+    if ($ifmt !~ /tar archive/) {
+        die __x("{f}: unknown input format: {fmt}\n",
+                f => $input, fmt => $ifmt);
+    }
+
+    if ($ifmt =~ /compress.d/) {
+        if ($ifmt =~ /compress'd/) {
+            @cmd = ("uncompress", "-c", $input);
+        } elsif ($ifmt =~ /gzip compressed/) {
+            @cmd = ("gzip", "-cd", $input);
+        } elsif ($ifmt =~ /bzip2 compressed/) {
+            @cmd = ("bzip2", "-cd", $input);
+        } elsif ($ifmt =~ /xz compressed/) {
+            @cmd = ("xz", "-cd", $input);
+        } else {
+            die __x("{f}: unknown input format: {fmt}\n",
+                    f => $input, fmt => $ifmt);
+        }
+
+        open PIPE, "-|", @cmd or die "uncompress $input: $!";
+        $estimate = 0;
+        $estimate += length while <PIPE>;
+        close PIPE or die "close: $!";
+    } else {
+        # Plain tar file, just get the size directly.  Tar files have
+        # a 512 byte block size (compared with typically 1K or 4K for
+        # filesystems) so this isn't very accurate.
+        $estimate = -s $input;
+    }
+}
+
+if ($debug) {
+    printf STDERR "input format = %s\n", $ifmt;
+    printf STDERR "estimate = %s bytes (%s 1K blocks, %s 4K blocks)\n",
+      $estimate, $estimate / 1024, $estimate / 4096;
+}
+
+$estimate += 256 * 1024;        # For superblocks &c.
+
+if ($type =~ /^ext[3-9]/) {
+    $estimate += 1024 * 1024;   # For ext3/4, add some more for the journal.
+}
+
+if ($type =~ /^ntfs/) {
+    $estimate += 4 * 1024 * 1024; # NTFS journal.
+}
+
+$estimate *= 1.10;              # Add 10%, see above.
+
+# Calculate the output size.
+
+if (!defined $size) {
+    $size = $estimate;
+} else {
+    if ($size =~ /^\+([.\d]+)([bKMGTPE])$/) {
+        $size = $estimate + sizebytes ($1, $2);
+    } elsif ($size =~ /^([.\d]+)([bKMGTPE])$/) {
+        $size = sizebytes ($1, $2);
+    } else {
+        die __x("virt-make-fs: cannot parse size parameter: {sz}\n",
+                sz => $size);
+    }
+}
+
+# Create the output disk.
+# Take the unusual step of invoking qemu-img here.
+
+my @cmd = ("qemu-img", "create", "-f", $format, $output, $size);
+system (@cmd) == 0 or
+    die __"qemu-img create: failed to create disk image, see earlier error messages\n";
+
+eval {
+    print STDERR "starting libguestfs ...\n" if $debug;
+
+    # Run libguestfs.
+    my $g = Sys::Guestfs->new ();
+    $g->add_drive ($output);
+    $g->launch ();
+
+    if ($type eq "ntfs") {
+        $g->available ([ "ntfs3g" ]);
+    }
+
+    # Partition the disk.
+    my $dev = "/dev/sda";
+    if (defined $partition) {
+        $partition = "mbr" if $partition eq "";
+        $g->part_disk ($dev, $partition);
+        $dev = "/dev/sda1";
+    }
+
+    print STDERR "creating $type filesystem on $dev ...\n" if $debug;
+
+    # Create the filesystem.
+    $g->mkfs ($type, $dev);
+    $g->mount_options ("", $dev, "/");
+
+    # Copy the data in.
+    my $ifile;
+
+    if ($ifmt eq "directory") {
+        my $pfile = create_pipe ();
+        my $cmd = sprintf ("tar -C %s -cf - . > $pfile &",
+                           shell_quote ($input));
+        print STDERR "command: $cmd\n" if $debug;
+        system ($cmd) == 0 or die __"tar: failed, see earlier messages\n";
+        $ifile = $pfile;
+    } else {
+        if ($ifmt =~ /compress.d/) {
+            my $pfile = create_pipe ();
+            my $cmd;
+            if ($ifmt =~ /compress'd/) {
+                $cmd = sprintf ("uncompress -c %s > $pfile",
+                                shell_quote ($input));
+            } elsif ($ifmt =~ /gzip compressed/) {
+                $cmd = sprintf ("gzip -cd %s", shell_quote ($input));
+            } elsif ($ifmt =~ /bzip2 compressed/) {
+                $cmd = sprintf ("bzip2 -cd %s", shell_quote ($input));
+            } elsif ($ifmt =~ /xz compressed/) {
+                $cmd = sprintf ("xz -cd %s", shell_quote ($input));
+            } else {
+                die __x("{f}: unknown input format: {fmt}\n",
+                        f => $input, fmt => $ifmt);
+            }
+            $cmd .= " > $pfile &";
+            print STDERR "command: $cmd\n" if $debug;
+            system ($cmd) == 0 or
+                die __"uncompress command failed, see earlier messages\n";
+            $ifile = $pfile;
+        } else {
+            print STDERR "reading directly from $input\n" if $debug;
+            $ifile = $input;
+        }
+    }
+
+    if ($debug) {
+        # For debugging, print statvfs before and after doing
+        # the tar-in.
+        my %stat = $g->statvfs ("/");
+        print STDERR "Before uploading ...\n";
+        print STDERR Dumper(\%stat);
+    }
+
+    print STDERR "Uploading from $ifile to / ...\n" if $debug;
+    $g->tar_in ($ifile, "/");
+
+    if ($debug) {
+        my %stat = $g->statvfs ("/");
+        print STDERR "After uploading ...\n";
+        print STDERR Dumper(\%stat);
+    }
+
+    print STDERR "finishing off\n" if $debug;
+    $g->umount_all ();
+    $g->sync ();
+    undef $g;
+};
+if ($@) {
+    # Error: delete the output before exiting.
+    my $err = $@;
+    unlink $output;
+    if ($err =~ /tar_in/) {
+        print STDERR __"virt-make-fs: error copying contents into filesystem\nAn error here usually means that the program did not estimate the\nfilesystem size correctly.  Please read the BUGS section of the manpage.\n";
+    }
+    print STDERR $err;
+    exit 1;
+}
+
+exit 0;
+
+sub sizebytes
+{
+    local $_ = shift;
+    my $unit = shift;
+
+    $_ *= 1024 if $unit =~ /[KMGTPE]/;
+    $_ *= 1024 if $unit =~ /[MGTPE]/;
+    $_ *= 1024 if $unit =~ /[GTPE]/;
+    $_ *= 1024 if $unit =~ /[TPE]/;
+    $_ *= 1024 if $unit =~ /[PE]/;
+    $_ *= 1024 if $unit =~ /[E]/;
+
+    return floor($_);
+}
+
+sub create_pipe
+{
+    local $_;
+    my $dir = tempdir (CLEANUP => 1);
+    my $pipe = "$dir/pipe";
+    mkfifo ($pipe, 0600) or
+        die "mkfifo: $pipe: $!";
+    return $pipe;
+}
+
+=head1 SEE ALSO
+
+L<guestfish(1)>,
+L<virt-resize(1)>,
+L<virt-tar(1)>,
+L<mkisofs(1)>,
+L<genisoimage(1)>,
+L<mksquashfs(1)>,
+L<mke2fs(8)>,
+L<resize2fs(8)>,
+L<guestfs(3)>,
+L<Sys::Guestfs(3)>,
+L<http://libguestfs.org/>.
+
+=head1 BUGS
+
+When reporting bugs, please enable debugging and capture the
+I<complete> output:
+
+ export LIBGUESTFS_DEBUG=1
+ virt-make-fs --debug [...] > /tmp/virt-make-fs.log 2>&1
+
+Attach /tmp/virt-make-fs.log to a new bug report at
+L<https://bugzilla.redhat.com/>
+
+=head1 AUTHOR
+
+Richard W.M. Jones L<http://et.redhat.com/~rjones/>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2010 Red Hat Inc.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.