path: root/tools/virt-make-fs

#!/usr/bin/perl -w
# virt-make-fs
# Copyright (C) 2010-2012 Red Hat Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use warnings;
use strict;

use Sys::Guestfs;
use Sys::Guestfs::Lib qw(feature_available);

use Pod::Usage;
use Getopt::Long;
use File::Temp qw(tempfile tempdir);
use POSIX qw(mkfifo floor);
use Data::Dumper;
use String::ShellQuote qw(shell_quote);
use Locale::TextDomain 'libguestfs';
use Fcntl qw(SEEK_SET);

=encoding utf8

=head1 NAME

virt-make-fs - Make a filesystem from a tar archive or files

=head1 SYNOPSIS

 virt-make-fs [--options] input.tar output.img

 virt-make-fs [--options] input.tar.gz output.img

 virt-make-fs [--options] directory output.img

=head1 DESCRIPTION

Virt-make-fs is a command line tool for creating a filesystem from a
tar archive or some files in a directory.  It is similar to tools like
L<mkisofs(1)>, L<genisoimage(1)> and L<mksquashfs(1)>.  Unlike those
tools, it can create common filesystem types like ext2/3 or NTFS,
which can be useful if you want to attach these filesystems to
existing virtual machines (eg. to import large amounts of read-only
data to a VM).

To create blank disks, use L<virt-format(1)>.  To create complex
layouts, use L<guestfish(1)>.

Basic usage is:

 virt-make-fs input output.img

where C<input> is either a directory containing files that you want to
add, or a tar archive (either uncompressed tar or gzip-compressed
tar); and C<output.img> is a disk image.  The input type is detected
automatically.  The output disk image defaults to a raw ext2 sparse
image unless you specify extra flags (see L</OPTIONS> below).

=head2 FILESYSTEM TYPE

The default filesystem type is C<ext2>.  Just about any filesystem
type that libguestfs supports can be used (but I<not> read-only
formats like ISO9660).  Here are some of the more common choices:

=over 4

=item I<ext3>

Note that ext3 filesystems contain a journal, typically 1-32 MB in size.
If you are not going to use the filesystem in a way that requires the
journal, then this is just wasted overhead.

=item I<ntfs> or I<vfat>

Useful if exporting data to a Windows guest.

=item I<minix>

Lower overhead than C<ext2>, but certain limitations on filename
length and total filesystem size.

=back

=head3 EXAMPLE

 virt-make-fs --type=minix input minixfs.img

=head2 TO PARTITION OR NOT TO PARTITION

Optionally virt-make-fs can add a partition table to the output disk.

Adding a partition can make the disk image more compatible with
certain virtualized operating systems which don't expect to see a
filesystem directly located on a block device (Linux doesn't care and
will happily handle both types).

On the other hand, if you have a partition table then the output image
is no longer a straight filesystem.  For example you cannot run
L<fsck(8)> directly on a partitioned disk image.  (However libguestfs
tools such as L<guestfish(1)> and L<virt-resize(1)> can still be
used).

=head3 EXAMPLE

Add an MBR partition:

 virt-make-fs --partition -- input disk.img

If the output disk image could be terabyte-sized or larger, it's
better to use an EFI/GPT-compatible partition table:

 virt-make-fs --partition=gpt --size=+4T --format=qcow2 input disk.img

=head2 EXTRA SPACE

Unlike formats such as tar and squashfs, a filesystem does not "just
fit" the files that it contains, but might have extra space.
Depending on how you are going to use the output, you might think this
extra space is wasted and want to minimize it, or you might want to
leave space so that more files can be added later.  Virt-make-fs
defaults to minimizing the extra space, but you can use the I<--size>
flag to leave space in the filesystem if you want it.

An alternative way to leave extra space but not make the output image
any bigger is to use an alternative disk image format (instead of the
default "raw" format).  Using I<--format=qcow2> will use the native
QEmu/KVM qcow2 image format (check your hypervisor supports this
before using it).  This allows you to choose a large I<--size> but the
extra space won't actually be allocated in the image until you try to
store something in it.

Don't forget that you can also use local commands including
L<resize2fs(8)> and L<virt-resize(1)> to resize existing filesystems,
or rerun virt-make-fs to build another image from scratch.

=head3 EXAMPLE

 virt-make-fs --format=qcow2 --size=+200M input output.img

=head1 OPTIONS

=over 4

=cut

my $help;

=item B<--help>

Display brief help.

=cut

my $version;

=item B<--version>

Display version number and exit.

=cut

my $debug;

=item B<--debug>

Enable debugging information.

=cut

=item B<--floppy>

Create a virtual floppy disk.

Currently this preselects the size (1440K), partition type (MBR) and
filesystem type (VFAT).  In future it may also choose the geometry.

=cut

my $size;

=item B<--size=E<lt>NE<gt>>

=item B<--size=+E<lt>NE<gt>>

=item B<-s E<lt>NE<gt>>

=item B<-s +E<lt>NE<gt>>

Use the I<--size> (or I<-s>) option to choose the size of the output
image.

If this option is I<not> given, then the output image will be just
large enough to contain all the files, with not much wasted space.

To choose a fixed size output disk, specify an absolute number
followed by b/K/M/G/T/P/E to mean bytes, Kilobytes, Megabytes,
Gigabytes, Terabytes, Petabytes or Exabytes.  This must be large
enough to contain all the input files, else you will get an error.

To leave extra space, specify C<+> (plus sign) and a number followed
by b/K/M/G/T/P/E to mean bytes, Kilobytes, Megabytes, Gigabytes,
Terabytes, Petabytes or Exabytes.  For example: I<--size=+200M> means
enough space for the input files, and (approximately) an extra 200 MB
free space.

Note that virt-make-fs estimates free space, and therefore will not
produce filesystems containing precisely the free space requested.
(It is much more expensive and time-consuming to produce a filesystem
which has precisely the desired free space).

=cut

my $format = "raw";

=item B<--format=E<lt>fmtE<gt>>

=item B<-F E<lt>fmtE<gt>>

Choose the output disk image format.

The default is C<raw> (raw sparse disk image).

For other choices, see the L<qemu-img(1)> manpage.  The only other
choice that would really make sense here is C<qcow2>.

=cut

my $type = "ext2";

=item B<--type=E<lt>fsE<gt>>

=item B<-t E<lt>fsE<gt>>

Choose the output filesystem type.

The default is C<ext2>.

Any filesystem which is supported read-write by libguestfs can be used
here.

=cut

my $partition;

=item B<--partition>

=item B<--partition=E<lt>parttypeE<gt>>

If specified, this flag adds an MBR partition table to the output disk
image.

You can change the partition table type, eg. I<--partition=gpt> for
large disks.

Note that if you just use a lonesome I<--partition>, the Perl option
parser might consider the next parameter to be the partition type.
For example:

 virt-make-fs --partition input.tar output.img

would cause virt-make-fs to think you wanted to use a partition type
of C<input.tar> which is completely wrong.  To avoid this, use I<-->
(a double dash) between options and the input and output arguments:

 virt-make-fs --partition -- input.tar output.img

For MBR, virt-make-fs sets the partition type byte automatically.

=back

=cut

GetOptions ("help|?" => \$help,
            "version" => \$version,
            "debug" => \$debug,
            "floppy" => sub {
                $size = "1440K";
                $partition = "mbr";
                $type = "vfat";
            },
            "s|size=s" => \$size,
            "F|format=s" => \$format,
            "t|type=s" => \$type,
            "partition:s" => \$partition,
    ) or pod2usage (2);
pod2usage (1) if $help;
if ($version) {
    my $g = Sys::Guestfs->new ();
    my %h = $g->version ();
    print "$h{major}.$h{minor}.$h{release}$h{extra}\n";
    exit
}

die __"virt-make-fs input output\n" if @ARGV != 2;

my $input = $ARGV[0];
my $output = $ARGV[1];

# Input.  What is it?  Estimate how much space it will need.
#
# Estimation is a Hard Problem.  Some factors which make it hard:
#
#   - Superblocks, block free bitmaps, FAT and other fixed overhead
#   - Indirect blocks (ext2, ext3), and extents
#   - Journal size
#   - Internal fragmentation of files
#
# What we could also do is try shrinking the filesystem after creating
# and populating it, but that is complex given partitions.

my $estimate;     # Estimated size required (in bytes).
my $ifmt;         # Input format.

if (-d $input) {
    $ifmt = "directory";

    my @cmd = ("du", "--apparent-size", "-b", "-s", $input);
    open PIPE, "-|", @cmd or die "du $input: $!";

    $_ = <PIPE>;
    if (/^(\d+)/) {
        $estimate = $1;
    } else {
        die __"unexpected output from 'du' command";
    }
} else {
    local $ENV{LANG} = "C";
    my @cmd = ("file", "-bsLz", $input);
    open PIPE, "-|", @cmd or die "file $input: $!";

    $ifmt = <PIPE>;
    chomp $ifmt;
    close PIPE;

    if ($ifmt !~ /tar archive/) {
        die __x("{f}: unknown input format: {fmt}\n",
                f => $input, fmt => $ifmt);
    }

    if ($ifmt =~ /compress.d/) {
        if ($ifmt =~ /compress'd/) {
            @cmd = ("uncompress", "-c", $input);
        } elsif ($ifmt =~ /gzip compressed/) {
            @cmd = ("gzip", "-cd", $input);
        } elsif ($ifmt =~ /bzip2 compressed/) {
            @cmd = ("bzip2", "-cd", $input);
        } elsif ($ifmt =~ /xz compressed/) {
            @cmd = ("xz", "-cd", $input);
        } else {
            die __x("{f}: unknown input format: {fmt}\n",
                    f => $input, fmt => $ifmt);
        }

        open PIPE, "-|", @cmd or die "uncompress $input: $!";
        $estimate = 0;
        $estimate += length while <PIPE>;
        close PIPE or die "close: $!";
    } else {
        # Plain tar file, just get the size directly.  Tar files have
        # a 512 byte block size (compared with typically 1K or 4K for
        # filesystems) so this isn't very accurate.
        $estimate = -s $input;
    }
}

if ($debug) {
    printf STDERR "input format = %s\n", $ifmt;
    printf STDERR "estimate = %s bytes (%s 1K blocks, %s 4K blocks)\n",
      $estimate, $estimate / 1024, $estimate / 4096;
}

$estimate += 256 * 1024;        # For superblocks &c.

if ($type =~ /^ext[3-9]/) {
    $estimate += 1024 * 1024;   # For ext3/4, add some more for the journal.
}

if ($type eq "ntfs") {
    $estimate += 4 * 1024 * 1024; # NTFS journal.
}

if ($type eq "btrfs") {
    # For BTRFS, the minimum metadata allocation is 256MB, with data
    # additional to that.  Note that we disable data and metadata
    # duplication below.
    $estimate += 256 * 1024 * 1024;
}

$estimate *= 1.10;              # Add 10%, see above.

# Calculate the output size.

if (!defined $size) {
    $size = $estimate;
} else {
    if ($size =~ /^\+([.\d]+)([bKMGTPE])$/) {
        $size = $estimate + sizebytes ($1, $2);
    } elsif ($size =~ /^([.\d]+)([bKMGTPE])$/) {
        $size = sizebytes ($1, $2);
    } else {
        die __x("virt-make-fs: cannot parse size parameter: {sz}\n",
                sz => $size);
    }
}

$size = int ($size);

# Create the output disk.
#
# Use qemu-img so we can control the output format, but capture any
# output temporarily and only display it if the command fails.

my @cmd = ("qemu-img", "create", "-f", $format, $output, $size);
if ($debug) {
    print STDERR ("running: ", join (" ", @cmd), "\n");
}

{
    my $tmpfh = tempfile ();
    my ($r, $oldout, $olderr);

    open $oldout, ">&STDOUT" or die __"cannot dup STDOUT";
    open $olderr, ">&STDERR" or die __"cannot dup STDERR";
    close STDOUT;
    close STDERR;
    open STDOUT, ">&", \$tmpfh or die __"cannot redirect STDOUT";
    open STDERR, ">&", \$tmpfh or die __"cannot redirect STDERR";
    $r = system (@cmd);
    open STDOUT, ">&", $oldout or die __"cannot restore STDOUT";
    open STDERR, ">&", $olderr or die __"cannot restore STDERR";

    unless ($r == 0) {
        print STDERR __"qemu-img create: failed to create disk image:\n";
        seek $tmpfh, 0, SEEK_SET;
        print STDERR $_ while <$tmpfh>;
        die "\n";
    }
}

eval {
    print STDERR "starting libguestfs ...\n" if $debug;

    # Run libguestfs.
    my $g = Sys::Guestfs->new ();
    $g->add_drive ($output, format => $format);
    $g->launch ();

    if ($type eq "ntfs" && !feature_available ($g, "ntfs3g", "ntfsprogs")) {
        die __"virt-make-fs: NTFS support was disabled when libguestfs was compiled\n"
    }

    # Partition the disk.
    my $dev = "/dev/sda";
    if (defined $partition) {
        $partition = "mbr" if $partition eq "";
        $g->part_disk ($dev, $partition);
        $dev = "/dev/sda1";

        # Set the partition type byte if it's MBR and the filesystem
        # type is one that we know about.
        my $mbr_id;
        if ($partition eq "mbr" || $partition eq "msdos") {
            if ($type eq "msdos") {
                # According to Wikipedia.  However I have not actually
                # tried this.
                $mbr_id = 0x1;
            } elsif ($type =~ /^v?fat$/) {
                $mbr_id = 0xb;
            } elsif ($type eq "ntfs") {
                $mbr_id = 0x7;
            } elsif ($type =~ /^ext\d$/) {
                $mbr_id = 0x83;
            } elsif ($type eq "minix") {
                $mbr_id = 0x81;
            }
        }
        $g->part_set_mbr_id ("/dev/sda", 1, $mbr_id) if defined $mbr_id;
    }

    print STDERR "creating $type filesystem on $dev ...\n" if $debug;

    # Create the filesystem.
    if ($type ne "btrfs") {
        eval {
            $g->mkfs ($type, $dev);
        };
        if ($@) {
            # Provide more guidance in the error message (RHBZ#823883).
            print STDERR "'mkfs' (create filesystem) operation failed.\n";
            if ($type eq "fat") {
                print STDERR "Instead of 'fat', try 'vfat' (long filenames) or 'msdos' (short filenames).\n";
            } else {
                print STDERR "Is '$type' a correct filesystem type?\n";
            }
            die
        }
    } else {
        $g->mkfs_btrfs ([$dev], datatype => "single", metadata => "single");
    }

    # Mount it.

    # For vfat, add the utf8 mount option because we want to be able
    # to encode any non-ASCII characters into UCS2 which is what
    # modern vfat uses on disk (RHBZ#823885).
    my $options = "";
    $options = "utf8" if $type eq "vfat";

    $g->mount_options ($options, $dev, "/");

    # Copy the data in.
    my $ifile;

    if ($ifmt eq "directory") {
        my $pfile = create_pipe ();
        my $cmd = sprintf ("tar -C %s -cf - . > $pfile &",
                           shell_quote ($input));
        print STDERR "command: $cmd\n" if $debug;
        system ($cmd) == 0 or die __"tar: failed, see earlier messages\n";
        $ifile = $pfile;
    } else {
        if ($ifmt =~ /compress.d/) {
            my $pfile = create_pipe ();
            my $cmd;
            if ($ifmt =~ /compress'd/) {
                $cmd = sprintf ("uncompress -c %s > $pfile",
                                shell_quote ($input));
            } elsif ($ifmt =~ /gzip compressed/) {
                $cmd = sprintf ("gzip -cd %s", shell_quote ($input));
            } elsif ($ifmt =~ /bzip2 compressed/) {
                $cmd = sprintf ("bzip2 -cd %s", shell_quote ($input));
            } elsif ($ifmt =~ /xz compressed/) {
                $cmd = sprintf ("xz -cd %s", shell_quote ($input));
            } else {
                die __x("{f}: unknown input format: {fmt}\n",
                        f => $input, fmt => $ifmt);
            }
            $cmd .= " > $pfile &";
            print STDERR "command: $cmd\n" if $debug;
            system ($cmd) == 0 or
                die __"uncompress command failed, see earlier messages\n";
            $ifile = $pfile;
        } else {
            print STDERR "reading directly from $input\n" if $debug;
            $ifile = $input;
        }
    }

    if ($debug) {
        # For debugging, print statvfs before and after doing
        # the tar-in.
        my %stat = $g->statvfs ("/");
        print STDERR "Before uploading ...\n";
        print STDERR Dumper(\%stat);
    }

    print STDERR "Uploading from $ifile to / ...\n" if $debug;
    $g->tar_in ($ifile, "/");

    if ($debug) {
        my %stat = $g->statvfs ("/");
        print STDERR "After uploading ...\n";
        print STDERR Dumper(\%stat);
    }

    print STDERR "finishing off\n" if $debug;
    $g->shutdown ();
    $g->close ()
};
if ($@) {
    # Error: delete the output before exiting.
    my $err = $@;
    unlink $output;
    if ($err =~ /tar_in/) {
        print STDERR __"virt-make-fs: error copying contents into filesystem\nAn error here usually means that the program did not estimate the\nfilesystem size correctly.  Please read the BUGS section of the manpage.\n";
    }
    print STDERR $err;
    exit 1;
}

exit 0;

sub sizebytes
{
    local $_ = shift;
    my $unit = shift;

    $_ *= 1024 if $unit =~ /[KMGTPE]/;
    $_ *= 1024 if $unit =~ /[MGTPE]/;
    $_ *= 1024 if $unit =~ /[GTPE]/;
    $_ *= 1024 if $unit =~ /[TPE]/;
    $_ *= 1024 if $unit =~ /[PE]/;
    $_ *= 1024 if $unit =~ /[E]/;

    return floor($_);
}

sub create_pipe
{
    local $_;
    my $dir = tempdir (CLEANUP => 1);
    my $pipe = "$dir/pipe";
    mkfifo ($pipe, 0600) or
        die "mkfifo: $pipe: $!";
    return $pipe;
}

=head1 SHELL QUOTING

Libvirt guest names can contain arbitrary characters, some of which
have meaning to the shell such as C<#> and space.  You may need to
quote or escape these characters on the command line.  See the shell
manual page L<sh(1)> for details.

=head1 SEE ALSO

L<guestfish(1)>,
L<virt-format(1)>,
L<virt-resize(1)>,
L<virt-tar-in(1)>,
L<mkisofs(1)>,
L<genisoimage(1)>,
L<mksquashfs(1)>,
L<mke2fs(8)>,
L<resize2fs(8)>,
L<guestfs(3)>,
L<Sys::Guestfs(3)>,
L<http://libguestfs.org/>.

=head1 AUTHOR

Richard W.M. Jones L<http://people.redhat.com/~rjones/>

=head1 COPYRIGHT

Copyright (C) 2010-2012 Red Hat Inc.