From 1d6f1a9cb0fb1be8467d8e2c0fbda1b7eca70c66 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 1 Sep 2011 14:08:50 +0100 Subject: Move febootstrap into src/ subdirectory. Now we have src/ for febootstrap and helper/ for febootstrap-supermin-helper. --- src/.depend | 29 +++ src/Makefile.am | 103 +++++++++ src/config.ml.in | 30 +++ src/febootstrap.ml | 422 +++++++++++++++++++++++++++++++++++ src/febootstrap.pod | 315 ++++++++++++++++++++++++++ src/febootstrap_cmdline.ml | 96 ++++++++ src/febootstrap_cmdline.mli | 49 ++++ src/febootstrap_debian.ml | 162 ++++++++++++++ src/febootstrap_package_handlers.ml | 80 +++++++ src/febootstrap_package_handlers.mli | 66 ++++++ src/febootstrap_pacman.ml | 131 +++++++++++ src/febootstrap_utils.ml | 149 +++++++++++++ src/febootstrap_utils.mli | 73 ++++++ src/febootstrap_yum_rpm.ml | 237 ++++++++++++++++++++ 14 files changed, 1942 insertions(+) create mode 100644 src/.depend create mode 100644 src/Makefile.am create mode 100644 src/config.ml.in create mode 100644 src/febootstrap.ml create mode 100644 src/febootstrap.pod create mode 100644 src/febootstrap_cmdline.ml create mode 100644 src/febootstrap_cmdline.mli create mode 100644 src/febootstrap_debian.ml create mode 100644 src/febootstrap_package_handlers.ml create mode 100644 src/febootstrap_package_handlers.mli create mode 100644 src/febootstrap_pacman.ml create mode 100644 src/febootstrap_utils.ml create mode 100644 src/febootstrap_utils.mli create mode 100644 src/febootstrap_yum_rpm.ml (limited to 'src') diff --git a/src/.depend b/src/.depend new file mode 100644 index 0000000..e71f272 --- /dev/null +++ b/src/.depend @@ -0,0 +1,29 @@ +config.cmo: +config.cmx: +febootstrap_cmdline.cmi: +febootstrap_cmdline.cmo: config.cmo febootstrap_cmdline.cmi +febootstrap_cmdline.cmx: config.cmx febootstrap_cmdline.cmi +febootstrap_utils.cmi: +febootstrap_utils.cmo: febootstrap_cmdline.cmi febootstrap_utils.cmi +febootstrap_utils.cmx: febootstrap_cmdline.cmx febootstrap_utils.cmi +febootstrap_package_handlers.cmi: +febootstrap_package_handlers.cmo: febootstrap_utils.cmi \ + febootstrap_cmdline.cmi febootstrap_package_handlers.cmi +febootstrap_package_handlers.cmx: febootstrap_utils.cmx \ + febootstrap_cmdline.cmx febootstrap_package_handlers.cmi +febootstrap_yum_rpm.cmo: febootstrap_utils.cmi \ + febootstrap_package_handlers.cmi febootstrap_cmdline.cmi config.cmo +febootstrap_yum_rpm.cmx: febootstrap_utils.cmx \ + febootstrap_package_handlers.cmx febootstrap_cmdline.cmx config.cmx +febootstrap_debian.cmo: febootstrap_utils.cmi \ + febootstrap_package_handlers.cmi febootstrap_cmdline.cmi config.cmo +febootstrap_debian.cmx: febootstrap_utils.cmx \ + febootstrap_package_handlers.cmx febootstrap_cmdline.cmx config.cmx +febootstrap_pacman.cmo: febootstrap_utils.cmi \ + febootstrap_package_handlers.cmi febootstrap_cmdline.cmi config.cmo +febootstrap_pacman.cmx: febootstrap_utils.cmx \ + febootstrap_package_handlers.cmx febootstrap_cmdline.cmx config.cmx +febootstrap.cmo: febootstrap_utils.cmi febootstrap_package_handlers.cmi \ + febootstrap_cmdline.cmi config.cmo +febootstrap.cmx: febootstrap_utils.cmx febootstrap_package_handlers.cmx \ + febootstrap_cmdline.cmx config.cmx diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..b715c9f --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,103 @@ +# febootstrap Makefile.am +# (C) Copyright 2009-2011 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# Written by Richard W.M. Jones + +# Note these must be in build dependency order. +SOURCES = \ + config.ml \ + febootstrap_cmdline.mli \ + febootstrap_cmdline.ml \ + febootstrap_utils.mli \ + febootstrap_utils.ml \ + febootstrap_package_handlers.mli \ + febootstrap_package_handlers.ml \ + febootstrap_yum_rpm.ml \ + febootstrap_debian.ml \ + febootstrap_pacman.ml \ + febootstrap.ml + +CLEANFILES = *~ *.cmi *.cmo *.cmx *.o febootstrap + +EXTRA_DIST = \ + febootstrap.8 \ + febootstrap.pod \ + $(SOURCES) + +man_MANS = \ + febootstrap.8 + +bin_SCRIPTS = febootstrap + +SOURCES_ML = $(filter %.ml,$(SOURCES)) +BOBJECTS = $(SOURCES_ML:.ml=.cmo) +XOBJECTS = $(SOURCES_ML:.ml=.cmx) + +if !HAVE_OCAMLOPT +OBJECTS = $(BOBJECTS) +BEST = c +else +OBJECTS = $(XOBJECTS) +BEST = opt +endif + +OCAMLPACKAGES = -package unix,str +OCAMLFLAGS = -warn-error CDEFLMPSUVXYZ + +febootstrap: $(OBJECTS) + $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) -linkpkg \ + $^ -o $@ + +.mli.cmi: + $(OCAMLFIND) ocamlc $(OCAMLFLAGS) $(OCAMLPACKAGES) -c $< -o $@ +.ml.cmo: + $(OCAMLFIND) ocamlc $(OCAMLFLAGS) $(OCAMLPACKAGES) -c $< -o $@ +.ml.cmx: + $(OCAMLFIND) ocamlopt $(OCAMLFLAGS) $(OCAMLPACKAGES) -c $< -o $@ + +depend: .depend + +.depend: $(SOURCES) + rm -f $@ $@-t + $(OCAMLFIND) ocamldep $^ > $@-t + mv $@-t $@ + +include .depend + +SUFFIXES = .cmo .cmi .cmx .ml .mli .mll .mly + +if HAVE_PERLDOC + +febootstrap.8: febootstrap.pod + pod2man \ + --section 8 \ + -c "Virtualization Support" \ + --release "$(PACKAGE_NAME)-$(PACKAGE_VERSION)" \ + $< > $@ + +noinst_DATA = \ + ../html/febootstrap.8.html + +../html/febootstrap.8.html: febootstrap.pod + mkdir -p ../html + pod2html \ + --css 'pod.css' \ + --htmldir ../html \ + --outfile ../html/febootstrap.8.html \ + febootstrap.pod + +endif diff --git a/src/config.ml.in b/src/config.ml.in new file mode 100644 index 0000000..26a8e3d --- /dev/null +++ b/src/config.ml.in @@ -0,0 +1,30 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * @configure_input@ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +let package_name = "@PACKAGE_NAME@" +let package_version = "@PACKAGE_VERSION@" +let yum = "@YUM@" +let rpm = "@RPM@" +let yumdownloader = "@YUMDOWNLOADER@" +let aptitude = "@APTITUDE@" +let apt_cache = "@APT_CACHE@" +let dpkg = "@DPKG@" +let apt_cache_depends_recurse_broken = @APT_CACHE_DEPENDS_RECURSE_BROKEN@ +let pacman = "@PACMAN@" +let host_cpu = "@host_cpu@" diff --git a/src/febootstrap.ml b/src/febootstrap.ml new file mode 100644 index 0000000..7e48206 --- /dev/null +++ b/src/febootstrap.ml @@ -0,0 +1,422 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Febootstrap_package_handlers +open Febootstrap_utils +open Febootstrap_cmdline + +(* Create a temporary directory for use by all the functions in this file. *) +let tmpdir = tmpdir () + +let () = + debug "%s %s" Config.package_name Config.package_version; + + (* Instead of printing out warnings as we go along, accumulate them + * in lists and print them all out at the end. + *) + let warn_unreadable = ref [] in + + (* Determine which package manager this system uses. *) + check_system (); + let ph = get_package_handler () in + + debug "selected package handler: %s" (get_package_handler_name ()); + + (* Not --names: check files exist. *) + if not names_mode then ( + List.iter ( + fun pkg -> + if not (file_exists pkg) then ( + eprintf "febootstrap: %s: no such file (did you miss out the --names option?)\n" pkg; + exit 1 + ) + ) packages + ); + + (* --names: resolve the package list to a full list of package names + * (including dependencies). + *) + let packages = + if names_mode then ( + let packages = ph.ph_resolve_dependencies_and_download packages in + debug "resolved packages: %s" (String.concat " " packages); + packages + ) + else packages in + + (* Get the list of files. *) + let files = + List.flatten ( + List.map ( + fun pkg -> + let files = ph.ph_list_files pkg in + List.map (fun (filename, ft) -> filename, ft, pkg) files + ) packages + ) in + + (* Canonicalize the name of directories, so that /a and /a/ are the same. *) + let files = + List.map ( + fun (filename, ft, pkg) -> + let len = String.length filename in + let filename = + if len > 1 (* don't rewrite "/" *) && ft.ft_dir + && filename.[len-1] = '/' then + String.sub filename 0 (len-1) + else + filename in + (filename, ft, pkg) + ) files in + + (* Sort and combine duplicate files. *) + let files = + let files = List.sort compare files in + + let combine (name1, ft1, pkg1) (name2, ft2, pkg2) = + (* Rules for combining files. *) + if ft1.ft_config || ft2.ft_config then ( + (* It's a fairly frequent bug in Fedora for two packages to + * incorrectly list the same config file. Allow this, provided + * the size of both files is 0. + *) + if ft1.ft_size = 0 && ft2.ft_size = 0 then + (name1, ft1, pkg1) + else ( + eprintf "febootstrap: error: %s is a config file which is listed in two packages (%s, %s)\n" + name1 pkg1 pkg2; + exit 1 + ) + ) + else if (ft1.ft_dir || ft2.ft_dir) && (not (ft1.ft_dir && ft2.ft_dir)) then ( + eprintf "febootstrap: error: %s appears as both directory and ordinary file (%s, %s)\n" + name1 pkg1 pkg2; + exit 1 + ) + else if ft1.ft_ghost then + (name2, ft2, pkg2) + else + (name1, ft1, pkg1) + in + + let rec loop = function + | [] -> [] + | (name1, _, _ as f1) :: (name2, _, _ as f2) :: fs when name1 = name2 -> + let f = combine f1 f2 in loop (f :: fs) + | f :: fs -> f :: loop fs + in + loop files in + + (* Because we may have excluded some packages, and also because of + * distribution packaging errors, it's not necessarily true that a + * directory is created before each file in that directory. + * Determine those missing directories and add them now. + *) + let files = + let insert_dir, dir_seen = + let h = Hashtbl.create (List.length files) in + let insert_dir dir = Hashtbl.replace h dir true in + let dir_seen dir = Hashtbl.mem h dir in + insert_dir, dir_seen + in + let files = + List.map ( + fun (path, { ft_dir = is_dir }, _ as f) -> + if is_dir then + insert_dir path; + + let rec loop path = + let parent = Filename.dirname path in + if dir_seen parent then [] + else ( + insert_dir parent; + let newdir = (parent, { ft_dir = true; ft_config = false; + ft_ghost = false; ft_mode = 0o40755; + ft_size = 0 }, + "") in + newdir :: loop parent + ) + in + List.rev (f :: loop path) + ) files in + List.flatten files in + + (* Debugging. *) + debug "%d files and directories" (List.length files); + if false then ( + List.iter ( + fun (name, { ft_dir = dir; ft_ghost = ghost; ft_config = config; + ft_mode = mode; ft_size = size }, pkg) -> + printf "%s [%s%s%s%o %d] from %s\n" name + (if dir then "dir " else "") + (if ghost then "ghost " else "") + (if config then "config " else "") + mode size + pkg + ) files + ); + + (* Split the list of files into ones for hostfiles and ones for base image. *) + let p_hmac = Str.regexp "^\\..*\\.hmac$" in + + let hostfiles = ref [] + and baseimgfiles = ref [] in + List.iter ( + fun (path, {ft_dir = dir; ft_ghost = ghost; ft_config = config} ,_ as f) -> + let file = Filename.basename path in + + (* Ignore boot files, kernel, kernel modules. Supermin appliances + * are booted from external kernel and initrd, and + * febootstrap-supermin-helper copies the host kernel modules. + * Note we want to keep the /boot and /lib/modules directory entries. + *) + if string_prefix "/boot/" path then () + else if string_prefix "/lib/modules/" path then () + + (* Always write directory names to both output files. *) + else if dir then ( + hostfiles := f :: !hostfiles; + baseimgfiles := f :: !baseimgfiles; + ) + + (* Timezone configuration is config, but copy it from host system. *) + else if path = "/etc/localtime" then + hostfiles := f :: !hostfiles + + (* Ignore FIPS files (.*.hmac) (RHBZ#654638). *) + else if Str.string_match p_hmac file 0 then () + + (* Ghost files are created empty in the base image. *) + else if ghost then + baseimgfiles := f :: !baseimgfiles + + (* For config files we can't rely on the host-installed copy + * since the admin may have modified then. We have to get the + * original file from the package and put it in the base image. + *) + else if config then + baseimgfiles := f :: !baseimgfiles + + (* Anything else comes from the host. *) + else + hostfiles := f :: !hostfiles + ) files; + let hostfiles = List.rev !hostfiles + and baseimgfiles = List.rev !baseimgfiles in + + (* Write hostfiles. *) + + (* Regexps used below. *) + let p_ld_so = Str.regexp "^ld-[.0-9]+\\.so$" in + let p_libbfd = Str.regexp "^libbfd-.*\\.so$" in + let p_libgcc = Str.regexp "^libgcc_s-.*\\.so\\.\\([0-9]+\\)$" in + let p_libntfs3g = Str.regexp "^libntfs-3g\\.so\\..*$" in + let p_lib123so = Str.regexp "^lib\\(.*\\)-[-.0-9]+\\.so$" in + let p_lib123so123 = + Str.regexp "^lib\\(.*\\)-[-.0-9]+\\.so\\.\\([0-9]+\\)\\." in + let p_libso123 = Str.regexp "^lib\\(.*\\)\\.so\\.\\([0-9]+\\)\\." in + let ntfs3g_once = ref false in + + let chan = open_out (tmpdir // "hostfiles") in + List.iter ( + fun (path, {ft_dir = is_dir; ft_ghost = ghost; ft_config = config; + ft_mode = mode }, _) -> + let dir = Filename.dirname path in + let file = Filename.basename path in + + if is_dir then + fprintf chan "%s\n" path + + (* Warn about hostfiles which are unreadable by non-root. We + * won't be able to add those to the appliance at run time, but + * there's not much else we can do about it except get the + * distros to fix this nonsense. + *) + else if mode land 0o004 = 0 then + warn_unreadable := path :: !warn_unreadable + + (* Replace fixed numbers in some library names by wildcards. *) + else if Str.string_match p_ld_so file 0 then + fprintf chan "%s/ld-*.so\n" dir + + (* Special case for libbfd. *) + else if Str.string_match p_libbfd file 0 then + fprintf chan "%s/libbfd-*.so\n" dir + + (* Special case for libgcc_s--.so.N *) + else if Str.string_match p_libgcc file 0 then + fprintf chan "%s/libgcc_s-*.so.%s\n" dir (Str.matched_group 1 file) + + (* Special case for libntfs-3g.so.* *) + else if Str.string_match p_libntfs3g file 0 then ( + if not !ntfs3g_once then ( + fprintf chan "%s/libntfs-3g.so.*\n" dir; + ntfs3g_once := true + ) + ) + + (* libfoo-1.2.3.so *) + else if Str.string_match p_lib123so file 0 then + fprintf chan "%s/lib%s-*.so\n" dir (Str.matched_group 1 file) + + (* libfoo-1.2.3.so.123 (but NOT '*.so.N') *) + else if Str.string_match p_lib123so123 file 0 then + fprintf chan "%s/lib%s-*.so.%s.*\n" dir + (Str.matched_group 1 file) (Str.matched_group 2 file) + + (* libfoo.so.1.2.3 (but NOT '*.so.N') *) + else if Str.string_match p_libso123 file 0 then + fprintf chan "%s/lib%s.so.%s.*\n" dir + (Str.matched_group 1 file) (Str.matched_group 2 file) + + (* Anything else comes from the host. *) + else + fprintf chan "%s\n" path + ) hostfiles; + close_out chan; + + (* Write base.img. + * + * We have to create directories and copy files to tmpdir/root + * and then call out to cpio to construct the initrd. + *) + let rootdir = tmpdir // "root" in + mkdir rootdir 0o755; + List.iter ( + fun (path, { ft_dir = is_dir; ft_ghost = ghost; ft_config = config; + ft_mode = mode }, pkg) -> + (* Always write directory names to both output files. *) + if is_dir then ( + (* Directory permissions are fixed up below. *) + if path <> "/" then mkdir (rootdir // path) 0o755 + ) + + (* Ghost files are just touched with the correct perms. *) + else if ghost then ( + let chan = open_out (rootdir // path) in + close_out chan; + chmod (rootdir // path) (mode land 0o777 lor 0o400) + ) + + (* For config files we can't rely on the host-installed copy + * since the admin may have modified it. We have to get the + * original file from the package. + *) + else if config then ( + let outfile = ph.ph_get_file_from_package pkg path in + + (* Note that the output config file might not be a regular file. *) + let statbuf = lstat outfile in + + let destfile = rootdir // path in + + (* Depending on the file type, copy it to destination. *) + match statbuf.st_kind with + | S_REG -> + (* Unreadable files (eg. /etc/gshadow). Make readable. *) + if statbuf.st_perm = 0 then chmod outfile 0o400; + let cmd = + sprintf "cp %s %s" + (Filename.quote outfile) (Filename.quote destfile) in + run_command cmd; + chmod destfile (mode land 0o777 lor 0o400) + | S_LNK -> + let link = readlink outfile in + symlink link destfile + | S_DIR -> assert false + | S_CHR + | S_BLK + | S_FIFO + | S_SOCK -> + eprintf "febootstrap: error: %s: don't know how to handle this type of file\n" path; + exit 1 + ) + + else + assert false (* should not be reached *) + ) baseimgfiles; + + (* Fix up directory permissions, in reverse order. Since we don't + * want to have a read-only directory that we can't write into above. + *) + List.iter ( + fun (path, { ft_dir = is_dir; ft_mode = mode }, _) -> + if is_dir then chmod (rootdir // path) (mode land 0o777 lor 0o700) + ) (List.rev baseimgfiles); + + (* Construct the 'base.img' initramfs. Feed in the list of filenames + * partly because we conveniently have them, and partly because + * this results in a nice alphabetical ordering in the cpio file. + *) + (*let cmd = sprintf "ls -lR %s" rootdir in + ignore (Sys.command cmd);*) + let cmd = + sprintf "(cd %s && cpio --quiet -o -0 -H newc) > %s" + rootdir (tmpdir // "base.img") in + let chan = open_process_out cmd in + List.iter (fun (path, _, _) -> fprintf chan ".%s\000" path) baseimgfiles; + let stat = close_process_out chan in + (match stat with + | WEXITED 0 -> () + | WEXITED i -> + eprintf "febootstrap: command '%s' failed (returned %d), see earlier error messages\n" cmd i; + exit i + | WSIGNALED i -> + eprintf "febootstrap: command '%s' killed by signal %d" cmd i; + exit 1 + | WSTOPPED i -> + eprintf "febootstrap: command '%s' stopped by signal %d" cmd i; + exit 1 + ); + + (* Undo directory permissions, because rm -rf can't delete files in + * unreadable directories. + *) + List.iter ( + fun (path, { ft_dir = is_dir; ft_mode = mode }, _) -> + if is_dir then chmod (rootdir // path) 0o755 + ) (List.rev baseimgfiles); + + (* Print warnings. *) + if warnings then ( + (match !warn_unreadable with + | [] -> () + | paths -> + eprintf "febootstrap: warning: some host files are unreadable by non-root\n"; + eprintf "febootstrap: warning: get your distro to fix these files:\n"; + List.iter + (fun path -> eprintf "\t%s\n%!" path) + (List.sort compare paths) + ); + ); + + (* Near-atomically copy files to the final output directory. *) + debug "writing %s ..." (outputdir // "base.img"); + let cmd = + sprintf "mv %s %s" + (Filename.quote (tmpdir // "base.img")) + (Filename.quote (outputdir // "base.img")) in + run_command cmd; + debug "writing %s ..." (outputdir // "hostfiles"); + let cmd = + sprintf "mv %s %s" + (Filename.quote (tmpdir // "hostfiles")) + (Filename.quote (outputdir // "hostfiles")) in + run_command cmd diff --git a/src/febootstrap.pod b/src/febootstrap.pod new file mode 100644 index 0000000..ac97f48 --- /dev/null +++ b/src/febootstrap.pod @@ -0,0 +1,315 @@ +=encoding utf8 + +=head1 NAME + +febootstrap - Bootstrapping tool for creating supermin appliances + +=head1 SYNOPSIS + + febootstrap [-o OUTPUTDIR] --names LIST OF PKGS ... + febootstrap [-o OUTPUTDIR] PKG FILE NAMES ... + +=head1 DESCRIPTION + +febootstrap is a tool for building supermin appliances. These are +tiny appliances (similar to virtual machines), usually around 100KB in +size, which get fully instantiated on-the-fly in a fraction of a +second when you need to boot one of them. + +Originally "fe" in febootstrap stood for "Fedora", but this tool is +now distro-independent and can build supermin appliances for several +popular Linux distros, and adding support for others is reasonably +easy. + +Note that this manual page documents febootstrap 3.x which is a +complete rewrite and quite different from version 2.x. If you are +looking for the febootstrap 2.x tools, then this is not the right +place. + +=head2 BASIC OPERATION + +There are two modes for using febootstrap. With the I<--names> +parameter, febootstrap takes a list of package names and creates a +supermin appliance containing those packages and all dependencies that +those packages require. In this mode febootstrap usually needs +network access because it may need to consult package repositories in +order to work out dependencies and download packages. + +Without I<--names>, febootstrap takes a list of packages (ie. +filenames of locally available packages). This package set must be +complete and consistent with no dependencies outside the set of +packages you provide. In this mode febootstrap does not require any +network access. It works by looking at the package files themselves. + +By "package" we mean the RPM, DEB, (etc.) package. A package name +might be the fully qualified name (eg. C) +or some abbreviation (eg. C). The precise format of the +name and what abbreviations are allowed depends on the package +manager. + +The supermin appliance that febootstrap writes consists of two files +called C and C (see L +below). By default these are written to the current directory. If +you specify the I<-o OUTPUTDIR> option then these files are written to +the named directory instead (traditionally this directory is named +C but you can call it whatever you want). + +In all cases febootstrap can only build a supermin appliance which is +identical in distro, version and architecture to the host. It does +I do cross-builds. + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Display brief command line usage, and exit. + +=item B<--exclude REGEXP> + +After doing dependency resolution, exclude packages which match the +regular expression. + +This option is only used with I<--names>, and it can be given multiple +times on the command line. + +=item B<--names> + +Provide a list of package names, instead of providing packages +directly. In this mode febootstrap may require network access. See +L above. + +=item B<--no-warnings> + +Don't print warnings about packaging problems. + +=item B<-o outputdir> + +Select the output directory where the two supermin appliance files are +written (C and C). The default directory is the +current directory. Note that if this files exist already in the +output directory then they will be overwritten. + +=item B<--save-temps> + +Don't remove temporary files and directories on exit. This is useful +for debugging. + +=item B<-v> + +=item B<--verbose> + +Enable verbose messages. + +=item B<-V> + +=item B<--version> + +Print the package name and version number, and exit. + +=item B<--yum-config CONFIGFILE> + +(Yum/RPM package handler only). Use an alternate configuration file +instead of C. If you also want to specify alternate +repositories then you can put them in this file directly or add a +C option to this file. For more information on the file +format see L. + +=back + +=head1 SUPERMIN APPLIANCES + +Supermin appliances consist of just enough information to be able to +build an appliance containing the same operating system (Linux +version, distro, release etc) as the host OS. Since the host and +appliance share many common files such as C and +C there is no reason to ship these files in the +appliance. They can simply be read from the host on demand when the +appliance is launched. Therefore to save space we just store the +names of the host files that we want. + +There are some files which cannot just be copied from the host in this +way. These include configuration files which the host admin might +have edited. So along with the list of host files, we also store a +skeleton base image which contains these files and the outline +directory structure. + +Therefore the supermin appliance normally consists of at least two +control files: + +=over 4 + +=item B + +The list of files that are to be copied from the host. This is a +plain text file with one pathname per line. Directories are included +in this file. + +Paths can contain wildcards, which are expanded when the appliance +is created, eg: + + /etc/yum.repos.d/*.repo + +would copy all of the C<*.repo> files into the appliance. + +Each pathname in the file should start with a C character. (In +older versions of febootstrap, paths started with C<./> and were +relative to the root directory, but you should not do that in new +files). + +=item B + +This uncompressed cpio file contains the skeleton filesystem. Mostly +it contains directories and a few configuration files. + +All paths in the cpio file should be relative to the root directory of +the appliance. + +Note that unlike C, paths and directories in the base image +don't need to have any relationship to the host filesystem. + +=back + +=head2 RECONSTRUCTING THE APPLIANCE + +The separate tool L is used to +reconstruct an appliance from the hostfiles and base image files. + +This program in fact iterates recursively over the files and +directories passed to it. A common layout is: + + supermin.d/ + supermin.d/base.img + supermin.d/extra.img + supermin.d/hostfiles + +and then invoking febootstrap-supermin-helper with just the +C directory path as an argument. + +In this way extra files can be added to the appliance just by creating +another cpio file (C in the example above) and dropping it +into the directory. When the appliance is constructed, the extra +files will appear in the appliance. + +=head3 DIRECTORIES BEFORE FILES + +In order for febootstrap-supermin-helper to run quickly, it does not +know how to create directories automatically. Inside hostfiles and +the cpio files, directories must be specified before any files that +they contain. For example: + + /usr + /usr/sbin + /usr/sbin/serviced + +It is fine to list the same directory name multiple times. + +=head3 LEXICOGRAPHICAL ORDER + +febootstrap-supermin-helper visits the supermin control files in +lexicographical order. Thus in the example above, in the order +C -E C -E C. + +This has an important effect: files contained in later cpio files +overwrite earlier files, and directories do not need to be specified +if they have already been created in earlier control files. + +=head3 EXAMPLE OF CREATING EXTRA CPIO FILE + +You can create a file like C very easily using a shell +snippet similar to this one: + + cd $tmpdir + mkdir -p usr/sbin + cp /path/to/serviced usr/sbin/ + echo -e "usr\nusr/sbin\nusr/sbin/serviced" | + cpio --quiet -o -H newc > extra.img + rm -rf usr + +Notice how we instruct cpio to create intermediate directories. + +=head2 MINIMIZING THE SUPERMIN APPLIANCE + +You may want to "minimize" the supermin appliance in order to save +time and space when it is instantiated. Typically you might want to +remove documentation, info files, man pages and locales. We used to +provide a separate tool called C for this +purpose, but it is no longer provided. Instead you can post-process +C yourself to remove any files or directories that you +don't want (by removing lines from the file). Be careful what you +remove because files may be necessary for correct operation of the +appliance. + +For example: + + < supermin.d/hostfiles \ + grep -v '^/usr/share/man/' | + grep -v '^/usr/share/doc/' | + grep -v '^/usr/share/info/' > supermin.d/hostfiles-t + mv supermin.d/hostfiles-t supermin.d/hostfiles + +=head2 KERNEL AND KERNEL MODULES + +Usually the kernel and kernel modules are I included in the +supermin appliance. When the appliance is instantiated, the kernel +modules from the host kernel are copied in, and it is booted using the +host kernel. + +febootstrap-supermin-helper is able to choose the best host kernel +available to boot the appliance. Users can override this by setting +environment variables (see L). + +=head2 BOOTING AND CACHING THE SUPERMIN APPLIANCE + +For fastest boot times you should cache the output of +febootstrap-supermin-helper. See the libguestfs source file +C for an example of how this is done. + +=head2 ENFORCING AVAILABILITY OF HOSTFILES + +febootstrap-supermin-helper builds the appliance by copying in host +files as listed in C. For this to work those host files +must be available. We usually enforce this by adding requirements +(eg. RPM C lines) on the package that uses the supermin +appliance, so that package cannot be installed without pulling in the +dependent packages and thus making sure the host files are available. + +=head1 SEE ALSO + +L, +L, +L, +L. + +=head1 AUTHORS + +=over 4 + +=item * + +Richard W.M. Jones L + +=item * + +Matthew Booth L + +=back + +=head1 COPYRIGHT + +Copyright (C) 2009-2011 Red Hat Inc. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. diff --git a/src/febootstrap_cmdline.ml b/src/febootstrap_cmdline.ml new file mode 100644 index 0000000..667e297 --- /dev/null +++ b/src/febootstrap_cmdline.ml @@ -0,0 +1,96 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Printf + +let excludes = ref [] +let names_mode = ref false +let outputdir = ref "." +let packages = ref [] +let save_temps = ref false +let verbose = ref false +let warnings = ref true +let yum_config = ref None + +let print_version () = + printf "%s %s\n" Config.package_name Config.package_version; + exit 0 + +let add_exclude re = + excludes := Str.regexp re :: !excludes + +let set_yum_config str = + yum_config := Some str + +let argspec = Arg.align [ + "--exclude", Arg.String add_exclude, + "regexp Exclude packages matching regexp"; + "--names", Arg.Set names_mode, + " Specify set of root package names on command line"; + "--no-warnings", Arg.Clear warnings, + " Suppress warnings"; + "-o", Arg.Set_string outputdir, + "outputdir Set output directory (default: \".\")"; + "--save-temp", Arg.Set save_temps, + " Don't delete temporary files and directories on exit."; + "--save-temps", Arg.Set save_temps, + " Don't delete temporary files and directories on exit."; + "-v", Arg.Set verbose, + " Enable verbose output"; + "--verbose", Arg.Set verbose, + " Enable verbose output"; + "-V", Arg.Unit print_version, + " Print package name and version, and exit"; + "--version", Arg.Unit print_version, + " Print package name and version, and exit"; + "--yum-config", Arg.String set_yum_config, + "file Set alternate yum configuration file"; +] +let anon_fn str = + packages := str :: !packages + +let usage_msg = + "\ +febootstrap - bootstrapping tool for creating supermin appliances +Copyright (C) 2009-2010 Red Hat Inc. + +Usage: + febootstrap [-o OUTPUTDIR] --names LIST OF PKGS ... + febootstrap [-o OUTPUTDIR] PKG FILE NAMES ... + +For full instructions see the febootstrap(8) man page. + +Options:\n" + +let () = + Arg.parse argspec anon_fn usage_msg; + if !packages = [] then ( + eprintf "febootstrap: no packages listed on the command line\n"; + exit 1 + ) + +let excludes = List.rev !excludes +let names_mode = !names_mode +let outputdir = !outputdir +let packages = List.rev !packages +let save_temps = !save_temps +let verbose = !verbose +let warnings = !warnings +let yum_config = !yum_config + +let debug fs = ksprintf (fun str -> if verbose then print_endline str) fs diff --git a/src/febootstrap_cmdline.mli b/src/febootstrap_cmdline.mli new file mode 100644 index 0000000..d948d80 --- /dev/null +++ b/src/febootstrap_cmdline.mli @@ -0,0 +1,49 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(** Command line parsing. *) + +val debug : ('a, unit, string, unit) format4 -> 'a + (** Print string (like printf), but only if --verbose was given on + the command line. *) + +val excludes : Str.regexp list + (** List of package regexps to exclude. *) + +val names_mode : bool + (** True if [--names] was given on the command line (otherwise + {!packages} is a list of filenames). *) + +val outputdir : string + (** Output directory. *) + +val packages : string list + (** List of packages or package names as supplied on the command line. *) + +val save_temps : bool + (** True if [--save-temps] was given on the command line. *) + +val verbose : bool + (** True if [--verbose] was given on the command line. + See also {!debug}. *) + +val warnings : bool + (** If true, print warnings. [--no-warnings] sets this to false. *) + +val yum_config : string option + (** Yum configuration file. *) diff --git a/src/febootstrap_debian.ml b/src/febootstrap_debian.ml new file mode 100644 index 0000000..23f3593 --- /dev/null +++ b/src/febootstrap_debian.ml @@ -0,0 +1,162 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(* Debian support. *) + +open Unix +open Printf + +open Febootstrap_package_handlers +open Febootstrap_utils +open Febootstrap_cmdline + +(* Create a temporary directory for use by all the functions in this file. *) +let tmpdir = tmpdir () + +let debian_detect () = + file_exists "/etc/debian_version" && + Config.aptitude <> "no" && Config.apt_cache <> "no" && Config.dpkg <> "no" + +let rec debian_resolve_dependencies_and_download names = + let cmd = + sprintf "%s depends --recurse -i %s | grep -v '^[<[:space:]]'" + Config.apt_cache + (String.concat " " (List.map Filename.quote names)) in + let pkgs = run_command_get_lines cmd in + let pkgs = + if Config.apt_cache_depends_recurse_broken then + workaround_broken_apt_cache_depends_recurse (sort_uniq pkgs) + else + pkgs in + + (* Exclude packages matching [--exclude] regexps on the command line. *) + let pkgs = + List.filter ( + fun name -> + not (List.exists (fun re -> Str.string_match re name 0) excludes) + ) pkgs in + + (* Download the packages. *) + let cmd = + sprintf "umask 0000; cd %s && %s download %s" + (Filename.quote tmpdir) + Config.aptitude + (String.concat " " (List.map Filename.quote pkgs)) in + run_command cmd; + + (* Find out what aptitude downloaded. *) + let files = Sys.readdir tmpdir in + + let pkgs = List.map ( + fun pkg -> + (* Look for 'pkg_*.deb' in the list of files. *) + let pre = pkg ^ "_" in + let r = ref "" in + try + for i = 0 to Array.length files - 1 do + if string_prefix pre files.(i) then ( + r := files.(i); + files.(i) <- ""; + raise Exit + ) + done; + eprintf "febootstrap: aptitude: error: no file was downloaded corresponding to package %s\n" pkg; + exit 1 + with + Exit -> !r + ) pkgs in + + List.sort compare pkgs + +(* On Ubuntu 10.04 LTS, apt-cache depends --recurse is broken. It + * doesn't return the full list of dependencies. Therefore recurse + * into these dependencies one by one until we reach a fixpoint. + *) +and workaround_broken_apt_cache_depends_recurse names = + debug "workaround for broken 'apt-cache depends --recurse' command:\n %s" + (String.concat " " names); + + let names' = + List.map ( + fun name -> + let cmd = + sprintf "%s depends --recurse -i %s | grep -v '^[<[:space:]]'" + Config.apt_cache (Filename.quote name) in + run_command_get_lines cmd + ) names in + let names' = List.flatten names' in + let names' = sort_uniq names' in + if names <> names' then + workaround_broken_apt_cache_depends_recurse names' + else + names + +let debian_list_files pkg = + debug "unpacking %s ..." pkg; + + (* We actually need to extract the file in order to get the + * information about modes etc. + *) + let pkgdir = tmpdir // pkg ^ ".d" in + mkdir pkgdir 0o755; + let cmd = + sprintf "umask 0000; dpkg-deb --fsys-tarfile %s | (cd %s && tar xf -)" + (tmpdir // pkg) pkgdir in + run_command cmd; + + let cmd = sprintf "cd %s && find ." pkgdir in + let lines = run_command_get_lines cmd in + + let files = List.map ( + fun path -> + assert (path.[0] = '.'); + (* No leading '.' *) + let path = + if path = "." then "/" + else String.sub path 1 (String.length path - 1) in + + (* Find out what it is and get the canonical filename. *) + let statbuf = lstat (pkgdir // path) in + let is_dir = statbuf.st_kind = S_DIR in + + (* No per-file metadata like in RPM, but we can synthesize it + * from the path. + *) + let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in + + let mode = statbuf.st_perm in + + (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; + ft_ghost = false; ft_size = statbuf.st_size }) + ) lines in + + files + +(* Easy because we already unpacked the archive above. *) +let debian_get_file_from_package pkg file = + tmpdir // pkg ^ ".d" // file + +let () = + let ph = { + ph_detect = debian_detect; + ph_resolve_dependencies_and_download = + debian_resolve_dependencies_and_download; + ph_list_files = debian_list_files; + ph_get_file_from_package = debian_get_file_from_package; + } in + register_package_handler "debian" ph diff --git a/src/febootstrap_package_handlers.ml b/src/febootstrap_package_handlers.ml new file mode 100644 index 0000000..ad3a233 --- /dev/null +++ b/src/febootstrap_package_handlers.ml @@ -0,0 +1,80 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +open Febootstrap_utils +open Febootstrap_cmdline + +type package_handler = { + ph_detect : unit -> bool; + ph_resolve_dependencies_and_download : string list -> string list; + ph_list_files : string -> (string * file_type) list; + ph_get_file_from_package : string -> string -> string +} +and file_type = { + ft_dir : bool; + ft_config : bool; + ft_ghost : bool; + ft_mode : int; + ft_size : int; +} + +let tmpdir = tmpdir () + +let handlers = ref [] + +let register_package_handler name ph = + debug "registering package handler: %s" name; + handlers := (name, ph) :: !handlers + +let handler = ref None + +let check_system () = + try + handler := Some ( + List.find ( + fun (_, ph) -> + ph.ph_detect () + ) !handlers + ) + with Not_found -> + eprintf "\ +febootstrap: could not detect package manager used by this system or distro. + +If this is a new Linux distro, or not Linux, or a Linux distro that uses +an unusual packaging format then you may need to port febootstrap. If +you are expecting that febootstrap should work on this system or distro +then it may be that the package detection code is not working. +"; + exit 1 + +let rec get_package_handler () = + match !handler with + | Some (_, ph) -> ph + | None -> + check_system (); + get_package_handler () + +let rec get_package_handler_name () = + match !handler with + | Some (name, _) -> name + | None -> + check_system (); + get_package_handler_name () diff --git a/src/febootstrap_package_handlers.mli b/src/febootstrap_package_handlers.mli new file mode 100644 index 0000000..c28d81f --- /dev/null +++ b/src/febootstrap_package_handlers.mli @@ -0,0 +1,66 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(** Generic package handler code. *) + +type package_handler = { + ph_detect : unit -> bool; + (** Detect if the current system uses this package manager. *) + + ph_resolve_dependencies_and_download : string list -> string list; + (** [ph_resolve_dependencies_and_download pkgs] + Take a list of package names, and using the package manager + resolve those to a list of all the packages that are required + including dependencies. Download the full list of packages and + dependencies into a tmpdir. Return the list of full filenames. + + Note this should also process the [excludes] list. *) + + ph_list_files : string -> (string * file_type) list; + (** [ph_list_files pkg] lists the files and file metadata in the + package called [pkg] (a package file). *) + + ph_get_file_from_package : string -> string -> string; + (** [ph_get_file_from_package pkg file] extracts the + single named file [file] from [pkg]. The path of the + extracted file is returned. *) +} + +(* These file types are inspired by the metadata specifically + * stored by RPM. We should look at what other package formats + * can use too. + *) +and file_type = { + ft_dir : bool; (** Is a directory. *) + ft_config : bool; (** Is a configuration file. *) + ft_ghost : bool; (** Is a ghost (created empty) file. *) + ft_mode : int; (** File mode. *) + ft_size : int; (** File size. *) +} + +val register_package_handler : string -> package_handler -> unit + (** Register a package handler. *) + +val check_system : unit -> unit + (** Check which package manager this system uses. *) + +val get_package_handler : unit -> package_handler + (** Get the selected package manager for this system. *) + +val get_package_handler_name : unit -> string + (** Get the name of the selected package manager for this system. *) diff --git a/src/febootstrap_pacman.ml b/src/febootstrap_pacman.ml new file mode 100644 index 0000000..6691ebe --- /dev/null +++ b/src/febootstrap_pacman.ml @@ -0,0 +1,131 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(* ArchLinux support. *) + +open Unix +open Printf + +open Febootstrap_package_handlers +open Febootstrap_utils +open Febootstrap_cmdline + +(* Create a temporary directory for use by all the functions in this file. *) +let tmpdir = tmpdir () + +let pacman_detect () = + file_exists "/etc/arch-release" && + Config.pacman <> "no" + +let pacman_resolve_dependencies_and_download names = + let cmd = + sprintf "pactree -u %s | sort -u" + (String.concat " " (List.map Filename.quote names)) in + let pkgs = run_command_get_lines cmd in + + (* Exclude packages matching [--exclude] regexps on the command line. *) + let pkgs = + List.filter ( + fun name -> + not (List.exists (fun re -> Str.string_match re name 0) excludes) + ) pkgs in + + (* Download the packages. I could use wget `pacman -Sp`, but this + * narrows the pacman -Sy window + *) + List.iter ( + fun pkg -> + let cmd = + sprintf "umask 0000; cd %s && mkdir -p var/lib/pacman && fakeroot pacman -Syw --noconfirm --cachedir=$(pwd) --root=$(pwd) %s" + (Filename.quote tmpdir) + pkg in + if Sys.command cmd <> 0 then ( + (* The package is not in the main repos, check the aur *) + let cmd = + sprintf "umask 0000; cd %s && wget http://aur.archlinux.org/packages/%s/%s.tar.gz && tar xf %s.tar.gz && cd %s && makepkg && mv %s-*.pkg.tar.xz %s" + (Filename.quote tmpdir) + pkg + pkg + pkg + pkg + pkg + (Filename.quote tmpdir) in + run_command cmd; + ) + ) pkgs; + + List.sort compare pkgs + +let pacman_list_files pkg = + debug "unpacking %s ..." pkg; + + (* We actually need to extract the file in order to get the + * information about modes etc. + *) + let pkgdir = tmpdir // pkg ^ ".d" in + mkdir pkgdir 0o755; + let cmd = + sprintf "pacman -Q %s | awk '{print $2}'" + pkg in + let ver = List.hd (run_command_get_lines cmd) in + let cmd = + sprintf "umask 0000; fakeroot tar -xf %s-%s* -C %s" + (Filename.quote tmpdir // pkg ) ver (Filename.quote pkgdir) in + run_command cmd; + + let cmd = sprintf "cd %s && find ." pkgdir in + let lines = run_command_get_lines cmd in + + let files = List.map ( + fun path -> + assert (path.[0] = '.'); + (* No leading '.' *) + let path = + if path = "." then "/" + else String.sub path 1 (String.length path - 1) in + + (* Find out what it is and get the canonical filename. *) + let statbuf = lstat (pkgdir // path) in + let is_dir = statbuf.st_kind = S_DIR in + + (* No per-file metadata like in RPM, but we can synthesize it + * from the path. + *) + let config = statbuf.st_kind = S_REG && string_prefix "/etc/" path in + + let mode = statbuf.st_perm in + + (path, { ft_dir = is_dir; ft_config = config; ft_mode = mode; + ft_ghost = false; ft_size = statbuf.st_size }) + ) lines in + + files + +(* Easy because we already unpacked the archive above. *) +let pacman_get_file_from_package pkg file = + tmpdir // pkg ^ ".d" // file + +let () = + let ph = { + ph_detect = pacman_detect; + ph_resolve_dependencies_and_download = + pacman_resolve_dependencies_and_download; + ph_list_files = pacman_list_files; + ph_get_file_from_package = pacman_get_file_from_package; + } in + register_package_handler "pacman" ph diff --git a/src/febootstrap_utils.ml b/src/febootstrap_utils.ml new file mode 100644 index 0000000..2265753 --- /dev/null +++ b/src/febootstrap_utils.ml @@ -0,0 +1,149 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +open Unix +open Printf + +let (//) = Filename.concat + +let file_exists name = + try access name [F_OK]; true + with Unix_error _ -> false + +let dir_exists name = + try (stat name).st_kind = S_DIR + with Unix_error _ -> false + +let rec uniq ?(cmp = Pervasives.compare) = function + | [] -> [] + | [x] -> [x] + | x :: y :: xs when cmp x y = 0 -> + uniq ~cmp (x :: xs) + | x :: y :: xs -> + x :: uniq ~cmp (y :: xs) + +let sort_uniq ?(cmp = Pervasives.compare) xs = + let xs = List.sort cmp xs in + let xs = uniq ~cmp xs in + xs + +let rec input_all_lines chan = + try let line = input_line chan in line :: input_all_lines chan + with End_of_file -> [] + +let run_command_get_lines cmd = + let chan = open_process_in cmd in + let lines = input_all_lines chan in + let stat = close_process_in chan in + (match stat with + | WEXITED 0 -> () + | WEXITED i -> + eprintf "febootstrap: command '%s' failed (returned %d), see earlier error messages\n" cmd i; + exit i + | WSIGNALED i -> + eprintf "febootstrap: command '%s' killed by signal %d" cmd i; + exit 1 + | WSTOPPED i -> + eprintf "febootstrap: command '%s' stopped by signal %d" cmd i; + exit 1 + ); + lines + +let run_command cmd = + if Sys.command cmd <> 0 then ( + eprintf "febootstrap: %s: command failed, see earlier errors\n" cmd; + exit 1 + ) + +let run_python code args = + let cmd = sprintf "python -c %s %s" + (Filename.quote code) + (String.concat " " (List.map Filename.quote args)) in + if Sys.command cmd <> 0 then ( + eprintf "febootstrap: external python program failed, see earlier error messages\n"; + exit 1 + ) + +let tmpdir () = + let chan = open_in "/dev/urandom" in + let data = String.create 16 in + really_input chan data 0 (String.length data); + close_in chan; + let data = Digest.to_hex (Digest.string data) in + (* Note this is secure, because if the name already exists, even as a + * symlink, mkdir(2) will fail. + *) + let tmpdir = Filename.temp_dir_name // sprintf "febootstrap%s.tmp" data in + Unix.mkdir tmpdir 0o700; + + (* Only remove the directory if --save-temps was *not* specified. *) + if not Febootstrap_cmdline.save_temps then + at_exit + (fun () -> + let cmd = sprintf "rm -rf %s" (Filename.quote tmpdir) in + ignore (Sys.command cmd)); + + tmpdir + +let rec find s sub = + let len = String.length s in + let sublen = String.length sub in + let rec loop i = + if i <= len-sublen then ( + let rec loop2 j = + if j < sublen then ( + if s.[i+j] = sub.[j] then loop2 (j+1) + else -1 + ) else + i (* found *) + in + let r = loop2 0 in + if r = -1 then loop (i+1) else r + ) else + -1 (* not found *) + in + loop 0 + +let rec string_split sep str = + let len = String.length str in + let seplen = String.length sep in + let i = find str sep in + if i = -1 then [str] + else ( + let s' = String.sub str 0 i in + let s'' = String.sub str (i+seplen) (len-i-seplen) in + s' :: string_split sep s'' + ) + +let string_prefix p str = + let len = String.length str in + let plen = String.length p in + len >= plen && String.sub str 0 plen = p + +let path_prefix p path = + let len = String.length path in + let plen = String.length p in + path = p || (len > plen && String.sub path 0 (plen+1) = (p ^ "/")) + +let rec filter_map f = function + | [] -> [] + | x :: xs -> + let x = f x in + match x with + | None -> filter_map f xs + | Some x -> x :: filter_map f xs diff --git a/src/febootstrap_utils.mli b/src/febootstrap_utils.mli new file mode 100644 index 0000000..3087ee0 --- /dev/null +++ b/src/febootstrap_utils.mli @@ -0,0 +1,73 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(** Utilities. *) + +val file_exists : string -> bool + (** Return [true] iff file exists. *) + +val dir_exists : string -> bool + (** Return [true] iff dir exists. *) + +val uniq : ?cmp:('a -> 'a -> int) -> 'a list -> 'a list + (** Uniquify a list (the list must be sorted first). *) + +val sort_uniq : ?cmp:('a -> 'a -> int) -> 'a list -> 'a list + (** Sort and uniquify a list. *) + +val input_all_lines : in_channel -> string list + (** Input all lines from a channel, returning a list of lines. *) + +val run_command_get_lines : string -> string list + (** Run the command and read the list of lines that it prints to stdout. *) + +val run_command : string -> unit + (** Run a command using {!Sys.command} and exit if it fails. Be careful + when constructing the command to properly quote any arguments + (using {!Filename.quote}). *) + +val run_python : string -> string list -> unit + (** [run_python code args] runs Python [code] with arguments [args]. + This does not return anything, but exits with an error message + if the Python code returns an error. *) + +val tmpdir : unit -> string + (** [tmpdir ()] returns a newly created temporary directory. The + tmp directory is automatically removed when the program exits. + Note that a fresh temporary directory is returned each time you + call this function. *) + +val (//) : string -> string -> string + (** [x // y] concatenates file paths [x] and [y] into a single path. *) + +val find : string -> string -> int +(** [find str sub] searches for [sub] in [str], returning the index + or -1 if not found. *) + +val string_split : string -> string -> string list + (** [string_split sep str] splits [str] at [sep]. *) + +val string_prefix : string -> string -> bool + (** [string_prefix prefix str] returns true iff [str] starts with [prefix]. *) + +val path_prefix : string -> string -> bool + (** [path_prefix prefix path] returns true iff [path] is [prefix] or + [path] starts with [prefix/]. *) + +val filter_map : ('a -> 'b option) -> 'a list -> 'b list + (** map + filter *) diff --git a/src/febootstrap_yum_rpm.ml b/src/febootstrap_yum_rpm.ml new file mode 100644 index 0000000..ca74f4a --- /dev/null +++ b/src/febootstrap_yum_rpm.ml @@ -0,0 +1,237 @@ +(* febootstrap 3 + * Copyright (C) 2009-2010 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + *) + +(* Yum and RPM support. *) + +open Unix +open Printf + +open Febootstrap_package_handlers +open Febootstrap_utils +open Febootstrap_cmdline + +(* Create a temporary directory for use by all the functions in this file. *) +let tmpdir = tmpdir () + +let yum_rpm_detect () = + (file_exists "/etc/redhat-release" || file_exists "/etc/fedora-release") && + Config.yum <> "no" && Config.rpm <> "no" + +let yum_rpm_resolve_dependencies_and_download names = + (* Liberate this data from python. *) + let tmpfile = tmpdir // "names.tmp" in + let py = sprintf " +import yum +import yum.misc +import sys + +yb = yum.YumBase () +yb.preconf.debuglevel = %d +yb.preconf.errorlevel = %d +if %s: + yb.preconf.fn = %S +yb.setCacheDir () + +# Look up the base packages from the command line. +deps = dict () +pkgs = yb.pkgSack.returnPackages (patterns=sys.argv[1:]) +for pkg in pkgs: + deps[pkg] = False + +# Recursively find all the dependencies. +stable = False +while not stable: + stable = True + for pkg in deps.keys(): + if deps[pkg] == False: + deps[pkg] = [] + stable = False + for r in pkg.requires: + ps = yb.whatProvides (r[0], r[1], r[2]) + best = yb._bestPackageFromList (ps.returnPackages ()) + if best.name != pkg.name: + deps[pkg].append (best) + if not deps.has_key (best): + deps[best] = False + deps[pkg] = yum.misc.unique (deps[pkg]) + +# Write it to a file because yum spews garbage on stdout. +f = open (%S, \"w\") +for pkg in deps.keys (): + f.write (\"%%s %%s %%s %%s %%s\\n\" %% + (pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch)) +f.close () +" + (if verbose then 1 else 0) + (if verbose then 1 else 0) + (match yum_config with None -> "False" | Some _ -> "True") + (match yum_config with None -> "" | Some filename -> filename) + tmpfile in + run_python py names; + let chan = open_in tmpfile in + let lines = input_all_lines chan in + close_in chan; + + (* Get fields. *) + let pkgs = + List.map ( + fun line -> + match string_split " " line with + | [name; epoch; version; release; arch] -> + name, int_of_string epoch, version, release, arch + | _ -> + eprintf "febootstrap: bad output from python script: '%s'" line; + exit 1 + ) lines in + + (* Something of a hack for x86_64: exclude all i[3456]86 packages. *) + let pkgs = + if Config.host_cpu = "x86_64" then ( + List.filter ( + function (_, _, _, _, ("i386"|"i486"|"i586"|"i686")) -> false + | _ -> true + ) pkgs + ) + else pkgs in + + (* Drop the kernel package to save time. *) + let pkgs = + List.filter (function ("kernel",_,_,_,_) -> false | _ -> true) pkgs in + + (* Exclude packages matching [--exclude] regexps on the command line. *) + let pkgs = + List.filter ( + fun (name, _, _, _, _) -> + not (List.exists (fun re -> Str.string_match re name 0) excludes) + ) pkgs in + + (* Sort the list of packages, and remove duplicates (by name). + * XXX This is not quite right: we really want to keep the latest + * package if duplicates are found, but that would require a full + * version compare function. + *) + let pkgs = List.sort (fun a b -> compare b a) pkgs in + let pkgs = + let cmp (name1, _, _, _, _) (name2, _, _, _, _) = compare name1 name2 in + uniq ~cmp pkgs in + let pkgs = List.sort compare pkgs in + + (* Construct package names. *) + let pkgnames = List.map ( + function + | name, 0, version, release, arch -> + sprintf "%s-%s-%s.%s" name version release arch + | name, epoch, version, release, arch -> + sprintf "%d:%s-%s-%s.%s" epoch name version release arch + ) pkgs in + + if pkgnames = [] then ( + eprintf "febootstrap: yum-rpm: error: no packages to download\n"; + exit 1 + ); + + let cmd = sprintf "yumdownloader%s%s --destdir %s %s" + (if verbose then "" else " --quiet") + (match yum_config with None -> "" + | Some filename -> sprintf " -c %s" filename) + (Filename.quote tmpdir) + (String.concat " " (List.map Filename.quote pkgnames)) in + run_command cmd; + + (* Return list of package filenames. *) + List.map ( + (* yumdownloader doesn't include epoch in the filename *) + fun (name, _, version, release, arch) -> + sprintf "%s/%s-%s-%s.%s.rpm" tmpdir name version release arch + ) pkgs + +let rec yum_rpm_list_files pkg = + (* Run rpm -qlp with some extra magic. *) + let cmd = + sprintf "rpm -q --qf '[%%{FILENAMES} %%{FILEFLAGS:fflags} %%{FILEMODES} %%{FILESIZES}\\n]' -p %s" + pkg in + let lines = run_command_get_lines cmd in + + let files = + filter_map ( + fun line -> + match string_split " " line with + | [filename; flags; mode; size] -> + let test_flag = String.contains flags in + let mode = int_of_string mode in + let size = int_of_string size in + if test_flag 'd' then None (* ignore documentation *) + else + Some (filename, { + ft_dir = mode land 0o40000 <> 0; + ft_ghost = test_flag 'g'; ft_config = test_flag 'c'; + ft_mode = mode; ft_size = size; + }) + | _ -> + eprintf "febootstrap: bad output from rpm command: '%s'" line; + exit 1 + ) lines in + + (* I've never understood why the base packages like 'filesystem' don't + * contain any /dev nodes at all. This leaves every program that + * bootstraps RPMs to create a varying set of device nodes themselves. + * This collection was copied from mock/backend.py. + *) + let files = + let b = Filename.basename pkg in + if string_prefix "filesystem-" b then ( + let dirs = [ "/proc"; "/sys"; "/dev"; "/dev/pts"; "/dev/shm"; + "/dev/mapper" ] in + let dirs = + List.map (fun name -> + name, { ft_dir = true; ft_ghost = false; + ft_config = false; ft_mode = 0o40755; + ft_size = 0 }) dirs in + let devs = [ "/dev/null"; "/dev/full"; "/dev/zero"; "/dev/random"; + "/dev/urandom"; "/dev/tty"; "/dev/console"; + "/dev/ptmx"; "/dev/stdin"; "/dev/stdout"; "/dev/stderr" ] in + (* No need to set the mode because these will go into hostfiles. *) + let devs = + List.map (fun name -> + name, { ft_dir = false; ft_ghost = false; + ft_config = false; ft_mode = 0o644; + ft_size = 0 }) devs in + dirs @ devs @ files + ) else files in + + files + +let yum_rpm_get_file_from_package pkg file = + debug "extracting %s from %s ..." file (Filename.basename pkg); + + let outfile = tmpdir // file in + let cmd = + sprintf "umask 0000; rpm2cpio %s | (cd %s && cpio --quiet -id .%s)" + (Filename.quote pkg) (Filename.quote tmpdir) (Filename.quote file) in + run_command cmd; + outfile + +let () = + let ph = { + ph_detect = yum_rpm_detect; + ph_resolve_dependencies_and_download = + yum_rpm_resolve_dependencies_and_download; + ph_list_files = yum_rpm_list_files; + ph_get_file_from_package = yum_rpm_get_file_from_package; + } in + register_package_handler "yum-rpm" ph -- cgit