diff options
| -rw-r--r-- | src/virtBootstrap/sources.py | 335 | ||||
| -rw-r--r-- | src/virtBootstrap/utils.py | 346 | ||||
| -rwxr-xr-x | src/virtBootstrap/virt_bootstrap.py | 22 |
3 files changed, 369 insertions, 334 deletions
diff --git a/src/virtBootstrap/sources.py b/src/virtBootstrap/sources.py index db4fe36..beb671a 100644 --- a/src/virtBootstrap/sources.py +++ b/src/virtBootstrap/sources.py @@ -20,313 +20,20 @@ Class definitions which process container image or archive from source and unpack them in destination directory. """ -import errno -import fcntl -import hashlib -import json import select import shutil -import tempfile import getpass import os import logging from subprocess import CalledProcessError, PIPE, Popen +from virtBootstrap import utils + # pylint: disable=invalid-name # Create logger logger = logging.getLogger(__name__) -# Default virtual size of qcow2 image -DEF_QCOW2_SIZE = '5G' -if os.geteuid() == 0: - LIBVIRT_CONN = "lxc:///" - DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images" -else: - LIBVIRT_CONN = "qemu:///session" - DEFAULT_IMG_DIR = os.environ['HOME'] - DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images" - - -def checksum(path, sum_type, sum_expected): - """ - Validate file using checksum. - """ - algorithm = getattr(hashlib, sum_type) - try: - handle = open(path, 'rb') - content = handle.read() - handle.close() - - actual = algorithm(content).hexdigest() - if not actual == sum_expected: - logger.warning("File '%s' has invalid hash sum.\nExpected: %s\n" - "Actual: %s", path, sum_expected, actual) - return False - return True - except Exception as err: - logger.warning("Error occured while validating " - "the hash sum of file: %s\n%s", path, err) - return False - - -def execute(cmd): - """ - Execute command and log debug message. - """ - cmd_str = ' '.join(cmd) - logger.debug("Call command:\n%s", cmd_str) - - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - output, err = proc.communicate() - - if output: - logger.debug("Stdout:\n%s", output) - if err: - logger.debug("Stderr:\n%s", err) - - if proc.returncode != 0: - raise CalledProcessError(proc.returncode, cmd_str) - - -def safe_untar(src, dest): - """ - Extract tarball within LXC container for safety. - """ - virt_sandbox = ['virt-sandbox', - '-c', LIBVIRT_CONN, - '-m', 'host-bind:/mnt=' + dest] # Bind destination folder - - # Compression type is auto detected from tar - # Exclude files under /dev to avoid "Cannot mknod: Operation not permitted" - params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*'] - execute(virt_sandbox + params) - - -def format_number(number): - """ - Turn numbers into human-readable metric-like numbers - """ - symbols = ['', 'KiB', 'MiB', 'GiB'] - step = 1024.0 - thresh = 999 - depth = 0 - max_depth = len(symbols) - 1 - - while number > thresh and depth < max_depth: - depth = depth + 1 - number = number / step - - if int(number) == float(number): - fmt = '%i %s' - else: - fmt = '%.2f %s' - - return(fmt % (number or 0, symbols[depth])) - - -def log_layer_extract(layer, current, total, progress): - """ - Create log message on layer extract. - """ - sum_type, sum_value, layer_file, layer_size = layer - progress("Extracting layer (%s/%s) with size: %s" - % (current, total, format_number(layer_size)), logger=logger) - logger.debug('Untar layer: (%s:%s) %s', sum_type, sum_value, layer_file) - - -def untar_layers(layers_list, dest_dir, progress): - """ - Untar each of layers from container image. - """ - nlayers = len(layers_list) - for index, layer in enumerate(layers_list): - log_layer_extract(layer, index + 1, nlayers, progress) - layer_file = layer[2] - - # Extract layer tarball into destination directory - safe_untar(layer_file, dest_dir) - - # Update progress value - progress(value=(float(index + 1) / nlayers * 50) + 50) - - -def get_mime_type(path): - """ - Get the mime type of a file. - """ - return Popen(["/usr/bin/file", "--mime-type", path], - stdout=PIPE).communicate()[0].split()[1] - - -def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE): - """ - Create qcow2 image from tarball. - """ - qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size] - - if not backing_file: - logger.info("Creating base qcow2 image") - execute(qemu_img_cmd) - - logger.info("Formatting qcow2 image") - execute(['virt-format', - '--format=qcow2', - '--partition=none', - '--filesystem=ext3', - '-a', layer_file]) - else: - # Add backing chain - qemu_img_cmd.insert(2, "-b") - qemu_img_cmd.insert(3, backing_file) - - logger.info("Creating qcow2 image with backing chain") - execute(qemu_img_cmd) - - # Get mime type of archive - mime_tar_file = get_mime_type(tar_file) - logger.debug("Detected mime type of archive: %s", mime_tar_file) - - # Extract tarball using "tar-in" command from libguestfs - tar_in_cmd = ["guestfish", - "-a", layer_file, - '-m', '/dev/sda', - 'tar-in', tar_file, "/"] - - compression_fmts = {'x-gzip': 'gzip', 'gzip': 'gzip', - 'x-xz': 'xz', - 'x-bzip2': 'bzip2', - 'x-compress': 'compress', - 'x-lzop': 'lzop'} - - # Check if tarball is compressed - mime_parts = mime_tar_file.split('/') - if mime_parts[0] == 'application' and \ - mime_parts[1] in compression_fmts: - tar_in_cmd.append('compress:' + compression_fmts[mime_parts[1]]) - - # Execute virt-tar-in command - execute(tar_in_cmd) - - -def extract_layers_in_qcow2(layers_list, dest_dir, progress): - """ - Extract docker layers in qcow2 images with backing chains. - """ - qcow2_backing_file = None - - nlayers = len(layers_list) - for index, layer in enumerate(layers_list): - log_layer_extract(layer, index + 1, nlayers, progress) - tar_file = layer[2] - - # Name format for the qcow2 image - qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index) - # Create the image layer - create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file) - # Keep the file path for the next layer - qcow2_backing_file = qcow2_layer_file - - # Update progress value - progress(value=(float(index + 1) / nlayers * 50) + 50) - - -def get_image_dir(no_cache=False): - """ - Get the directory where image layers are stored. - - @param no_cache: Boolean, indicates whether to use temporary directory - """ - if no_cache: - return tempfile.mkdtemp('virt-bootstrap') - - if not os.path.exists(DEFAULT_IMG_DIR): - os.makedirs(DEFAULT_IMG_DIR) - - return DEFAULT_IMG_DIR - - -def get_image_details(src, raw=False, - insecure=False, username=False, password=False): - """ - Return details of container image from "skopeo inspect" commnad. - """ - cmd = ['skopeo', 'inspect', src] - if raw: - cmd.append('--raw') - if insecure: - cmd.append('--tls-verify=false') - if username and password: - cmd.append("--creds=%s:%s" % (username, password)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - output, error = proc.communicate() - if error: - raise ValueError("Image could not be retrieved:", error) - return json.loads(output) - - -def size_to_bytes(string, fmt): - """ - Convert human readable formats to bytes. - """ - formats = {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4} - return (string * pow(1024, formats[fmt.upper()]) if fmt in formats - else False) - - -def is_new_layer_message(line): - """ - Return T/F whether a line in skopeo's progress is indicating - start the process of new image layer. - - Reference: - - https://github.com/containers/image/blob/master/copy/copy.go#L464 - - https://github.com/containers/image/blob/master/copy/copy.go#L459 - """ - return line.startswith('Copying blob') or line.startswith('Skipping fetch') - - -def is_layer_config_message(line): - """ - Return T/F indicating whether the message from skopeo copies the manifest - file. - - Reference: - - https://github.com/containers/image/blob/master/copy/copy.go#L414 - """ - return line.startswith('Copying config') - - -def make_async(fd): - """ - Add the O_NONBLOCK flag to a file descriptor. - """ - fcntl.fcntl(fd, fcntl.F_SETFL, - fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) - - -def read_async(fd): - """ - Read some data from a file descriptor, ignoring EAGAIN errors - """ - try: - return fd.read() - except IOError as e: - if e.errno != errno.EAGAIN: - raise - else: - return '' - - -def str2float(element): - """ - Convert string to float or return None. - """ - try: - return float(element) - except ValueError: - return None - class FileSource(object): """ @@ -350,7 +57,7 @@ class FileSource(object): if self.output_format == 'dir': self.progress("Extracting files into destination directory", value=0, logger=logger) - safe_untar(self.path, dest) + utils.safe_untar(self.path, dest) elif self.output_format == 'qcow2': # Remove the old path @@ -360,7 +67,7 @@ class FileSource(object): self.progress("Extracting files into qcow2 image", value=0, logger=logger) - create_qcow2(self.path, qcow2_file) + utils.create_qcow2(self.path, qcow2_file) else: raise Exception("Unknown format:" + self.output_format) @@ -410,13 +117,13 @@ class DockerSource(object): image = image[:-1] self.url = "docker://" + registry + image - self.images_dir = get_image_dir(self.no_cache) + self.images_dir = utils.get_image_dir(self.no_cache) # Retrive manifest from registry - self.manifest = get_image_details(self.url, raw=True, - insecure=self.insecure, - username=self.username, - password=self.password) + self.manifest = utils.get_image_details(self.url, raw=True, + insecure=self.insecure, + username=self.username, + password=self.password) # Get layers' digest, sum_type, size and file_path in a list self.layers = [] @@ -456,11 +163,11 @@ class DockerSource(object): # Wait for data to become available stdout = select.select([proc.stdout], [], [])[0] # Split output into line - output = read_async(stdout[0]).strip().split('\n') + output = utils.read_async(stdout[0]).strip().split('\n') for line in output: if line: # is not empty line_split = line.split() - if is_new_layer_message(line): + if utils.is_new_layer_message(line): current_layer += 1 self.progress("Downloading layer (%s/%s)" % (current_layer, total_layers_num)) @@ -472,7 +179,7 @@ class DockerSource(object): total_layers_num) # Stop parsing when manifest is copied. - elif is_layer_config_message(line): + elif utils.is_layer_config_message(line): break else: continue # continue if the inner loop didn't break @@ -489,12 +196,12 @@ class DockerSource(object): Calculate percentage and update the progress of virt-bootstrap. """ - d_size, d_format = str2float(line_split[0]), line_split[1] - t_size, t_format = str2float(line_split[3]), line_split[4] + d_size, d_format = utils.str2float(line_split[0]), line_split[1] + t_size, t_format = utils.str2float(line_split[3]), line_split[4] if d_size and t_size: - downloaded_size = size_to_bytes(d_size, d_format) - total_size = size_to_bytes(t_size, t_format) + downloaded_size = utils.size_to_bytes(d_size, d_format) + total_size = utils.size_to_bytes(t_size, t_format) if downloaded_size and total_size: try: self.progress(value=(50 @@ -510,7 +217,7 @@ class DockerSource(object): proc = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True) # Without `make_async`, `fd.read` in `read_async` blocks. - make_async(proc.stdout) + utils.make_async(proc.stdout) if not self.parse_output(proc): raise CalledProcessError(cmd, proc.stderr.read()) @@ -523,7 +230,7 @@ class DockerSource(object): for sum_type, sum_expected, path, _ignore in self.layers: logger.debug("Checking layer: %s", path) if not (os.path.exists(path) - and checksum(path, sum_type, sum_expected)): + and utils.checksum(path, sum_type, sum_expected)): return False return True @@ -551,11 +258,11 @@ class DockerSource(object): if self.output_format == 'dir': self.progress("Extracting container layers", value=50, logger=logger) - untar_layers(self.layers, dest, self.progress) + utils.untar_layers(self.layers, dest, self.progress) elif self.output_format == 'qcow2': self.progress("Extracting container layers into qcow2 images", value=50, logger=logger) - extract_layers_in_qcow2(self.layers, dest, self.progress) + utils.extract_layers_in_qcow2(self.layers, dest, self.progress) else: raise Exception("Unknown format:" + self.output_format) @@ -569,5 +276,5 @@ class DockerSource(object): finally: # Clean up - if self.no_cache and self.images_dir != DEFAULT_IMG_DIR: + if self.no_cache and self.images_dir != utils.DEFAULT_IMG_DIR: shutil.rmtree(self.images_dir) diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py new file mode 100644 index 0000000..ae5d440 --- /dev/null +++ b/src/virtBootstrap/utils.py @@ -0,0 +1,346 @@ +# Authors: +# Cedric Bosdonnat <cbosdonnat@suse.com> +# Radostin Stoyanov <rstoyanov1@gmail.com> +# +# Copyright (C) 2017 SUSE, Inc. +# Copyright (c) 2017 Radostin Stoyanov +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Module which contains utility functions used by virt-bootstrap. +""" + +import errno +import fcntl +import hashlib +import json +import os +import sys +import tempfile +import logging + +from subprocess import CalledProcessError, PIPE, Popen + +# pylint: disable=invalid-name +# Create logger +logger = logging.getLogger(__name__) +# Default virtual size of qcow2 image +DEF_QCOW2_SIZE = '5G' +if os.geteuid() == 0: + LIBVIRT_CONN = "lxc:///" + DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images" +else: + LIBVIRT_CONN = "qemu:///session" + DEFAULT_IMG_DIR = os.environ['HOME'] + DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images" + + +def checksum(path, sum_type, sum_expected): + """ + Validate file using checksum. + """ + algorithm = getattr(hashlib, sum_type) + try: + handle = open(path, 'rb') + content = handle.read() + handle.close() + + actual = algorithm(content).hexdigest() + if not actual == sum_expected: + logger.warning("File '%s' has invalid hash sum.\nExpected: %s\n" + "Actual: %s", path, sum_expected, actual) + return False + return True + except Exception as err: + logger.warning("Error occured while validating " + "the hash sum of file: %s\n%s", path, err) + return False + + +def execute(cmd): + """ + Execute command and log debug message. + """ + cmd_str = ' '.join(cmd) + logger.debug("Call command:\n%s", cmd_str) + + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + output, err = proc.communicate() + + if output: + logger.debug("Stdout:\n%s", output) + if err: + logger.debug("Stderr:\n%s", err) + + if proc.returncode != 0: + raise CalledProcessError(proc.returncode, cmd_str) + + +def safe_untar(src, dest): + """ + Extract tarball within LXC container for safety. + """ + virt_sandbox = ['virt-sandbox', + '-c', LIBVIRT_CONN, + '-m', 'host-bind:/mnt=' + dest] # Bind destination folder + + # Compression type is auto detected from tar + # Exclude files under /dev to avoid "Cannot mknod: Operation not permitted" + params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*'] + execute(virt_sandbox + params) + + +def format_number(number): + """ + Turn numbers into human-readable metric-like numbers + """ + symbols = ['', 'KiB', 'MiB', 'GiB'] + step = 1024.0 + thresh = 999 + depth = 0 + max_depth = len(symbols) - 1 + + while number > thresh and depth < max_depth: + depth = depth + 1 + number = number / step + + if int(number) == float(number): + fmt = '%i %s' + else: + fmt = '%.2f %s' + + return(fmt % (number or 0, symbols[depth])) + + +def size_to_bytes(string, fmt): + """ + Convert human readable formats to bytes. + """ + formats = {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4} + return (string * pow(1024, formats[fmt.upper()]) if fmt in formats + else False) + + +def log_layer_extract(layer, current, total, progress): + """ + Create log message on layer extract. + """ + sum_type, sum_value, layer_file, layer_size = layer + progress("Extracting layer (%s/%s) with size: %s" + % (current, total, format_number(layer_size)), logger=logger) + logger.debug('Untar layer: (%s:%s) %s', sum_type, sum_value, layer_file) + + +def untar_layers(layers_list, dest_dir, progress): + """ + Untar each of layers from container image. + """ + nlayers = len(layers_list) + for index, layer in enumerate(layers_list): + log_layer_extract(layer, index + 1, nlayers, progress) + layer_file = layer[2] + + # Extract layer tarball into destination directory + safe_untar(layer_file, dest_dir) + + # Update progress value + progress(value=(float(index + 1) / nlayers * 50) + 50) + + +def get_mime_type(path): + """ + Get the mime type of a file. + """ + return Popen(["/usr/bin/file", "--mime-type", path], + stdout=PIPE).communicate()[0].split()[1] + + +def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE): + """ + Create qcow2 image from tarball. + """ + qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size] + + if not backing_file: + logger.info("Creating base qcow2 image") + execute(qemu_img_cmd) + + logger.info("Formatting qcow2 image") + execute(['virt-format', + '--format=qcow2', + '--partition=none', + '--filesystem=ext3', + '-a', layer_file]) + else: + # Add backing chain + qemu_img_cmd.insert(2, "-b") + qemu_img_cmd.insert(3, backing_file) + + logger.info("Creating qcow2 image with backing chain") + execute(qemu_img_cmd) + + # Get mime type of archive + mime_tar_file = get_mime_type(tar_file) + logger.debug("Detected mime type of archive: %s", mime_tar_file) + + # Extract tarball using "tar-in" command from libguestfs + tar_in_cmd = ["guestfish", + "-a", layer_file, + '-m', '/dev/sda', + 'tar-in', tar_file, "/"] + + compression_fmts = {'x-gzip': 'gzip', 'gzip': 'gzip', + 'x-xz': 'xz', + 'x-bzip2': 'bzip2', + 'x-compress': 'compress', + 'x-lzop': 'lzop'} + + # Check if tarball is compressed + mime_parts = mime_tar_file.split('/') + if mime_parts[0] == 'application' and \ + mime_parts[1] in compression_fmts: + tar_in_cmd.append('compress:' + compression_fmts[mime_parts[1]]) + + # Execute virt-tar-in command + execute(tar_in_cmd) + + +def extract_layers_in_qcow2(layers_list, dest_dir, progress): + """ + Extract docker layers in qcow2 images with backing chains. + """ + qcow2_backing_file = None + + nlayers = len(layers_list) + for index, layer in enumerate(layers_list): + log_layer_extract(layer, index + 1, nlayers, progress) + tar_file = layer[2] + + # Name format for the qcow2 image + qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index) + # Create the image layer + create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file) + # Keep the file path for the next layer + qcow2_backing_file = qcow2_layer_file + + # Update progress value + progress(value=(float(index + 1) / nlayers * 50) + 50) + + +def get_image_dir(no_cache=False): + """ + Get the directory where image layers are stored. + + @param no_cache: Boolean, indicates whether to use temporary directory + """ + if no_cache: + return tempfile.mkdtemp('virt-bootstrap') + + if not os.path.exists(DEFAULT_IMG_DIR): + os.makedirs(DEFAULT_IMG_DIR) + + return DEFAULT_IMG_DIR + + +def get_image_details(src, raw=False, + insecure=False, username=False, password=False): + """ + Return details of container image from "skopeo inspect" commnad. + """ + cmd = ['skopeo', 'inspect', src] + if raw: + cmd.append('--raw') + if insecure: + cmd.append('--tls-verify=false') + if username and password: + cmd.append("--creds=%s:%s" % (username, password)) + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + output, error = proc.communicate() + if error: + raise ValueError("Image could not be retrieved:", error) + return json.loads(output) + + +def is_new_layer_message(line): + """ + Return T/F whether a line in skopeo's progress is indicating + start the process of new image layer. + + Reference: + - https://github.com/containers/image/blob/master/copy/copy.go#L464 + - https://github.com/containers/image/blob/master/copy/copy.go#L459 + """ + return line.startswith('Copying blob') or line.startswith('Skipping fetch') + + +def is_layer_config_message(line): + """ + Return T/F indicating whether the message from skopeo copies the manifest + file. + + Reference: + - https://github.com/containers/image/blob/master/copy/copy.go#L414 + """ + return line.startswith('Copying config') + + +def make_async(fd): + """ + Add the O_NONBLOCK flag to a file descriptor. + """ + fcntl.fcntl(fd, fcntl.F_SETFL, + fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + +def read_async(fd): + """ + Read some data from a file descriptor, ignoring EAGAIN errors + """ + try: + return fd.read() + except IOError as e: + if e.errno != errno.EAGAIN: + raise + else: + return '' + + +def str2float(element): + """ + Convert string to float or return None. + """ + try: + return float(element) + except ValueError: + return None + + +def write_progress(prog): + """ + Write progress output to console + """ + # Get terminal width + try: + terminal_width = int(Popen(["stty", "size"], stdout=PIPE).stdout + .read().split()[1]) + except Exception: + terminal_width = 80 + # Prepare message + msg = "\rStatus: %s, Progress: %.2f%%" % (prog['status'], prog['value']) + # Fill with whitespace and return cursor at the begging + msg = "%s\r" % msg.ljust(terminal_width) + # Write message to console + sys.stdout.write(msg) + sys.stdout.flush() diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py index b2a32d7..85aca33 100755 --- a/src/virtBootstrap/virt_bootstrap.py +++ b/src/virtBootstrap/virt_bootstrap.py @@ -35,6 +35,7 @@ except ImportError: from virtBootstrap import sources from virtBootstrap import progress +from virtBootstrap import utils gettext.bindtextdomain("virt-bootstrap", "/usr/share/locale") @@ -81,25 +82,6 @@ def set_root_password(rootfs, password): raise CalledProcessError(chpasswd.returncode, cmd=args, output=None) -def write_progress(prog): - """ - Write progress output to console - """ - # Get terminal width - try: - terminal_width = int(Popen(["stty", "size"], stdout=PIPE).stdout - .read().split()[1]) - except Exception: - terminal_width = 80 - # Prepare message - msg = "\rStatus: %s, Progress: %.2f%%" % (prog['status'], prog['value']) - # Fill with whitespace and return cursor at the begging - msg = "%s\r" % msg.ljust(terminal_width) - # Write message to console - sys.stdout.write(msg) - sys.stdout.flush() - - # pylint: disable=too-many-arguments def bootstrap(uri, dest, fmt='dir', @@ -199,7 +181,7 @@ def main(): help=_("Suppresses messages notifying about" "current state or actions of virt-bootstrap")) parser.add_argument("--status-only", action="store_const", - const=write_progress, + const=utils.write_progress, help=_("Show only the current status and progress" "of virt-bootstrap")) |