Gather common utility functions in "utils" module

Add new module to collect utility functions used in virt-bootstrap.
Move the function definitions from "sources" and "virt_bootstrap" to
the new module.

1 files changed, 21 insertions, 314 deletions

diff --git a/src/virtBootstrap/sources.py b/src/virtBootstrap/sources.py
index db4fe36..beb671a 100644
--- a/src/virtBootstrap/sources.py
+++ b/src/virtBootstrap/sources.py
@@ -20,313 +20,20 @@ Class definitions which process container image or
 archive from source and unpack them in destination directory.
 """
 
-import errno
-import fcntl
-import hashlib
-import json
 import select
 import shutil
-import tempfile
 import getpass
 import os
 import logging
 from subprocess import CalledProcessError, PIPE, Popen
 
+from virtBootstrap import utils
+
 
 # pylint: disable=invalid-name
 # Create logger
 logger = logging.getLogger(__name__)
 
-# Default virtual size of qcow2 image
-DEF_QCOW2_SIZE = '5G'
-if os.geteuid() == 0:
-    LIBVIRT_CONN = "lxc:///"
-    DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
-else:
-    LIBVIRT_CONN = "qemu:///session"
-    DEFAULT_IMG_DIR = os.environ['HOME']
-    DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"
-
-
-def checksum(path, sum_type, sum_expected):
-    """
-    Validate file using checksum.
-    """
-    algorithm = getattr(hashlib, sum_type)
-    try:
-        handle = open(path, 'rb')
-        content = handle.read()
-        handle.close()
-
-        actual = algorithm(content).hexdigest()
-        if not actual == sum_expected:
-            logger.warning("File '%s' has invalid hash sum.\nExpected: %s\n"
-                           "Actual: %s", path, sum_expected, actual)
-            return False
-        return True
-    except Exception as err:
-        logger.warning("Error occured while validating "
-                       "the hash sum of file: %s\n%s", path, err)
-        return False
-
-
-def execute(cmd):
-    """
-    Execute command and log debug message.
-    """
-    cmd_str = ' '.join(cmd)
-    logger.debug("Call command:\n%s", cmd_str)
-
-    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
-    output, err = proc.communicate()
-
-    if output:
-        logger.debug("Stdout:\n%s", output)
-    if err:
-        logger.debug("Stderr:\n%s", err)
-
-    if proc.returncode != 0:
-        raise CalledProcessError(proc.returncode, cmd_str)
-
-
-def safe_untar(src, dest):
-    """
-    Extract tarball within LXC container for safety.
-    """
-    virt_sandbox = ['virt-sandbox',
-                    '-c', LIBVIRT_CONN,
-                    '-m', 'host-bind:/mnt=' + dest]  # Bind destination folder
-
-    # Compression type is auto detected from tar
-    # Exclude files under /dev to avoid "Cannot mknod: Operation not permitted"
-    params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*']
-    execute(virt_sandbox + params)
-
-
-def format_number(number):
-    """
-    Turn numbers into human-readable metric-like numbers
-    """
-    symbols = ['', 'KiB', 'MiB', 'GiB']
-    step = 1024.0
-    thresh = 999
-    depth = 0
-    max_depth = len(symbols) - 1
-
-    while number > thresh and depth < max_depth:
-        depth = depth + 1
-        number = number / step
-
-    if int(number) == float(number):
-        fmt = '%i %s'
-    else:
-        fmt = '%.2f %s'
-
-    return(fmt % (number or 0, symbols[depth]))
-
-
-def log_layer_extract(layer, current, total, progress):
-    """
-    Create log message on layer extract.
-    """
-    sum_type, sum_value, layer_file, layer_size = layer
-    progress("Extracting layer (%s/%s) with size: %s"
-             % (current, total, format_number(layer_size)), logger=logger)
-    logger.debug('Untar layer: (%s:%s) %s', sum_type, sum_value, layer_file)
-
-
-def untar_layers(layers_list, dest_dir, progress):
-    """
-    Untar each of layers from container image.
-    """
-    nlayers = len(layers_list)
-    for index, layer in enumerate(layers_list):
-        log_layer_extract(layer, index + 1, nlayers, progress)
-        layer_file = layer[2]
-
-        # Extract layer tarball into destination directory
-        safe_untar(layer_file, dest_dir)
-
-        # Update progress value
-        progress(value=(float(index + 1) / nlayers * 50) + 50)
-
-
-def get_mime_type(path):
-    """
-        Get the mime type of a file.
-    """
-    return Popen(["/usr/bin/file", "--mime-type", path],
-                 stdout=PIPE).communicate()[0].split()[1]
-
-
-def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
-    """
-    Create qcow2 image from tarball.
-    """
-    qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
-
-    if not backing_file:
-        logger.info("Creating base qcow2 image")
-        execute(qemu_img_cmd)
-
-        logger.info("Formatting qcow2 image")
-        execute(['virt-format',
-                 '--format=qcow2',
-                 '--partition=none',
-                 '--filesystem=ext3',
-                 '-a', layer_file])
-    else:
-        # Add backing chain
-        qemu_img_cmd.insert(2, "-b")
-        qemu_img_cmd.insert(3, backing_file)
-
-        logger.info("Creating qcow2 image with backing chain")
-        execute(qemu_img_cmd)
-
-    # Get mime type of archive
-    mime_tar_file = get_mime_type(tar_file)
-    logger.debug("Detected mime type of archive: %s", mime_tar_file)
-
-    # Extract tarball using "tar-in" command from libguestfs
-    tar_in_cmd = ["guestfish",
-                  "-a", layer_file,
-                  '-m', '/dev/sda',
-                  'tar-in', tar_file, "/"]
-
-    compression_fmts = {'x-gzip': 'gzip', 'gzip': 'gzip',
-                        'x-xz': 'xz',
-                        'x-bzip2': 'bzip2',
-                        'x-compress': 'compress',
-                        'x-lzop': 'lzop'}
-
-    # Check if tarball is compressed
-    mime_parts = mime_tar_file.split('/')
-    if mime_parts[0] == 'application' and \
-       mime_parts[1] in compression_fmts:
-        tar_in_cmd.append('compress:' + compression_fmts[mime_parts[1]])
-
-    # Execute virt-tar-in command
-    execute(tar_in_cmd)
-
-
-def extract_layers_in_qcow2(layers_list, dest_dir, progress):
-    """
-    Extract docker layers in qcow2 images with backing chains.
-    """
-    qcow2_backing_file = None
-
-    nlayers = len(layers_list)
-    for index, layer in enumerate(layers_list):
-        log_layer_extract(layer, index + 1, nlayers, progress)
-        tar_file = layer[2]
-
-        # Name format for the qcow2 image
-        qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
-        # Create the image layer
-        create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
-        # Keep the file path for the next layer
-        qcow2_backing_file = qcow2_layer_file
-
-        # Update progress value
-        progress(value=(float(index + 1) / nlayers * 50) + 50)
-
-
-def get_image_dir(no_cache=False):
-    """
-    Get the directory where image layers are stored.
-
-    @param no_cache: Boolean, indicates whether to use temporary directory
-    """
-    if no_cache:
-        return tempfile.mkdtemp('virt-bootstrap')
-
-    if not os.path.exists(DEFAULT_IMG_DIR):
-        os.makedirs(DEFAULT_IMG_DIR)
-
-    return DEFAULT_IMG_DIR
-
-
-def get_image_details(src, raw=False,
-                      insecure=False, username=False, password=False):
-    """
-    Return details of container image from "skopeo inspect" commnad.
-    """
-    cmd = ['skopeo', 'inspect', src]
-    if raw:
-        cmd.append('--raw')
-    if insecure:
-        cmd.append('--tls-verify=false')
-    if username and password:
-        cmd.append("--creds=%s:%s" % (username, password))
-    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
-    output, error = proc.communicate()
-    if error:
-        raise ValueError("Image could not be retrieved:", error)
-    return json.loads(output)
-
-
-def size_to_bytes(string, fmt):
-    """
-    Convert human readable formats to bytes.
-    """
-    formats = {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4}
-    return (string * pow(1024, formats[fmt.upper()]) if fmt in formats
-            else False)
-
-
-def is_new_layer_message(line):
-    """
-    Return T/F whether a line in skopeo's progress is indicating
-    start the process of new image layer.
-
-    Reference:
-    - https://github.com/containers/image/blob/master/copy/copy.go#L464
-    - https://github.com/containers/image/blob/master/copy/copy.go#L459
-    """
-    return line.startswith('Copying blob') or line.startswith('Skipping fetch')
-
-
-def is_layer_config_message(line):
-    """
-    Return T/F indicating whether the message from skopeo copies the manifest
-    file.
-
-    Reference:
-    - https://github.com/containers/image/blob/master/copy/copy.go#L414
-    """
-    return line.startswith('Copying config')
-
-
-def make_async(fd):
-    """
-    Add the O_NONBLOCK flag to a file descriptor.
-    """
-    fcntl.fcntl(fd, fcntl.F_SETFL,
-                fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
-
-
-def read_async(fd):
-    """
-    Read some data from a file descriptor, ignoring EAGAIN errors
-    """
-    try:
-        return fd.read()
-    except IOError as e:
-        if e.errno != errno.EAGAIN:
-            raise
-        else:
-            return ''
-
-
-def str2float(element):
-    """
-    Convert string to float or return None.
-    """
-    try:
-        return float(element)
-    except ValueError:
-        return None
-
 
 class FileSource(object):
     """
@@ -350,7 +57,7 @@ class FileSource(object):
         if self.output_format == 'dir':
             self.progress("Extracting files into destination directory",
                           value=0, logger=logger)
-            safe_untar(self.path, dest)
+            utils.safe_untar(self.path, dest)
 
         elif self.output_format == 'qcow2':
             # Remove the old path
@@ -360,7 +67,7 @@ class FileSource(object):
 
             self.progress("Extracting files into qcow2 image", value=0,
                           logger=logger)
-            create_qcow2(self.path, qcow2_file)
+            utils.create_qcow2(self.path, qcow2_file)
         else:
             raise Exception("Unknown format:" + self.output_format)
 
@@ -410,13 +117,13 @@ class DockerSource(object):
             image = image[:-1]
 
         self.url = "docker://" + registry + image
-        self.images_dir = get_image_dir(self.no_cache)
+        self.images_dir = utils.get_image_dir(self.no_cache)
 
         # Retrive manifest from registry
-        self.manifest = get_image_details(self.url, raw=True,
-                                          insecure=self.insecure,
-                                          username=self.username,
-                                          password=self.password)
+        self.manifest = utils.get_image_details(self.url, raw=True,
+                                                insecure=self.insecure,
+                                                username=self.username,
+                                                password=self.password)
 
         # Get layers' digest, sum_type, size and file_path in a list
         self.layers = []
@@ -456,11 +163,11 @@ class DockerSource(object):
             # Wait for data to become available
             stdout = select.select([proc.stdout], [], [])[0]
             # Split output into line
-            output = read_async(stdout[0]).strip().split('\n')
+            output = utils.read_async(stdout[0]).strip().split('\n')
             for line in output:
                 if line:  # is not empty
                     line_split = line.split()
-                    if is_new_layer_message(line):
+                    if utils.is_new_layer_message(line):
                         current_layer += 1
                         self.progress("Downloading layer (%s/%s)"
                                       % (current_layer, total_layers_num))
@@ -472,7 +179,7 @@ class DockerSource(object):
                                                          total_layers_num)
 
                     # Stop parsing when manifest is copied.
-                    elif is_layer_config_message(line):
+                    elif utils.is_layer_config_message(line):
                         break
             else:
                 continue  # continue if the inner loop didn't break
@@ -489,12 +196,12 @@ class DockerSource(object):
 
         Calculate percentage and update the progress of virt-bootstrap.
         """
-        d_size, d_format = str2float(line_split[0]), line_split[1]
-        t_size, t_format = str2float(line_split[3]), line_split[4]
+        d_size, d_format = utils.str2float(line_split[0]), line_split[1]
+        t_size, t_format = utils.str2float(line_split[3]), line_split[4]
 
         if d_size and t_size:
-            downloaded_size = size_to_bytes(d_size, d_format)
-            total_size = size_to_bytes(t_size, t_format)
+            downloaded_size = utils.size_to_bytes(d_size, d_format)
+            total_size = utils.size_to_bytes(t_size, t_format)
             if downloaded_size and total_size:
                 try:
                     self.progress(value=(50
@@ -510,7 +217,7 @@ class DockerSource(object):
         proc = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
 
         # Without `make_async`, `fd.read` in `read_async` blocks.
-        make_async(proc.stdout)
+        utils.make_async(proc.stdout)
         if not self.parse_output(proc):
             raise CalledProcessError(cmd, proc.stderr.read())
 
@@ -523,7 +230,7 @@ class DockerSource(object):
         for sum_type, sum_expected, path, _ignore in self.layers:
             logger.debug("Checking layer: %s", path)
             if not (os.path.exists(path)
-                    and checksum(path, sum_type, sum_expected)):
+                    and utils.checksum(path, sum_type, sum_expected)):
                 return False
         return True
 
@@ -551,11 +258,11 @@ class DockerSource(object):
             if self.output_format == 'dir':
                 self.progress("Extracting container layers", value=50,
                               logger=logger)
-                untar_layers(self.layers, dest, self.progress)
+                utils.untar_layers(self.layers, dest, self.progress)
             elif self.output_format == 'qcow2':
                 self.progress("Extracting container layers into qcow2 images",
                               value=50, logger=logger)
-                extract_layers_in_qcow2(self.layers, dest, self.progress)
+                utils.extract_layers_in_qcow2(self.layers, dest, self.progress)
             else:
                 raise Exception("Unknown format:" + self.output_format)
 
@@ -569,5 +276,5 @@ class DockerSource(object):
 
         finally:
             # Clean up
-            if self.no_cache and self.images_dir != DEFAULT_IMG_DIR:
+            if self.no_cache and self.images_dir != utils.DEFAULT_IMG_DIR:
                 shutil.rmtree(self.images_dir)