path: root/src/virtBootstrap/utils.py

# Authors:
#   Cedric Bosdonnat <cbosdonnat@suse.com>
#   Radostin Stoyanov <rstoyanov1@gmail.com>
#
# Copyright (C) 2017 SUSE, Inc.
# Copyright (c) 2017 Radostin Stoyanov
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Module which contains utility functions used by virt-bootstrap.
"""

import errno
import fcntl
import hashlib
import json
import os
import subprocess
import sys
import tempfile
import logging
import re

import passlib.hosts

# pylint: disable=invalid-name
# Create logger
logger = logging.getLogger(__name__)

DEFAULT_OUTPUT_FORMAT = 'dir'
# Default virtual size of qcow2 image
DEF_QCOW2_SIZE = '5G'
if os.geteuid() == 0:
    LIBVIRT_CONN = "lxc:///"
    DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
else:
    LIBVIRT_CONN = "qemu:///session"
    DEFAULT_IMG_DIR = os.environ['HOME']
    DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"


def get_compression_type(tar_file):
    """
    Get compression type of tar file.
    """
    # Get mime type of archive
    mime_tar_file = get_mime_type(tar_file)
    logger.debug("Detected mime type of archive: %s", mime_tar_file)

    compression_fmts = {
        'x-gzip': 'gzip',
        'gzip': 'gzip',
        'x-xz': 'xz',
        'x-bzip2': 'bzip2',
        'x-compress': 'compress',
        'x-lzop': 'lzop'
    }

    # Check if tarball is compressed
    mime_type, mime_subtype = mime_tar_file.split('/')
    if mime_type == 'application' and mime_subtype in compression_fmts:
        return compression_fmts[mime_subtype]
    return None


def checksum(path, sum_type, sum_expected):
    """
    Validate file using checksum.
    """
    algorithm = getattr(hashlib, sum_type)
    try:
        handle = open(path, 'rb')
        content = handle.read()
        handle.close()

        actual = algorithm(content).hexdigest()
        if not actual == sum_expected:
            logger.warning("File '%s' has invalid hash sum.\nExpected: %s\n"
                           "Actual: %s", path, sum_expected, actual)
            return False
        return True
    except Exception as err:
        logger.warning("Error occured while validating "
                       "the hash sum of file: %s\n%s", path, err)
        return False


def execute(cmd):
    """
    Execute command and log debug message.
    """
    cmd_str = ' '.join(cmd)
    logger.debug("Call command:\n%s", cmd_str)

    proc = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    output, err = proc.communicate()

    if output:
        logger.debug("Stdout:\n%s", output.decode('utf-8'))
    if err:
        logger.debug("Stderr:\n%s", err.decode('utf-8'))

    if proc.returncode != 0:
        raise subprocess.CalledProcessError(proc.returncode, cmd_str)


def safe_untar(src, dest):
    """
    Extract tarball within LXC container for safety.
    """
    virt_sandbox = ['virt-sandbox',
                    '-c', LIBVIRT_CONN,
                    '--name=bootstrap_%s' % os.getpid(),
                    '-m', 'host-bind:/mnt=' + dest]  # Bind destination folder

    # Compression type is auto detected from tar
    # Exclude files under /dev to avoid "Cannot mknod: Operation not permitted"
    # Note: Here we use --absolute-names flag to get around the error message
    # "Cannot open: Permission denied" when symlynks are extracted, with the
    # qemu:/// driver. This flag must not be used outside virt-sandbox.
    params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*',
              '--overwrite', '--absolute-names']
    execute(virt_sandbox + params)


def bytes_to_size(number):
    """
    Turn numbers into human-readable metric-like numbers
    """
    symbols = ['', 'KiB', 'MiB', 'GiB']
    step = 1024.0
    thresh = 999
    depth = 0
    max_depth = len(symbols) - 1

    while number > thresh and depth < max_depth:
        depth = depth + 1
        number = number / step

    if int(number) == float(number):
        fmt = '%i %s'
    else:
        fmt = '%.2f %s'

    return(fmt % (number or 0, symbols[depth])).strip()


def size_to_bytes(number, fmt):
    """
    Convert human readable formats to bytes.
    """
    formats = {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3, 'TB': 4}
    return (int(number) * pow(1024, formats[fmt.upper()]) if fmt in formats
            else False)


def log_layer_extract(tar_file, tar_size, current, total, progress):
    """
    Create log message on layer extract.
    """
    msg = 'Extracting layer (%s/%s) with size: %s' % (
        current,
        total,
        bytes_to_size(tar_size or os.path.getsize(tar_file))
    )
    progress(msg, logger=logger)
    logger.debug('Untar layer: %s', tar_file)


def untar_layers(layers_list, dest_dir, progress):
    """
    Untar each of layers from container image.
    """
    nlayers = len(layers_list)
    for index, layer in enumerate(layers_list):
        tar_file, tar_size = layer
        log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress)

        # Extract layer tarball into destination directory
        safe_untar(tar_file, dest_dir)

        # Update progress value
        progress(value=(float(index + 1) / nlayers * 50) + 50)


def get_mime_type(path):
    """
        Get the mime type of a file.
    """
    proc = subprocess.Popen(
        ["/usr/bin/file", "--mime-type", path],
        stdout=subprocess.PIPE
    )
    proc.wait()
    with proc.stdout as output:
        return output.read().decode('utf-8').split()[1]


def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
    """
    Create qcow2 image from tarball.
    """
    qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]

    if not backing_file:
        logger.info("Creating base qcow2 image")
        execute(qemu_img_cmd)

        logger.info("Formatting qcow2 image")
        execute(['virt-format',
                 '--format=qcow2',
                 '--partition=none',
                 '--filesystem=ext3',
                 '-a', layer_file])
    else:
        # Add backing chain
        qemu_img_cmd.insert(2, "-b")
        qemu_img_cmd.insert(3, backing_file)

        logger.info("Creating qcow2 image with backing chain")
        execute(qemu_img_cmd)

    # Extract tarball using "tar-in" command from libguestfs
    tar_in_cmd = ["guestfish",
                  "-a", layer_file,
                  '-m', '/dev/sda',
                  'tar-in', tar_file, "/"]

    # Check if tarball is compressed
    compression = get_compression_type(tar_file)
    if compression is not None:
        tar_in_cmd.append('compress:' + compression)

    # Execute virt-tar-in command
    execute(tar_in_cmd)


def extract_layers_in_qcow2(layers_list, dest_dir, progress):
    """
    Extract docker layers in qcow2 images with backing chains.
    """
    qcow2_backing_file = None

    nlayers = len(layers_list)
    for index, layer in enumerate(layers_list):
        tar_file, tar_size = layer
        log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress)

        # Name format for the qcow2 image
        qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
        # Create the image layer
        create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
        # Keep the file path for the next layer
        qcow2_backing_file = qcow2_layer_file

        # Update progress value
        progress(value=(float(index + 1) / nlayers * 50) + 50)


def get_image_dir(no_cache=False):
    """
    Get the directory where image layers are stored.

    @param no_cache: Boolean, indicates whether to use temporary directory
    """
    if no_cache:
        return tempfile.mkdtemp('virt-bootstrap')

    if not os.path.exists(DEFAULT_IMG_DIR):
        os.makedirs(DEFAULT_IMG_DIR)

    return DEFAULT_IMG_DIR


def get_image_details(src, raw=False,
                      insecure=False, username=False, password=False):
    """
    Return details of container image from "skopeo inspect" commnad.
    """
    cmd = ['skopeo', 'inspect', src]
    if raw:
        cmd.append('--raw')
    if insecure:
        cmd.append('--tls-verify=false')
    if username and password:
        cmd.append("--creds=%s:%s" % (username, password))
    proc = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    output, error = proc.communicate()
    if error:
        raise ValueError("Image could not be retrieved:",
                         error.decode('utf-8'))
    return json.loads(output.decode('utf-8'))


def is_new_layer_message(line):
    """
    Return T/F whether a line in skopeo's progress is indicating
    start the process of new image layer.

    Reference:
    - https://github.com/containers/image/blob/master/copy/copy.go#L464
    - https://github.com/containers/image/blob/master/copy/copy.go#L459
    """
    return line.startswith('Copying blob') or line.startswith('Skipping fetch')


def is_layer_config_message(line):
    """
    Return T/F indicating whether the message from skopeo copies the manifest
    file.

    Reference:
    - https://github.com/containers/image/blob/master/copy/copy.go#L414
    """
    return line.startswith('Copying config')


def make_async(fd):
    """
    Add the O_NONBLOCK flag to a file descriptor.
    """
    fcntl.fcntl(fd, fcntl.F_SETFL,
                fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)


def read_async(fd):
    """
    Read some data from a file descriptor, ignoring EAGAIN errors
    """
    try:
        return fd.read()
    except IOError as e:
        if e.errno != errno.EAGAIN:
            raise
        else:
            return ''


def str2float(element):
    """
    Convert string to float or return None.
    """
    try:
        return float(element)
    except ValueError:
        return None


def set_root_password_in_rootfs(rootfs, password):
    """
    Set password on the root user within root filesystem
    """
    shadow_file = os.path.join(rootfs, "etc/shadow")

    shadow_file_permissions = os.stat(shadow_file)[0]
    # Set read-write permissions to shadow file
    # 438 = 0110110110 = -rw-rw-rw-
    os.chmod(shadow_file, 438)
    try:
        with open(shadow_file) as orig_file:
            shadow_content = orig_file.read().split('\n')

        for index, line in enumerate(shadow_content):
            if line.startswith('root'):
                line_split = line.split(':')
                line_split[1] = passlib.hosts.linux_context.hash(password)
                shadow_content[index] = ':'.join(line_split)
                break

        with open(shadow_file, "w") as new_file:
            new_file.write('\n'.join(shadow_content))

    except Exception:
        raise

    finally:
        # Restore original permissions
        os.chmod(shadow_file, shadow_file_permissions)


def set_root_password_in_image(image, password):
    """
    Set password on the root user within image
    """
    password_hash = passlib.hosts.linux_context.hash(password)
    execute(['virt-edit',
             '-a', image, '/etc/shadow',
             '-e', 's,^root:.*?:,root:%s:,' % re.escape(password_hash)])


def set_root_password(fmt, dest, root_password):
    """
    Set root password
    """
    if fmt == "dir":
        set_root_password_in_rootfs(dest, root_password)
    elif fmt == "qcow2":
        layers = [layer for layer in os.listdir(dest)
                  if layer.startswith('layer-')]
        set_root_password_in_image(os.path.join(dest, max(layers)),
                                   root_password)


def write_progress(prog):
    """
    Write progress output to console
    """
    # Get terminal width
    try:
        terminal_width = int(
            subprocess.Popen(
                ["stty", "size"],
                stdout=subprocess.PIPE
            ).stdout.read().decode('utf-8').split()[1]
        )
    except Exception:
        terminal_width = 80
    # Prepare message
    msg = "\rStatus: %s, Progress: %.2f%%" % (prog['status'], prog['value'])
    # Fill with whitespace and return cursor at the begging
    msg = "%s\r" % msg.ljust(terminal_width)
    # Write message to console
    sys.stdout.write(msg)
    sys.stdout.flush()


# The implementation for remapping ownership of all files inside a
# container's rootfs is inspired by the tool uidmapshift:
#
# Original author: Serge Hallyn <serge.hallyn@ubuntu.com>
# Original license: GPLv2
# http://bazaar.launchpad.net/%7Eserge-hallyn/+junk/nsexec/view/head:/uidmapshift.c

def get_map_id(old_id, opts):
    """
    Calculate new map_id.
    """
    if old_id >= opts['first'] and old_id < opts['last']:
        return old_id + opts['offset']
    return -1


def get_mapping_opts(mapping):
    """
    Get range options from UID/GID mapping
    """
    start = mapping[0] if mapping[0] > -1 else 0
    target = mapping[1] if mapping[1] > -1 else 0
    count = mapping[2] if mapping[2] > -1 else 1

    opts = {
        'first': start,
        'last': start + count,
        'offset': target - start
    }
    return opts


def map_id(path, map_uid, map_gid):
    """
    Remapping ownership of all files inside a container's rootfs.

    map_gid and map_uid: Contain integers in a list with format:
        [<start>, <target>, <count>]
    """
    if map_uid:
        uid_opts = get_mapping_opts(map_uid)
    if map_gid:
        gid_opts = get_mapping_opts(map_gid)

    for root, _ignore, files in os.walk(os.path.realpath(path)):
        for name in [root] + files:
            file_path = os.path.join(root, name)

            stat_info = os.lstat(file_path)
            old_uid = stat_info.st_uid
            old_gid = stat_info.st_gid

            new_uid = get_map_id(old_uid, uid_opts) if map_uid else -1
            new_gid = get_map_id(old_gid, gid_opts) if map_gid else -1
            os.lchown(file_path, new_uid, new_gid)


def mapping_uid_gid(dest, uid_map, gid_map):
    """
    Mapping ownership for each uid_map and gid_map.
    """
    len_diff = len(uid_map) - len(gid_map)

    if len_diff < 0:
        uid_map += [None] * abs(len_diff)
    elif len_diff > 0:
        gid_map += [None] * len_diff

    for uid, gid in zip(uid_map, gid_map):
        map_id(dest, uid, gid)