Move source files in src/virtBootstrap

Preparatory commit before setup.py introduction.

1 files changed, 276 insertions, 0 deletions

diff --git a/src/virtBootstrap/sources.py b/src/virtBootstrap/sources.py
new file mode 100644
index 0000000..015e0b4
--- /dev/null
+++ b/src/virtBootstrap/sources.py
@@ -0,0 +1,276 @@
+#!/usr/bin/python
+# Authors: Cedric Bosdonnat <cbosdonnat@suse.com>
+#
+# Copyright (C) 2017 SUSE, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+
+import hashlib
+import json
+import shutil
+import tempfile
+import getpass
+import os
+import logging
+from subprocess import call, CalledProcessError, PIPE, Popen
+
+# Default virtual size of qcow2 image
+DEF_QCOW2_SIZE = '5G'
+# default_image_dir - Path where Docker images (tarballs) will be stored
+if os.geteuid() == 0:
+    virt_sandbox_connection = "lxc:///"
+    default_image_dir = "/var/lib/virt-bootstrap/docker_images"
+else:
+    virt_sandbox_connection = "qemu:///session"
+    default_image_dir = \
+        os.environ['HOME'] + "/.local/share/virt-bootstrap/docker_images"
+
+
+def checksum(path, sum_type, sum_expected):
+    algorithm = getattr(hashlib, sum_type)
+    try:
+        fd = open(path, 'rb')
+        content = fd.read()
+        fd.close()
+
+        actual = algorithm(content).hexdigest()
+        return actual == sum_expected
+    except Exception:
+        return False
+
+
+def safe_untar(src, dest):
+    # Extract tarball in LXC container for safety
+    virt_sandbox = ['virt-sandbox',
+                    '-c', virt_sandbox_connection,
+                    '-m', 'host-bind:/mnt=' + dest]  # Bind destination folder
+
+    # Compression type is auto detected from tar
+    # Exclude files under /dev to avoid "Cannot mknod: Operation not permitted"
+    params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*']
+    if call(virt_sandbox + params) != 0:
+        logging.error(_('virt-sandbox exit with non-zero code. '
+                        'Please check if "libvirtd" is running.'))
+
+
+def get_layer_info(digest, image_dir):
+    sum_type, sum_value = digest.split(':')
+    layer_file = "{}/{}.tar".format(image_dir, sum_value)
+    return (sum_type, sum_value, layer_file)
+
+
+def untar_layers(layers_list, image_dir, dest_dir):
+    for layer in layers_list:
+        sum_type, sum_value, layer_file = get_layer_info(layer['digest'],
+                                                         image_dir)
+        logging.info('Untar layer file: ({}) {}'.format(sum_type, layer_file))
+
+        # Verify the checksum
+        if not checksum(layer_file, sum_type, sum_value):
+            raise Exception("Digest not matching: " + layer['digest'])
+
+        # Extract layer tarball into destination directory
+        safe_untar(layer_file, dest_dir)
+
+
+def get_mime_type(path):
+    """
+        Get the mime type of a file.
+    """
+    return Popen(["/usr/bin/file", "--mime-type", path],
+                 stdout=PIPE).communicate()[0].split()[1]
+
+
+def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
+    qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
+
+    if not backing_file:
+        logging.info("Create base qcow2 image")
+        check_call(qemu_img_cmd)
+
+        logging.info("Format qcow2 image")
+        check_call(['virt-format',
+                    '--format=qcow2',
+                    '--partition=none',
+                    '--filesystem=ext3',
+                    '-a', layer_file])
+    else:
+        # Add backing chain
+        qemu_img_cmd.insert(2, "-b")
+        qemu_img_cmd.insert(3, backing_file)
+
+        logging.info("Crate qcow2 image with backing chain")
+        check_call(qemu_img_cmd)
+
+    # Get mime type of archive
+    mime_tar_file = get_mime_type(tar_file)
+    logging.debug("Detected mime type of archive: %s", mime_tar_file)
+
+    # Extract tarball using "tar-in" command from libguestfs
+    tar_in_cmd = ["guestfish",
+                  "-a", layer_file,
+                  '-m', '/dev/sda',
+                  'tar-in', tar_file, "/"]
+
+    compression_fmts = {'x-gzip': 'gzip', 'gzip': 'gzip',
+                        'x-xz': 'xz',
+                        'x-bzip2': 'bzip2',
+                        'x-compress': 'compress',
+                        'x-lzop': 'lzop'}
+
+    # Check if tarball is compressed
+    mime_parts = mime_tar_file.split('/')
+    if mime_parts[0] == 'application' and \
+       mime_parts[1] in compression_fmts:
+        tar_in_cmd.append('compress:' + compression_fmts[mime_parts[1]])
+
+    # Execute virt-tar-in command
+    check_call(tar_in_cmd)
+
+
+def extract_layers_in_qcow2(layers_list, image_dir, dest_dir):
+    qcow2_backing_file = None
+
+    for index, layer in enumerate(layers_list):
+        # Get layer file information
+        sum_type, sum_value, tar_file = \
+         get_layer_info(layer['digest'], image_dir)
+
+        logging.info('Untar layer file: ({}) {}'.format(sum_type, tar_file))
+
+        # Verify the checksum
+        if not checksum(tar_file, sum_type, sum_value):
+            raise Exception("Digest not matching: " + layer['digest'])
+
+        # Name format for the qcow2 image
+        qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
+        # Create the image layer
+        create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
+        # Keep the file path for the next layer
+        qcow2_backing_file = qcow2_layer_file
+
+
+class FileSource:
+    def __init__(self, url, username, password, fmt, insecure, no_cache):
+        self.path = url.path
+        self.output_format = fmt
+
+    def unpack(self, dest):
+        '''
+        Safely extract root filesystem from tarball
+
+        @param dest: Directory path where the files to be extraced
+        '''
+        if self.output_format == 'dir':
+            logging.info("Extracting files into destination directory")
+            safe_untar(self.path, dest)
+
+        elif self.output_format == 'qcow2':
+            # Remove the old path
+            file_name = os.path.basename(self.path)
+            qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest,
+                                                               file_name))
+
+            logging.info("Extracting files into qcow2 image")
+            create_qcow2(self.path, qcow2_file)
+        else:
+            raise Exception("Unknown format:" + self.output_format)
+
+        logging.info("Extraction completed successfully!")
+        logging.info("Files are stored in: " + dest)
+
+
+class DockerSource:
+    def __init__(self, url, username, password, fmt, insecure, no_cache):
+        '''
+        Bootstrap root filesystem from Docker registry
+
+        @param url: Address of source registry
+        @param username: Username to access source registry
+        @param password: Password to access source registry
+        @param fmt: Format used to store image [dir, qcow2]
+        @param insecure: Do not require HTTPS and certificate verification
+        @param no_cache: Whether to store downloaded images or not
+        '''
+
+        self.registry = url.netloc
+        self.image = url.path
+        self.username = username
+        self.password = password
+        self.output_format = fmt
+        self.insecure = insecure
+        self.no_cache = no_cache
+        if self.image and not self.image.startswith('/'):
+            self.image = '/' + self.image
+        self.url = "docker://" + self.registry + self.image
+
+    def unpack(self, dest):
+        '''
+        Extract image files from Docker image
+
+        @param dest: Directory path where the files to be extraced
+        '''
+
+        if self.no_cache:
+            tmp_dest = tempfile.mkdtemp('virt-bootstrap')
+            images_dir = tmp_dest
+        else:
+            if not os.path.exists(default_image_dir):
+                os.makedirs(default_image_dir)
+            images_dir = default_image_dir
+
+        try:
+            # Run skopeo copy into a tmp folder
+            # Note: we don't want to expose --src-cert-dir to users as
+            #       they should place the certificates in the system
+            #       folders for broader enablement
+            skopeo_copy = ["skopeo", "copy", self.url, "dir:"+images_dir]
+
+            if self.insecure:
+                skopeo_copy.append('--src-tls-verify=false')
+            if self.username:
+                if not self.password:
+                    self.password = getpass.getpass()
+                skopeo_copy.append('--src-creds={}:{}'.format(self.username,
+                                                              self.password))
+            # Run "skopeo copy" command
+            check_call(skopeo_copy)
+
+            # Get the layers list from the manifest
+            mf = open(images_dir+"/manifest.json", "r")
+            manifest = json.load(mf)
+
+            # Layers are in order - root layer first
+            # Reference:
+            # https://github.com/containers/image/blob/master/image/oci.go#L100
+            if self.output_format == 'dir':
+                untar_layers(manifest['layers'], images_dir, dest)
+            elif self.output_format == 'qcow2':
+                extract_layers_in_qcow2(manifest['layers'], images_dir, dest)
+            else:
+                raise Exception("Unknown format:" + self.output_format)
+
+        except Exception:
+            raise
+
+        else:
+            logging.info("Download and extract completed!")
+            logging.info("Files are stored in: " + dest)
+
+        finally:
+            # Clean up
+            if self.no_cache:
+                shutil.rmtree(tmp_dest)