diff options
| -rw-r--r-- | pylintrc | 435 | ||||
| -rwxr-xr-x | setup.py | 66 | ||||
| -rw-r--r-- | src/virtBootstrap/sources.py | 93 | ||||
| -rwxr-xr-x | src/virtBootstrap/virt_bootstrap.py | 48 |
4 files changed, 589 insertions, 53 deletions
diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..4788aec --- /dev/null +++ b/pylintrc @@ -0,0 +1,435 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +#extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +#ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +# ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +# load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" + +# R0201 - Method could be a function +# C0320 - Unnecessary parens after 'print' keyword +# W0703 - Catching too general exception +# R0912 - Used when a function or method has too many branches,making it hard to follow. + +disable=R0201,C0325,W0703,R0912 + + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins=_ + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +# FIXME -- something which needs fixing +# TODO -- future plan +# XXX -- some concern +notes=FIXME,XXX,TODO + + +[BASIC] + +# Naming hint for argument names +argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct argument names +argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for attribute names +attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct attribute names +attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming hint for function names +function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct function names +function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,virtBootstrap + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=yes + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for method names +method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct method names +method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +#name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_|main + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming hint for variable names +variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?<?https?://\S+>?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +#spelling-dict= + +# List of comma separated words that should not be checked. +#spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +#spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +#spelling-store-unknown-words=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=8 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +#ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +#int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +#known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception @@ -1,8 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8; -*- +""" +Setup script used for building, testing, and installing modules +based on setuptools. +""" + import codecs import os +import sys +from subprocess import call +from setuptools import Command from setuptools import setup @@ -15,6 +23,52 @@ def read(fname): return fobj.read() +class CheckPylint(Command): + """ + Check python source files with pylint and pycodestyle. + """ + + user_options = [('errors-only', 'e', 'only report errors')] + description = "Check code using pylint and pycodestyle" + + def initialize_options(self): + """ + Initialize the options to default values. + """ + # pylint: disable=attribute-defined-outside-init + self.errors_only = False + + def finalize_options(self): + """ + Check final option values. + """ + pass + + def run(self): + """ + Call pycodestyle and pylint here. + """ + + res = 0 + files = ' '.join(["setup.py", "src/virtBootstrap/*.py"]) + output_format = "colorized" if sys.stdout.isatty() else "text" + + print(">>> Running pycodestyle ...") + cmd = "pycodestyle " + if (call(cmd + files, shell=True) != 0): + res = 1 + + print(">>> Running pylint ...") + args = "" + if self.errors_only: + args = "-E" + cmd = "pylint %s --output-format=%s " % (args, format(output_format)) + if (call(cmd + files, shell=True) != 0): + res = 1 + + sys.exit(res) + + setup( name='virt-bootstrap', version='0.1.0', @@ -53,5 +107,15 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6' - ] + + ], + cmdclass={ + 'pylint': CheckPylint + }, + extras_require={ + 'dev': [ + 'pylint', + 'pycodestyle' + ] + } ) diff --git a/src/virtBootstrap/sources.py b/src/virtBootstrap/sources.py index a1f8544..87278d2 100644 --- a/src/virtBootstrap/sources.py +++ b/src/virtBootstrap/sources.py @@ -15,6 +15,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Class definitions which process container image or +archive from source and unpack them in destination directory. +""" + import hashlib import json import shutil @@ -26,22 +31,24 @@ from subprocess import call, CalledProcessError, PIPE, Popen # Default virtual size of qcow2 image DEF_QCOW2_SIZE = '5G' -# default_image_dir - Path where Docker images (tarballs) will be stored if os.geteuid() == 0: - virt_sandbox_connection = "lxc:///" - default_image_dir = "/var/lib/virt-bootstrap/docker_images" + LIBVIRT_CONN = "lxc:///" + DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images" else: - virt_sandbox_connection = "qemu:///session" - default_image_dir = \ - os.environ['HOME'] + "/.local/share/virt-bootstrap/docker_images" + LIBVIRT_CONN = "qemu:///session" + DEFAULT_IMG_DIR = os.environ['HOME'] + DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images" def checksum(path, sum_type, sum_expected): + """ + Validate file using checksum. + """ algorithm = getattr(hashlib, sum_type) try: - fd = open(path, 'rb') - content = fd.read() - fd.close() + handle = open(path, 'rb') + content = handle.read() + handle.close() actual = algorithm(content).hexdigest() return actual == sum_expected @@ -50,9 +57,11 @@ def checksum(path, sum_type, sum_expected): def safe_untar(src, dest): - # Extract tarball in LXC container for safety + """ + Extract tarball within LXC container for safety. + """ virt_sandbox = ['virt-sandbox', - '-c', virt_sandbox_connection, + '-c', LIBVIRT_CONN, '-m', 'host-bind:/mnt=' + dest] # Bind destination folder # Compression type is auto detected from tar @@ -64,16 +73,22 @@ def safe_untar(src, dest): def get_layer_info(digest, image_dir): + """ + Get checksum type/value and path to corresponding tarball. + """ sum_type, sum_value = digest.split(':') layer_file = "{}/{}.tar".format(image_dir, sum_value) return (sum_type, sum_value, layer_file) def untar_layers(layers_list, image_dir, dest_dir): + """ + Untar each of layers from container image. + """ for layer in layers_list: sum_type, sum_value, layer_file = get_layer_info(layer['digest'], image_dir) - logging.info('Untar layer file: ({}) {}'.format(sum_type, layer_file)) + logging.info('Untar layer file: (%s) %s', sum_type, layer_file) # Verify the checksum if not checksum(layer_file, sum_type, sum_value): @@ -139,6 +154,9 @@ def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE): def extract_layers_in_qcow2(layers_list, image_dir, dest_dir): + """ + Extract docker layers in qcow2 images with backing chains. + """ qcow2_backing_file = None for index, layer in enumerate(layers_list): @@ -146,7 +164,7 @@ def extract_layers_in_qcow2(layers_list, image_dir, dest_dir): sum_type, sum_value, tar_file = \ get_layer_info(layer['digest'], image_dir) - logging.info('Untar layer file: ({}) {}'.format(sum_type, tar_file)) + logging.info('Untar layer file: (%s) %s', sum_type, tar_file) # Verify the checksum if not checksum(tar_file, sum_type, sum_value): @@ -160,17 +178,20 @@ def extract_layers_in_qcow2(layers_list, image_dir, dest_dir): qcow2_backing_file = qcow2_layer_file -class FileSource: - def __init__(self, url, username, password, fmt, insecure, no_cache): +class FileSource(object): + """ + Extract root filesystem from file. + """ + def __init__(self, url, args): self.path = url.path - self.output_format = fmt + self.output_format = args.format def unpack(self, dest): - ''' + """ Safely extract root filesystem from tarball @param dest: Directory path where the files to be extraced - ''' + """ if self.output_format == 'dir': logging.info("Extracting files into destination directory") safe_untar(self.path, dest) @@ -190,9 +211,13 @@ class FileSource: logging.info("Files are stored in: " + dest) -class DockerSource: - def __init__(self, url, username, password, fmt, insecure, no_cache): - ''' +class DockerSource(object): + """ + Extract files from Docker image + """ + + def __init__(self, url, args): + """ Bootstrap root filesystem from Docker registry @param url: Address of source registry @@ -201,33 +226,33 @@ class DockerSource: @param fmt: Format used to store image [dir, qcow2] @param insecure: Do not require HTTPS and certificate verification @param no_cache: Whether to store downloaded images or not - ''' + """ self.registry = url.netloc self.image = url.path - self.username = username - self.password = password - self.output_format = fmt - self.insecure = insecure - self.no_cache = no_cache + self.username = args.username + self.password = args.password + self.output_format = args.format + self.insecure = args.not_secure + self.no_cache = args.no_cache if self.image and not self.image.startswith('/'): self.image = '/' + self.image self.url = "docker://" + self.registry + self.image def unpack(self, dest): - ''' + """ Extract image files from Docker image @param dest: Directory path where the files to be extraced - ''' + """ if self.no_cache: tmp_dest = tempfile.mkdtemp('virt-bootstrap') images_dir = tmp_dest else: - if not os.path.exists(default_image_dir): - os.makedirs(default_image_dir) - images_dir = default_image_dir + if not os.path.exists(DEFAULT_IMG_DIR): + os.makedirs(DEFAULT_IMG_DIR) + images_dir = DEFAULT_IMG_DIR try: # Run skopeo copy into a tmp folder @@ -247,8 +272,8 @@ class DockerSource: check_call(skopeo_copy) # Get the layers list from the manifest - mf = open(images_dir+"/manifest.json", "r") - manifest = json.load(mf) + manifest_file = open(images_dir+"/manifest.json", "r") + manifest = json.load(manifest_file) # Layers are in order - root layer first # Reference: diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py index 572e603..3973720 100755 --- a/src/virtBootstrap/virt_bootstrap.py +++ b/src/virtBootstrap/virt_bootstrap.py @@ -16,6 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Main executable file which process input arguments +and calls corresponding methods on appropriate object. +""" + import argparse import gettext import sys @@ -27,7 +32,7 @@ try: except ImportError: from urllib.parse import urlparse -import sources +from virtBootstrap import sources gettext.bindtextdomain("virt-bootstrap", "/usr/share/locale") @@ -37,11 +42,19 @@ try: localedir="/usr/share/locale", codeset='utf-8') except IOError: - import __builtin__ - __builtin__.__dict__['_'] = unicode + try: + import __builtin__ + # pylint: disable=undefined-variable + __builtin__.__dict__['_'] = unicode + except ImportError: + import builtin + builtin.__dict__['_'] = str def get_source(args): + """ + Get object which match the source type + """ url = urlparse(args.uri) scheme = url.scheme @@ -51,26 +64,27 @@ def get_source(args): try: class_name = "%sSource" % scheme.capitalize() clazz = getattr(sources, class_name) - return clazz(url, - args.username, - args.password, - args.format, - args.not_secure, - args.no_cache) + return clazz(url, args) except Exception: raise Exception("Invalid image URI scheme: '%s'" % url.scheme) def set_root_password(rootfs, password): + """ + Set password on the root user in rootfs + """ users = 'root:%s' % password args = ['chpasswd', '-R', rootfs] - p = Popen(args, stdin=PIPE) - p.communicate(input=users) - if p.returncode != 0: - raise CalledProcessError(p.returncode, cmd=args, output=None) + chpasswd = Popen(args, stdin=PIPE) + chpasswd.communicate(input=users) + if chpasswd.returncode != 0: + raise CalledProcessError(chpasswd.returncode, cmd=args, output=None) def bootstrap(args): + """ + Get source object and call unpack method + """ source = get_source(args) if not os.path.exists(args.dest): os.makedirs(args.dest) @@ -123,12 +137,10 @@ def main(): bootstrap(args) sys.exit(0) - except KeyboardInterrupt as e: + except KeyboardInterrupt: sys.exit(0) - except ValueError as e: - for line in e: - for l in line: - sys.stderr.write("%s: %s\n" % (sys.argv[0], l)) + except ValueError as err: + sys.stderr.write("%s: %s\n" % (sys.argv[0], err)) sys.stderr.flush() sys.exit(1) |