diff options
author | Jan Pokorný <jpokorny@redhat.com> | 2013-11-18 11:14:43 +0100 |
---|---|---|
committer | Jan Pokorný <jpokorny@redhat.com> | 2013-11-18 11:14:43 +0100 |
commit | b182bb2e0a9266b313112d507e9803e5ef2de387 (patch) | |
tree | f348cb4d1b254d2e7fa8cc61a1f82d3b4a2e9e99 | |
download | clufter-b182bb2e0a9266b313112d507e9803e5ef2de387.tar.gz clufter-b182bb2e0a9266b313112d507e9803e5ef2de387.tar.xz clufter-b182bb2e0a9266b313112d507e9803e5ef2de387.zip |
Initial commit
Signed-off-by: Jan Pokorný <jpokorny@redhat.com>
72 files changed, 3418 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2610b39 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.py[co] +/tests/lon-test/ diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..6580369 --- /dev/null +++ b/__init__.py @@ -0,0 +1,8 @@ +name = __package__ +version = '0.1' +copyright = """\ +Copyright 2012 Red Hat, Inc. +Licensed under GPLv2 +""".rstrip() + +metadata = (name, version, copyright) diff --git a/command.py b/command.py new file mode 100644 index 0000000..b87dd53 --- /dev/null +++ b/command.py @@ -0,0 +1,8 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + + +def as_command(filters): + pass diff --git a/command_manager.py b/command_manager.py new file mode 100644 index 0000000..f7d3cd4 --- /dev/null +++ b/command_manager.py @@ -0,0 +1,14 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + + +class CommandManager(object): + """Class responsible to route commands to filters or other actions""" + def __init__(self, filter_manager): + self.filter_manager = filter_manager + + def __call__(self, args): + pass + # self.filter_manager(string) diff --git a/commands/__init__.py b/commands/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/commands/__init__.py diff --git a/commands/ccs2pcs.py b/commands/ccs2pcs.py new file mode 100644 index 0000000..141449a --- /dev/null +++ b/commands/ccs2pcs.py @@ -0,0 +1,20 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from ..command import as_command + + +@as_command(['ccs2ccsflat', ('ccs2pcs', 'ccs2coro')]) +def ccs2pcs(input='/etc/cluster/cluster.conf', + output='./cib.xml', + coro='/.corosync.conf'): + """Converts cman-based cluster configuration to Pacemaker-based one + + Options: + input input cman-based cluster configuration file + output output pacemaker-based configuration file + coro output Corosync configuration file + """ + return ('file', input), (('file', output), ('file', coro)) diff --git a/doc/HACKING b/doc/HACKING new file mode 100644 index 0000000..989bd6b --- /dev/null +++ b/doc/HACKING @@ -0,0 +1,8 @@ +Notable points: + +1. plugins has to use absolute (clufter.*) imports otherwise issues can + occur in some contexts of use + +2. please use only lower-cased identifiers for plugins, UPPER-CASED are + reserved for implicit ones (technically, not plugins, but can be used + on a few places accepting plugins) such as XML diff --git a/filter.py b/filter.py new file mode 100644 index 0000000..51340e1 --- /dev/null +++ b/filter.py @@ -0,0 +1,407 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Base filter stuff (metaclass, decorator, etc.)""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import logging +from os.path import dirname, join +from copy import deepcopy +from collections import OrderedDict, defaultdict + +from lxml import etree + +from .plugin_registry import PluginRegistry +from .utils import ClufterError, head_tail, hybridproperty, filtervarspop + +log = logging.getLogger(__name__) + +DEFAULT_ROOT_DIR = join(dirname(__file__), 'filters') + +CLUFTER_NS = 'http://people.redhat.com/jpokorny/ns/clufter' +XSL_NS = 'http://www.w3.org/1999/XSL/Transform' + +# XXX: consult standard/books +_TOP_LEVEL_XSL = ( + 'import', + 'include', + 'key', + 'namespace-alias', + 'attribute-set', + 'variable', + 'output', + 'template', + 'strip-space' +) +TOP_LEVEL_XSL = ["{{{0}}}{1}".format(XSL_NS, e) for e in _TOP_LEVEL_XSL] + + +class FilterError(ClufterError): + pass + + +class filters(PluginRegistry): + """Filter registry (to be used as a metaclass for filters)""" + pass + + +class Filter(object): + """Base for filters performing the actual conversion + + Base principles: + - protocols: string label denoting how to int-/externalize + - create filter instance = pass particular formats, + all = start conversion + """ + __metaclass__ = filters + + def __init__(self, in_format, out_format): + self._in_format, self._out_format = in_format, out_format + + @hybridproperty + def in_format(this): + """Input format identifier/class for the filter""" + return this._in_format + + @hybridproperty + def out_format(this): + """Output format identifier/class for the filter""" + return this._out_format + + def __call__(self, in_obj, **kwargs): + """Default is to use a function decorated with `deco`""" + out_decl = self._fnc(self, in_obj, **kwargs) + return self.out_format(*out_decl) + + @classmethod + def deco(cls, in_format, out_format): + """Decorator as an easy factory of actual filters""" + def deco_fnc(fnc): + log.debug("Filter: deco for {0}" + .format(fnc)) + attrs = { + '__module__': fnc.__module__, + '__doc__': fnc.__doc__, + '_in_format': in_format, + '_out_format': out_format, + '_fnc': staticmethod(fnc), + } + # optimization: shorten type() -> new() -> probe + ret = cls.probe(fnc.__name__, (cls, ), attrs) + return ret + return deco_fnc + + +def tag_log(s, elem): + """Logging helper""" + return s.format(elem.tag, ', '.join(':'.join(i) for i in elem.items())) + + +class XMLFilter(Filter): + @staticmethod + def _traverse(in_fmt, walk, walk_default=None, et=None, preprocess=lambda s, n, r: s, + proceed=lambda *x: x, + postprocess=lambda x: x[0] if len(x) == 1 else x): + """Generic traverse through XML as per symbols within schema tree""" + tree_stack = [('', (None, walk), OrderedDict())] + skip_until = [] + if walk_default is None: + skip_until = [('start', tag) for tag in walk] + et = et or in_fmt('etree') + + for context in etree.iterwalk(et, events=('start', 'end')): + event, elem = context + log.debug("Got: {0} {1}".format(event, elem.tag)) + if skip_until and (event, elem.tag) not in skip_until: + continue + log.debug("Not skipped: {0}".format(elem.tag)) + skip_until = () # reset skipping any time we get further + if event == 'start': + # going down + log.debug(tag_log("Moving downwards: {0} ({1})", elem)) + if elem.tag in tree_stack[-1][1][1] or walk_default is not None: + if elem.tag not in tree_stack[-1][1][1]: + log.debug("Not") + walk_new_sym, walk_new_rest = walk_default, tree_stack[-1][1][1].copy() + else: + walk_new_sym, walk_new_rest = tree_stack[-1][1][1][elem.tag] + walk_new_sym = preprocess(walk_new_sym, elem.tag, tree_stack[-1][1][0]) + tree_stack[-1][1][1][elem.tag] = (walk_new_sym, walk_new_rest) + tree_stack.append((elem.tag, (walk_new_sym, walk_new_rest), OrderedDict())) + if walk_new_rest is {}: + # safe optimization + skip_until = [('end', elem.tag)] + # XXX: optimization prunning, probably no good + #else: + # skip_until = [('end', tag) for tag in tree_stack[-1][1][1]] + # skip_until = [('end', elem.tag)] + # log.debug("Skipping (A) until: {0}".format(skip_until)) + + else: + # going up + log.debug(tag_log("Moving upwards: {0} ({1})", elem)) + log.debug("Expecting {0}".format(elem.tag)) + if elem.tag == tree_stack[-1][0]: + walk, children = tree_stack.pop()[1:3] + tree_stack[-1][2][elem] = proceed(walk[0], elem, children) + log.debug("Proceeded {0}".format( + etree.tostring(tree_stack[-1][2][elem]).replace('\n', ''))) + # XXX: optimization prunning, probably no good + #else: + # skip_until = [('end', tree_stack[-1][0])] + # log.debug("Skipping (C) until: {0}".format(skip_until)) + + ret = tree_stack[-1][2].values() + return postprocess(ret) + + @staticmethod + def _xslt_preprocess(sym, name, parent_sym=None): + """Preprocessing of schema tree XSLT snippets to real (sub)templates + + If callable is observed instead of XSLT snippet, keep it untouched. + Used by `proceed_xslt` and `get_template` methods (hence class-wide). + """ + if isinstance(sym, tuple): + return sym # already proceeded + if isinstance(sym, basestring): + log.debug("preprocessing {0}".format(sym)) + # XXX <xsl:output method="xml" + # XXX memoize as a constant + deepcopy + sym = ('<clufter:snippet' + ' xmlns:xsl="{0}"' + ' xmlns:clufter="{1}">' + ' {2}' + ' </clufter:snippet>'.format(XSL_NS, CLUFTER_NS, sym)) + ret = etree.XML(sym) + hooks = OrderedDict() + toplevel = [] + + #if len(ret) and parent_sym: + # top = filter(lambda x: x.tag in TOP_LEVEL_XSL, ret) + # for e in top: + # toplevel.append(e) + # ret.remove(e) + + if parent_sym and isinstance(parent_sym[0], etree._Element): + top = filter(lambda x: x.tag in TOP_LEVEL_XSL, parent_sym[0]) + for e in top: + print "at", sym, "appending", etree.tostring(e) + ret.append(deepcopy(e)) + #for e in toplevel: + # parent_sym[0].append(e) + + log.debug("walking {0}".format(etree.tostring(ret))) + for event, elem in etree.iterwalk(ret, events=('start', )): + # XXX xpath/specific tag filter + # register each recurse point at the tag required + # so it can be utilized in bottom-up pairing (from + # particular definitions to where it is expected) + + # not needed + #if elem is ret: + # continue + log.debug("Got {0}".format(elem.tag)) + if elem.tag == '{{{0}}}recursion'.format(CLUFTER_NS): + up = elem + walk = [] + while up != ret: + walk.append(up.getparent().index(up)) + up = up.getparent() + #walk = reversed(tuple(walk)) # XXX reversed, dangerous? + walk.reverse() + walk = tuple(walk) + at = elem.attrib.get('at', '*') + prev = hooks.setdefault(at, walk) + if prev is not walk: + raise FilterError(None, "Ambigous match for `{0}'" + " tag ({1} vs {2})".format(at, walk, prev)) + + log.debug("hooks {0}".format(hooks)) + return (ret, hooks) + elif callable(sym): + return sym + else: + log.debug("preprocess XSLT traverse symbols: skipping {0}" + .format(name)) + return None + + @classmethod + def proceed_xslt(cls, in_obj, **kwargs): + """Apply iteratively XSLT snippets as per the schema tree (walk)""" + # XXX postprocess: omitted as standard defines the only root element + + def proceed(transformer, elem, children): + if not callable(transformer): + # expect (xslt, hooks) + return do_proceed(transformer, elem, children) + return transformer(elem, children) + + def do_proceed(xslt, elem, children): + # in bottom-up manner + snippet = deepcopy(xslt[0]) # in-situ template manipulation + hooks = xslt[1] + scheduled = OrderedDict() # XXX to keep the law and order + for _, c_elem in etree.iterwalk(elem, events=('start',)): + if c_elem is elem: + continue + if c_elem in children: + c_up = c_elem + while not c_up.tag in hooks and c_up.getparent() != elem: + c_up = c_up.getparent() + if c_up.tag in hooks or '*' in hooks: + target_tag = c_up.tag if c_up.tag in hooks else '*' + l = scheduled.setdefault(hooks[target_tag], []) + l.append(children[c_elem].getroot()) + for index_history, substitutes in scheduled.iteritems(): + #inserted = False + tag = reduce(lambda x, y: x[y], index_history, snippet) + parent = tag.getparent() + #index = parent.index(tag) + + for s in substitutes: + #assert s.tag == "{{{0}}}snippet".format(CLUFTER_NS) + if s.tag == "{{{0}}}snippet".format(CLUFTER_NS): + # only single root "detached" supported (first == last) + dst = parent + dst.attrib.update(dict(s.attrib)) + dst.extend(s) + else: + parent.append(s) + + cl = snippet.xpath("//clufter:recursion", + namespaces={'clufter': CLUFTER_NS}) + if len(cl): + log.info("Not all tags from clufter namespace used") + # remove these remnants so cleanup_namespaces works well + for e in cl: + e.getparent().remove(e) + + # xslt + xslt_root = etree.Element('{{{0}}}stylesheet'.format(XSL_NS), + version="1.0") + top = filter(lambda x: x.tag in TOP_LEVEL_XSL, snippet) + for e in top: + print "e", etree.tostring(e) + xslt_root.append(e) + if len(snippet): + log.debug("snippet {0}".format(etree.tostring(snippet))) + template = etree.Element('{{{0}}}template'.format(XSL_NS), + match=elem.tag) + template.extend(snippet) # XXX was append + xslt_root.append(template) + print "ee", etree.tostring(xslt_root) + #else: + # # we dont't apply if there is nothing local and not at root + # print "zdrham", elem.tag + # return elem + + elem = etree.ElementTree(elem) # XXX not getroottree? + log.debug("Applying {0}, {1}".format(type(elem), etree.tostring(elem))) + log.debug("Applying on {0}".format(etree.tostring(xslt_root))) + #ret = elem.xslt(xslt_root) + xslt = etree.XSLT(xslt_root) + ret = xslt(elem) + #etree.cleanup_namespaces(ret) + + return ret + + def postprocess(ret): + assert len(ret) == 1 + ret = ret[0] + log.debug("Applying postprocess onto {0}".format(etree.tostring(ret))) + if ret.getroot().tag == "{{{0}}}snippet".format(CLUFTER_NS): + ret = ret.getroot()[0] + # XXX: ugly solution to get rid of the unneeded namespace + # (cleanup_namespaces did not work here) + ret = etree.fromstring(etree.tostring(ret)) + etree.cleanup_namespaces(ret) + return ret + + kwargs.update(preprocess=cls._xslt_preprocess, proceed=proceed, + postprocess=postprocess, sparse=True) + return cls.proceed(in_obj, **kwargs) + + @classmethod + def _xslt_template(cls, walk): + """Generate (try to) complete XSLT template from the sparse snippets""" + scheduled_walk = [walk] + scheduled_subst = OrderedDict() + ret = [] + while len(scheduled_walk): + cur_walk = scheduled_walk.pop() + for key, (transformer, children) in cur_walk.iteritems(): + scheduled_walk.append(children) + if transformer is None or callable(transformer): + if callable(transformer): + log.warn("Cannot generate complete XSLT when callable" + " present") + if key in scheduled_subst: + for tag in scheduled_subst.pop(key)[:]: + for child_tag in children.iterkeys(): + l = scheduled_subst.setdefault(child_tag, []) + l.append(tag) + continue + + snippet = deepcopy(transformer[0]) # in-situ manipulation + + xslt_root = etree.Element('{{{0}}}stylesheet'.format(XSL_NS), + version="1.0") + top = filter(lambda x: x.tag in TOP_LEVEL_XSL, snippet) + for e in top: + xslt_root.append(e) + if len(snippet): + snippet.tag = '{{{0}}}template'.format(XSL_NS) + snippet.attrib['match'] = key + xslt_root.append(snippet) + + hooks = transformer[1] + if not key in scheduled_subst: + if cur_walk is walk: + ret.append(xslt_root) + else: + raise FilterError(cls, "XSLT inconsistency 1") + # in parallel: 1? + if key in scheduled_subst: + for tag in scheduled_subst.pop(key): + e = etree.Element('{http://www.w3.org/1999/XSL/Transform}apply-templates', + select=".//{0}".format(key)) + parent = tag.getparent() + parent[parent.index(tag)] = e + ret[-1].append(snippet) + # in parallel: 2? + for target_tag, index_history in hooks.iteritems(): + tag = reduce(lambda x, y: x[y], index_history, snippet) + l = scheduled_subst.setdefault(target_tag, []) + l.append(tag) + + assert not len(scheduled_subst) # XXX either fail or remove forcibly + map(lambda x: etree.cleanup_namespaces(x), ret) + ret = map(lambda x: x, ret) + + return (lambda x: x[0] if len(x) == 1 else x)(ret) + + @classmethod + def proceed(cls, in_obj, root_dir=DEFAULT_ROOT_DIR, **kwargs): + """Push-button to be called from the filter itself""" + d = dict(symbol=cls.name) + d.update(kwargs) + walk = in_obj.walk_schema(root_dir, **filtervarspop(d, ( + 'symbol', 'sparse'))) + return cls._traverse(in_obj, walk, **d) + + @classmethod + def proceed_xslt_filter(cls, in_obj, **kwargs): + """Push-button to be called from the filter itself (with walk_default)""" + # identity transform + kwargs['walk_default'] = "" + return cls.proceed_xslt(in_obj, **kwargs) + + @classmethod + def get_template(cls, in_obj, root_dir=DEFAULT_ROOT_DIR, **kwargs): + """Generate the overall XSLT template""" + d = dict(symbol=cls.name) + d.update(kwargs) + walk = in_obj.walk_schema(root_dir, preprocess=cls._xslt_preprocess, + sparse=False, **filtervarspop(d, ('symbol',))) + return cls._xslt_template(walk) diff --git a/filter_manager.py b/filter_manager.py new file mode 100644 index 0000000..f5a13fe --- /dev/null +++ b/filter_manager.py @@ -0,0 +1,57 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Filter manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import logging + +from .filter import filters + +log = logging.getLogger(__name__) + + +class FilterManager(object): + """Class responsible to manage filters and filtering itself""" + def __init__(self, fmt_mgr, registry=filters, paths=(), filters={}): + self._registry = registry + filters = dict(registry.discover(paths), **filters) + log.debug("Filters before resolving: {0}" + .format(filters)) + self._filters = self._resolve(fmt_mgr.formats, filters) + + @staticmethod + def _resolve(formats, filters): + for flt_name, flt_cls in filters.items(): + in_format = formats.get(flt_cls.in_format) + out_format = formats.get(flt_cls.out_format) + if in_format is not None and out_format is not None: + log.debug("Resolve at `{0}' filter: `{1}' -> {2}," + " `{3}' -> {4}" + .format(flt_name, flt_cls.in_format, in_format, + flt_cls.out_format, out_format)) + filters[flt_name] = flt_cls(in_format, out_format) + continue + # drop the filter if cannot resolve either format + if not in_format: + log.warning("Resolve at `{0}' filter: `{1}' input format fail" + .format(flt_name, flt_cls.in_format)) + if not out_format: + log.warning("Resolve at `{0}' filter: `{1}' output format fail" + .format(flt_name, flt_cls.out_format)) + filters.pop(flt_name) + return filters + + @property + def filters(self): + return self._filters.copy() + + @property + def registry(self): + return self._registry + + def __call__(self, which, in_decl, **kwargs): + flt = self._filters[which] + in_obj = flt.in_format(*in_decl) + return flt(in_obj, **kwargs) diff --git a/filters/XML2simpleconfig.py b/filters/XML2simpleconfig.py new file mode 100644 index 0000000..6799ad1 --- /dev/null +++ b/filters/XML2simpleconfig.py @@ -0,0 +1,14 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from clufter.filter import XMLFilter +from lxml import etree + + +@XMLFilter.deco('XML', 'simpleconfig') +def xml2simpleconfig(flt, in_obj): + for context in etree.iterwalk(in_obj('etree'), events=('start', 'end')): + pass + pass diff --git a/filters/__init__.py b/filters/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/__init__.py diff --git a/filters/ccs2ccsflat.py b/filters/ccs2ccsflat.py new file mode 100644 index 0000000..889e006 --- /dev/null +++ b/filters/ccs2ccsflat.py @@ -0,0 +1,38 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from os.path import pardir +from subprocess import Popen, PIPE +import logging + +from clufter.filter import Filter, FilterError +from clufter.utils import which + +log = logging.getLogger(__name__) +# XXX +CCS_FLATTEN = which('ccs_flatten', pardir) or '' + + +@Filter.deco('ccs', 'ccsflat') +def ccs2ccsflat(self, in_obj, verify=False): + # XXX currently ccs_flatten does not handle stdin (tempfile.mkstemp?) + if verify: + in_obj.verify() + in_file = in_obj('file') + try: + command = [CCS_FLATTEN, in_file] + log.info("running `{0}'".format(' '.join(command))) + proc = Popen(command, stdout=PIPE, stderr=PIPE) + except OSError: + raise FilterError(self, "ccs_flatten binary seems unavailable") + out, err = proc.communicate() + if proc.returncode != 0: + raise FilterError(self, "ccs_flatten exit code: {0}\n\t{1}", + proc.returncode, err) + elif out == '': + # "No resource trees defined; nothing to do" + with file(in_file, 'r') as f: + out = f.read() + return ('bytestring', out) diff --git a/filters/ccs2coro.py b/filters/ccs2coro.py new file mode 100644 index 0000000..cd541c0 --- /dev/null +++ b/filters/ccs2coro.py @@ -0,0 +1,13 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from clufter.filter import XMLFilter +#from lxml import etree + + +@XMLFilter.deco('ccs', 'coroxml') +def ccs2coroxml(flt, in_obj): + #print etree.tostring(in_obj('etree').xslt(flt.get_template(in_obj))) + return ('etree', flt.proceed_xslt(in_obj)) diff --git a/filters/ccs_obfuscate_credentials.py b/filters/ccs_obfuscate_credentials.py new file mode 100644 index 0000000..c046b0f --- /dev/null +++ b/filters/ccs_obfuscate_credentials.py @@ -0,0 +1,13 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from clufter.filter import XMLFilter +#from lxml import etree + + +@XMLFilter.deco('ccs', 'ccs') +def ccs_obfuscate_credentials(flt, in_obj): + #print etree.tostring(in_obj('etree').xslt(flt.get_template(in_obj))) + return ('etree', flt.proceed_xslt_filter(in_obj, symbol='obfuscate_credentials')) diff --git a/filters/ccs_obfuscate_identifiers.py b/filters/ccs_obfuscate_identifiers.py new file mode 100644 index 0000000..b01b0bb --- /dev/null +++ b/filters/ccs_obfuscate_identifiers.py @@ -0,0 +1,13 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from clufter.filter import XMLFilter +#from lxml import etree + + +@XMLFilter.deco('ccs', 'ccs') +def ccs_obfuscate_identifiers(flt, in_obj): + #print etree.tostring(in_obj('etree').xslt(flt.get_template(in_obj))) + return ('etree', flt.proceed_xslt_filter(in_obj, symbol='obfuscate_identifiers')) diff --git a/filters/ccsflat2pcs.py b/filters/ccsflat2pcs.py new file mode 100644 index 0000000..df8f697 --- /dev/null +++ b/filters/ccsflat2pcs.py @@ -0,0 +1,11 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +from clufter.filter import Filter + + +@Filter.deco('ccsflat', 'pcs') +def ccsflat2pcs(flt, in_obj): + pass diff --git a/filters/cib/__init__.py b/filters/cib/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cib/__init__.py diff --git a/filters/cluster/__init__.py b/filters/cluster/__init__.py new file mode 100644 index 0000000..965ec03 --- /dev/null +++ b/filters/cluster/__init__.py @@ -0,0 +1,94 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <corosync> + <clufter:recursion at="clusternodes"/> + <clufter:recursion at="cman"/> + <clufter:recursion at="logging"/> + <totem version="2" + cluster_name="{@name}"> + <clufter:recursion at="totem"/> + </totem> + </corosync> +''' + +ccsflat2pcs = '''\ + <cib validate-with="pacemaker-1.1" admin_epoch="1" epoch="1" num_updates="0" have-quorum="1"> + <configuration> + <crm_config> + <!-- cluster_property_set id="cib-bootstrap-options"> + <nvpair id="startup-fencing" name="startup-fencing" value="true"/> + <nvpair id="stonith-enabled" name="stonith-enabled" value="true"/> + <nvpair id="default-resource-stickiness" name="default-resource-stickiness" value="INFINITY"/> + </cluster_property_set --> + </crm_config> + <clufter:recursion at="clusternodes"/> + <clufter:recursion at="rm"/> + </configuration> + <status/> + </cib> +''' + +obfuscate_credentials = '''\ + <xsl:template match="*"> + <xsl:copy> + <xsl:copy-of select="@*"/> + <clufter:recursion/> + </xsl:copy> + </xsl:template> +''' + +# check http://stackoverflow.com/questions/4509662/how-to-generate-unique-string +# todo: comments on top-level not supported +obfuscate_identifiers = '''\ + <xsl:template match="@*|node()"> + <xsl:copy> + <xsl:apply-templates select="@*|node()"/> + <!--clufter:recursion/--> + </xsl:copy> + </xsl:template> + + <xsl:template match="cluster/@name"> + <xsl:attribute name="{name()}"> + <xsl:value-of select="'CLUSTER'"/> + </xsl:attribute> + </xsl:template> + + <xsl:variable name="ClusterNode" select="cluster/clusternodes/clusternode[@name]"/> + <xsl:template match="cluster/clusternodes/clusternode/@name + |cluster/clusternodes/clusternode/fence/method/device/@nodename + |cluster/rm/failoverdomains/failoverdomain/failoverdomainnode/@name"> + <xsl:variable name="attr_name" select="."/> + <xsl:attribute name="{name()}"> + <xsl:value-of select="concat('CLUSTER-NODE-UNDEF-', generate-id(.))"/> + </xsl:attribute> + <xsl:attribute name="{name()}"> + <xsl:for-each select="$ClusterNode"> + <xsl:if test="@name = $attr_name"> + <xsl:value-of select="concat('CLUSTER-NODE-', position())"/> + </xsl:if> + </xsl:for-each> + </xsl:attribute> + <xsl:apply-templates select="@*|node()"/> + </xsl:template> + + <xsl:variable name="FenceDevice" select="cluster/fencedevices/fencedevice[@name]"/> + <xsl:template match="cluster/fencedevices/fencedevice/@name + |cluster/clusternodes/clusternode/fence/method/device/@name"> + <xsl:variable name="attr_name" select="."/> + <xsl:attribute name="{name()}"> + <xsl:value-of select="concat('FENCE-DEVICE-UNDEF-', generate-id(.))"/> + </xsl:attribute> + <xsl:attribute name="{name()}"> + <xsl:for-each select="$FenceDevice"> + <xsl:if test="@name = $attr_name"> + <xsl:value-of select="concat('FENCE-DEVICE-', position())"/> + </xsl:if> + </xsl:for-each> + </xsl:attribute> + <xsl:apply-templates select="@*|node()"/> + </xsl:template> +''' diff --git a/filters/cluster/clusternodes/__init__.py b/filters/cluster/clusternodes/__init__.py new file mode 100644 index 0000000..5136ea7 --- /dev/null +++ b/filters/cluster/clusternodes/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <nodelist> + <clufter:recursion at="clusternode"/> + </nodelist> +''' diff --git a/filters/cluster/clusternodes/clusternode/__init__.py b/filters/cluster/clusternodes/clusternode/__init__.py new file mode 100644 index 0000000..1017155 --- /dev/null +++ b/filters/cluster/clusternodes/clusternode/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <node id="{@nodeid}" ring0_addr="{@name}"/> +''' + +aobfuscate_identifiers = '''\ + <xsl:copy> + <xsl:copy-of select="@*"/> + <xsl:attribute name="name"> + <!-- xsl:value-of select="concat('NODE-', @nodeid)"/ --> + <xsl:value-of select="concat('NODE-', count())"/> + </xsl:attribute> + </xsl:copy> +''' diff --git a/filters/cluster/clusternodes/clusternode/fence/__init__.py b/filters/cluster/clusternodes/clusternode/fence/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/clusternodes/clusternode/fence/__init__.py diff --git a/filters/cluster/clusternodes/clusternode/fence/method/__init__.py b/filters/cluster/clusternodes/clusternode/fence/method/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/clusternodes/clusternode/fence/method/__init__.py diff --git a/filters/cluster/clusternodes/clusternode/fence/method/device.py b/filters/cluster/clusternodes/clusternode/fence/method/device.py new file mode 100644 index 0000000..c3e84ae --- /dev/null +++ b/filters/cluster/clusternodes/clusternode/fence/method/device.py @@ -0,0 +1,24 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +obfuscate_credentials = '''\ + <xsl:copy> + <xsl:copy-of select="@*"/> + <xsl:for-each select="@*"> + <xsl:if test="contains(concat( + '|login', + '|'), concat('|', name(), '|'))"> + <xsl:attribute name="{name()}">SECRET-LOGIN</xsl:attribute> + </xsl:if> + <xsl:if test="contains(concat( + '|community', + '|passwd', + '|snmp_priv_passwd', + '|'), concat('|', name(), '|'))"> + <xsl:attribute name="{name()}">SECRET-PASSWORD</xsl:attribute> + </xsl:if> + </xsl:for-each> + </xsl:copy> +''' diff --git a/filters/cluster/cman/__init__.py b/filters/cluster/cman/__init__.py new file mode 100644 index 0000000..b547bc3 --- /dev/null +++ b/filters/cluster/cman/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <quorum provider="corosync_votequorum"> + <xsl:copy-of select="@*[ + contains(concat( + '|expected_votes', + '|two_node', + '|'), concat('|', name(), '|'))]" /> + </quorum> +''' diff --git a/filters/cluster/fencedaemon/__init__.py b/filters/cluster/fencedaemon/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/fencedaemon/__init__.py diff --git a/filters/cluster/fencedevices/__init__.py b/filters/cluster/fencedevices/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/fencedevices/__init__.py diff --git a/filters/cluster/fencedevices/fencedevice.py b/filters/cluster/fencedevices/fencedevice.py new file mode 100644 index 0000000..dc571e5 --- /dev/null +++ b/filters/cluster/fencedevices/fencedevice.py @@ -0,0 +1,39 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +obfuscate_credentials = '''\ + <xsl:copy> + <xsl:copy-of select="@*"/> + <xsl:for-each select="@*[contains(concat( + '|passwd', + '|snmp_priv_passwd', + '|'), concat('|', name(), '|'))]"> + <xsl:attribute name="{name()}">SECRET-PASSWORD</xsl:attribute> + </xsl:for-each> + <xsl:for-each select="@*[contains(concat( + '|login', + '|'), concat('|', name(), '|'))]"> + <xsl:attribute name="{name()}">SECRET-LOGIN</xsl:attribute> + </xsl:for-each> + </xsl:copy> +''' + + +obfuscate_identifiers = '''\ + <xsl:copy> + <xsl:copy-of select="@*"/> + <xsl:for-each select="@*[contains(concat( + '|passwd', + '|snmp_priv_passwd', + '|'), concat('|', name(), '|'))]"> + <xsl:attribute name="{name()}">SECRET-PASSWORD</xsl:attribute> + </xsl:for-each> + <xsl:for-each select="@*[contains(concat( + '|login', + '|'), concat('|', name(), '|'))]"> + <xsl:attribute name="{name()}">SECRET-LOGIN</xsl:attribute> + </xsl:for-each> + </xsl:copy> +''' diff --git a/filters/cluster/logging/__init__.py b/filters/cluster/logging/__init__.py new file mode 100644 index 0000000..ad78812 --- /dev/null +++ b/filters/cluster/logging/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <logging> + <!-- XXX: the latter match (if any) should overwrite the former --> + <clufter:recursion at="logging_daemon"/> + </logging> +''' diff --git a/filters/cluster/logging/logging_daemon/__init__.py b/filters/cluster/logging/logging_daemon/__init__.py new file mode 100644 index 0000000..05884ff --- /dev/null +++ b/filters/cluster/logging/logging_daemon/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <xsl:for-each select="self::node()[@name='corosync' and @subsys]"> + <logger_subsys> + <xsl:copy-of select="@*[ + contains(concat( + '|debug', + '|logfile', + '|subsys', + '|to_logfile', + '|to_syslog', + '|'), concat('|', name(), '|'))]" /> + </logger_subsys> + </xsl:for-each> +''' diff --git a/filters/cluster/rm/__init__.py b/filters/cluster/rm/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/rm/__init__.py diff --git a/filters/cluster/rm/failoverdomains/__init__.py b/filters/cluster/rm/failoverdomains/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/rm/failoverdomains/__init__.py diff --git a/filters/cluster/rm/failoverdomains/failoverdomain/__init__.py b/filters/cluster/rm/failoverdomains/failoverdomain/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/filters/cluster/rm/failoverdomains/failoverdomain/__init__.py diff --git a/filters/cluster/rm/failoverdomains/failoverdomain/failoverdomainnode.py b/filters/cluster/rm/failoverdomains/failoverdomain/failoverdomainnode.py new file mode 100644 index 0000000..4a2c258 --- /dev/null +++ b/filters/cluster/rm/failoverdomains/failoverdomain/failoverdomainnode.py @@ -0,0 +1,24 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +obfuscate_identifiers = '''\ + <xsl:copy> + <xsl:copy-of select="@*"/> + <xsl:if test="@name"> + <xsl:attribute name="name"> + <xsl:for-each select="(key('clusternode_key', @name) | .)[0]"> + <xsl:choose> + <xsl:when test="generate-id() = generate-id(current())"> + <xsl:value-of select="'UNTRACKED-NODE'"/> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="concat('NODE-', position())"/> + </xsl:otherwise> + </xsl:choose> + </xsl:for-each> + </xsl:attribute> + </xsl:if> + </xsl:copy> +''' diff --git a/filters/cluster/totem/__init__.py b/filters/cluster/totem/__init__.py new file mode 100644 index 0000000..7834c4a --- /dev/null +++ b/filters/cluster/totem/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <xsl:copy-of select="@*[ + contains(concat( + '|consensus', + '|fail_recv_const', + '|join', + '|max_messages', + '|miss_count_const', + '|netmtu', + '|rrp_mode', + '|rrp_problem_count_threshold', + '|secauth', + '|seqno_unchanged_const', + '|token', + '|token_retransmits_before_loss_const', + '|window_size', + '|'), concat('|', name(), '|'))]" /> + <clufter:recursion at="interface"/> +''' diff --git a/filters/cluster/totem/interface/__init__.py b/filters/cluster/totem/interface/__init__.py new file mode 100644 index 0000000..f41dd99 --- /dev/null +++ b/filters/cluster/totem/interface/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +ccs2coroxml = '''\ + <xsl:copy> + <xsl:copy-of select="@*[ + contains(concat( + '|ringnumber', + '|bindnetaddr', + '|broadcast', + '|mcastaddr', + '|mcastport', + '|ttl', + '|'), concat('|', name(), '|'))]" /> + </xsl:copy> +''' diff --git a/format.py b/format.py new file mode 100644 index 0000000..8dd7108 --- /dev/null +++ b/format.py @@ -0,0 +1,310 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Base format stuff (metaclass, classes, etc.)""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +# TODO: NamedTuple for tree_stack + +import imp +import logging +from copy import deepcopy +from os import extsep, walk +from os.path import commonprefix, splitext, basename +from sys import modules + +from lxml import etree + +from .plugin_registry import PluginRegistry +from .utils import ClufterError, classproperty + +log = logging.getLogger(__name__) +MAX_DEPTH = 1000 + + +class FormatError(ClufterError): + pass + + +class formats(PluginRegistry): + """Format registry (to be used as a metaclass for formats)""" + use_local = True + + def __init__(cls, name, bases, attrs): + cls._protocols = {} + # protocols merge: top-down through inheritance + for base in reversed(cls.__bases__): + if hasattr(base, '_protocols'): + cls._protocols.update(base._protocols) + # updated with locally defined protocols (marked by `producing` wrapper) + for attr, obj in attrs.iteritems(): + if hasattr(obj, '_protocol'): + cls._protocols.update(**{obj._protocol: obj}) + + +def producing(protocol, protect=False): + """Decorator for externalization method understood by the `formats` magic + + Caching of representations is a bonus.""" + def deco_meth(meth): + def deco_args(self, protocol, protect_safe=False, *args): + try: + produced = self._representations[protocol] + except KeyError: + produced = meth(self, protocol, *args) + # computed -> stored normalization: tuple nontupled or len == 1 + if not isinstance(produced, tuple) or len(produced) == 1: + produced = (produced, ) + self._representations[protocol] = produced + + # stored -> computed normalization: detuple if len == 1 + if isinstance(produced, tuple) and len(produced) == 1: + produced = produced[0] + if protect and not protect_safe: + log.debug("{0}:{1}:Forced deepcopy of `{2}' instance" + .format(self.__class__.name, meth.__name__, + type(produced).__name__)) + produced = deepcopy(produced) + return produced + + setattr(deco_args, '_protocol', protocol) # mark for later recognition + return deco_args + return deco_meth + + +class Format(object): + """Base for configuration formats + + Base principles: + - protocols: string label denoting how to int-/externalize + - union of protocols within inheritance hierarchy and + locally defined ones (prioritized) + - to define one, add a method decorated with `@producing(<proto>)` + - be default, all such protocols are suitable for both + int- and externalization, but you can prevent the latter + context by raising an exception in the method body + - create format instance = internalize, call = externalize + - protocols are property of the class, representation of an instance + + + Little bit of explanation: + + FORMAT INSTANCE + (concrete data) + INSTANTIATION /-----------------------\ CALL /-------- + (internalize) | -protocols = set(...) | (externalize) | effect + ----------------->| -representations = |------------------>| and/or ... + (protocol, *args) | {protocol: (*args)} | (protocol, *args) | value + ^ \-----------------------/ ^ \-------- + | | + | | + +-- adds (protocol, *args) to representations, | + usually does nothing else (lazy approach) | + | + with each protocol there is a representation --+ + of concrete data associated, either this or + any other that can be promoted (awakening + from the previous laziness) to the desired + one is required + + Externalization methods are marked with `producing` decorator + that takes protocol name as a parameter. + + """ + __metaclass__ = formats + + def swallow(self, protocol, *args): + """"Called by implicit constructor to get a format instance""" + if protocol == 'native': + protocol = self.native_protocol + + assert protocol in self._protocols + prev = self._representations.setdefault(protocol, args) + assert prev is args + + def produce(self, protocol, *args, **kwargs): + """"Called by implicit invocation to get data externalized""" + if protocol == 'native': + protocol = self.native_protocol + + assert protocol in self._protocols + return self._protocols[protocol](self, protocol, *args, **kwargs) + + def __init__(self, protocol, *args): + """Format constructor, i.e., object = concrete internal data""" + self._representations = {} + self.swallow(protocol, *args) + + def __call__(self, protocol='native', *args, **kwargs): + """Way to externalize object's internal data""" + return self.produce(protocol, *args) + + @property + def protocols(self): + """Set of supported protocols for int-/externalization""" + return self._protocols.copy() # installed by meta-level + + @property + def representations(self): + """Mapping of `protocol: initializing_data`""" + return self._representations.copy() + + #### + + native_protocol = 'bytestring' + + @producing('bytestring') + def get_bytestring(self, protocol): + if 'file' in self._representations: # break the possible loop + with file(self('file'), 'rb') as f: + return f.read() + + @producing('file') + def get_file(self, protocol, filename): + with file(filename, 'wb') as f: + f.write(self('bytestring')) + return filename + + +class XML(Format): + """"Base for XML-based configuration formats""" + @classproperty + def root(self): + """Root tag of the XML document""" + raise ValueError # NotImplemented + + @classproperty + def rng_schema(self): + """Relax-ng schema for validation document""" + return None + + @staticmethod + def _walk_schema_step_up(tree_stack): + """Step up within the tree_stack (bottom-up return in dir structure)""" + child_root, child_data = (lambda x, *y: (x, y))(*tree_stack.pop()) + log.debug("Moving upwards: `{0}' -> `{1}'" + .format(child_root, tree_stack[-1][0])) + current_tracking = tree_stack[-1][2] + if len(child_data[1]): + name = basename(child_root) + if name in current_tracking: + to_update = current_tracking[name][1] + else: + to_update = current_tracking + to_update.update(child_data[1]) + + return current_tracking + + @staticmethod + def _walk_schema_step_down(tree_stack, root): + """Step down within the tree_stack (top-down diving in dir structure) + + Based on the fact that we traverse down by one level to already + (shallowly) explored level so the item is already tracked at + the parent and the shallow knowledge (dict) is passed down to be + potentially extended -- as dict is mutable, this knowledge + is shared between parent and child level. + """ + log.debug("Moving downwards:`{0}' -> `{1}'" + .format(tree_stack[-1][0], root)) + tree_stack.append((root, None, {})) + + return tree_stack[-1][2] # current tracking + + @classmethod + def walk_schema(cls, root_dir, symbol=None, preprocess=lambda s, n: s, + sparse=True): + """ + Get recipe for visiting symbol(s) within the XML as (sparsely) arranged + + Example of output:: + + { + 'A': (<symbol>, { + 'C': (<symbol>, {}), + 'D': (<symbol>, { + 'F': (<symbol>, { + }) + }) + }) + 'Z': (<symbol>, {}) + } + + NB: order of keys really does not matter. + """ + xml_root = cls.root + particular_namespace = '.'.join((cls.namespace, symbol or xml_root)) + result = {} + tree_stack = [(root_dir, None, result)] # for bottom-up reconstruction + for root, dirs, files in walk(root_dir): + # multi-step upwards and (followed by)/or single step downwards + while commonprefix((root, tree_stack[-1][0])) != tree_stack[-1][0]: + cls._walk_schema_step_up(tree_stack) + if root != tree_stack[-1][0]: + current_tracking = cls._walk_schema_step_down(tree_stack, root) + else: + assert root == root_dir + # at root, we do not traverse to any other dir than `xml_root` + map(lambda d: d != xml_root and dirs.remove(d), dirs[:]) + current_tracking = tree_stack[-1][2] + + for i in dirs + files: + name, ext = splitext(i) # does not hurt even if it is a dir + if name.startswith('_') or i in files and ext != extsep + 'py': + continue + + log.debug("Trying `{0}' at `{1}'".format(name, root)) + mfile, mpath, mdesc = imp.find_module(name, [root]) + # need to obfuscate the name due to, e.g., "logging" clash + mname = '.'.join((particular_namespace, 'walk_' + name)) + # suppress problems with missing parent in module hierarchy + modules.setdefault(particular_namespace, modules[__name__]) + if mname in modules: + mod = modules[mname] + if hasattr(mod, '__path__') and mod.__path__[0] != mpath: + # XXX robust? + raise FormatError(cls, "`{0}' already present" + .format(mname)) + else: + try: + mod = imp.load_module(mname, mfile, mpath, mdesc) + except ImportError: + log.debug("Cannot load `{0}'".format(mpath)) + continue + finally: + if mfile: + mfile.close() + + available = set(dir(mod)) - set(dir(type(mod))) + swag = None + if not symbol or symbol in available: + swag = getattr(mod, symbol) if symbol else tuple(available) + swag = preprocess(swag, name) + if swag is None and (sparse or i in files): + continue # files are terminals anyway + current_tracking[name] = (swag, {}) + + for i in xrange(MAX_DEPTH): + if cls._walk_schema_step_up(tree_stack) is result: + return result + else: + raise RuntimeError('INFLOOP detected') + + ### + + native_protocol = 'etree' + + @producing('bytestring') + def get_bytestring(self, protocol): + ret = super(XML, self).get_bytestring(self) + if ret is not None: + return ret + + # fallback + return etree.tostring(self('etree', protect_safe=True), + pretty_print=True) + + @producing('etree', protect=True) + def get_etree(self, protocol): + return etree.fromstring(self('bytestring')).getroottree() diff --git a/format_manager.py b/format_manager.py new file mode 100644 index 0000000..a78ebea --- /dev/null +++ b/format_manager.py @@ -0,0 +1,23 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Format manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +from .format import formats + + +class FormatManager(object): + """Class responsible for available formats of data to be converted""" + def __init__(self, registry=formats, paths=(), formats={}): + self._registry = registry + self._formats = dict(registry.discover(paths), **formats) + + @property + def registry(self): + return self._registry + + @property + def formats(self): + return self._formats.copy() diff --git a/formats/__init__.py b/formats/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/formats/__init__.py diff --git a/formats/ccs.py b/formats/ccs.py new file mode 100644 index 0000000..dd254b2 --- /dev/null +++ b/formats/ccs.py @@ -0,0 +1,25 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Cluster configuration system (ccs) format""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +from ..format import XML + + +class ccs(XML): + """Cman-based cluster stack configuration (cluster.conf) + + Sometimes called Cluster Configuration System (ccs). + """ + # XML + root = 'cluster' + + +class ccsflat(ccs): + """Cman-based cluster stack configuration (cluster.conf) + + Sometimes (ehm, exclusively by me) called Cluster Configuration System Flat. + """ + pass diff --git a/formats/coro.py b/formats/coro.py new file mode 100644 index 0000000..8f46b18 --- /dev/null +++ b/formats/coro.py @@ -0,0 +1,17 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Corosync executive configuration""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +from ..format import XML + + +class coroxml(XML): + """Corosync executive configuration, XML version (corosync.xml) + + See corosync.xml(5). + """ + # XMLFormat + root = 'corosync' diff --git a/formats/corosync/corosync.rng b/formats/corosync/corosync.rng new file mode 100644 index 0000000..7b9a585 --- /dev/null +++ b/formats/corosync/corosync.rng @@ -0,0 +1,1123 @@ +<?xml version="1.0"?> +<grammar xmlns="http://relaxng.org/ns/structure/1.0" xmlns:a="http://relaxng.org/ns/compatibility/annotations/1.0" xmlns:a4doc="http://people.redhat.com/jpokorny/ns/a4doc" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + + <start> + <ref name="corosync"/> + </start> + <define name="corosync"> + <element name="corosync"> + <interleave> + <optional> + <ref name="logging"/> + </optional> + <optional> + <ref name="nodelist"/> + </optional> + <optional> + <ref name="resources"/> + </optional> + <optional> + <ref name="quorum"/> + </optional> + <ref name="totem"/> + <optional> + <ref name="uidgid"/> + </optional> + </interleave> + </element> + </define> + + <define name="logging"> + <element name="logging"> + <a:documentation>In this configuration section, one can +adjust logging.</a:documentation> + <group> + <optional> + <attribute name="fileline" a:defaultValue="off"> + <a:documentation>This specifies that file and line should +be printed.</a:documentation> + + <choice> + <value>off</value> + <value>on</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="function_name" a:defaultValue="off"> + <a:documentation>This specifies that the code function name +should be printed.</a:documentation> + + <choice> + <value>off</value> + <value>on</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="timestamp" a:defaultValue="off"> + <a:documentation>This specifies that a timestamp is placed +on all log messages.</a:documentation> + + <choice> + <value>off</value> + <value>on</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="debug" a:defaultValue="off"> + <a:documentation>This specifies whether debug output is +logged for this particular logger. Also can contain value 'trace', +which is highest level of debug informations.</a:documentation> + + <choice> + <value>off</value> + <value>on</value> + <value>trace</value> + </choice> + </attribute> + </optional><optional> + <attribute name="logfile"> + <a:documentation>If the *to_logfile* option is set to +'yes', this option specifies the pathname of the log file.</a:documentation> + + <text/> + </attribute> + </optional><optional> + <attribute name="logfile_priority" a:defaultValue="info"> + <a:documentation>This specifies the logfile level for this particular subsystem. Ignored +if *debug* is 'on'. Note: 'debug' is the same as if *debug* is 'on'.</a:documentation> + + <choice> + <value>alert</value> + <value>crit</value> + <value>debug</value> + <value>emerg</value> + <value>err</value> + <value>info</value> + <value>notice</value> + <value>warning</value> + </choice> + </attribute> + </optional><optional> + <attribute name="syslog_facility" a:defaultValue="daemon"> + <a:documentation>This specifies the syslog facility type +that will be used for any messages sent to syslog.</a:documentation> + + <choice> + <value>daemon</value> + <value>local0</value> + <value>local1</value> + <value>local2</value> + <value>local3</value> + <value>local4</value> + <value>local5</value> + <value>local6</value> + <value>local7</value> + </choice> + </attribute> + </optional><optional> + <attribute name="syslog_priority" a:defaultValue="info"> + <a:documentation>This specifies the syslog level for this +particular subsystem. Ignored if *debug* is 'on'. Note: 'debug' +is the same as *debug* is 'on'.</a:documentation> + + <choice> + <value>alert</value> + <value>crit</value> + <value>debug</value> + <value>emerg</value> + <value>err</value> + <value>info</value> + <value>notice</value> + <value>warning</value> + </choice> + </attribute> + </optional><optional> + <attribute name="tags"> + + <text/> + </attribute> + </optional><optional> + <attribute name="to_logfile" a:defaultValue="no"> + <a:documentation>This specifies whether to use +the respective destination of logging output. + +Please note, if you are using *to_logfile* and want to rotate the file, +use `logrotate(8)` with the option `copytruncate`, e.g. + +---- +/var/log/corosync.log { + missingok + compress + notifempty + daily + rotate 7 + copytruncate +} +----</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional><optional> + <attribute name="to_stderr" a:defaultValue="yes"> + <a:documentation>This specifies whether to use +the respective destination of logging output.</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional><optional> + <attribute name="to_syslog" a:defaultValue="yes"> + <a:documentation>This specifies whether to use +the respective destination of logging output.</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional> + <zeroOrMore> + <ref name="logger_subsys"/> + </zeroOrMore> + </group> + </element> + </define> + <define name="logger_subsys"> + <element name="logger_subsys"> + <group> + <attribute name="subsys"> + <a:documentation>This specifies the subsystem identity +(name) for which logging is specified. This is the name used by +a service in the `log_init` call, e.g., 'CPG'.</a:documentation> + + <text/> + </attribute> + <optional> + <attribute name="debug" a:defaultValue="off"> + <a:documentation>This specifies whether debug output is +logged for this particular logger. Also can contain value 'trace', +which is highest level of debug informations.</a:documentation> + + <choice> + <value>off</value> + <value>on</value> + <value>trace</value> + </choice> + </attribute> + </optional><optional> + <attribute name="logfile"> + <a:documentation>If the *to_logfile* option is set to +'yes', this option specifies the pathname of the log file.</a:documentation> + + <text/> + </attribute> + </optional><optional> + <attribute name="logfile_priority" a:defaultValue="info"> + <a:documentation>This specifies the logfile level for this particular subsystem. Ignored +if *debug* is 'on'. Note: 'debug' is the same as if *debug* is 'on'.</a:documentation> + + <choice> + <value>alert</value> + <value>crit</value> + <value>debug</value> + <value>emerg</value> + <value>err</value> + <value>info</value> + <value>notice</value> + <value>warning</value> + </choice> + </attribute> + </optional><optional> + <attribute name="syslog_facility" a:defaultValue="daemon"> + <a:documentation>This specifies the syslog facility type +that will be used for any messages sent to syslog.</a:documentation> + + <choice> + <value>daemon</value> + <value>local0</value> + <value>local1</value> + <value>local2</value> + <value>local3</value> + <value>local4</value> + <value>local5</value> + <value>local6</value> + <value>local7</value> + </choice> + </attribute> + </optional><optional> + <attribute name="syslog_priority" a:defaultValue="info"> + <a:documentation>This specifies the syslog level for this +particular subsystem. Ignored if *debug* is 'on'. Note: 'debug' +is the same as *debug* is 'on'.</a:documentation> + + <choice> + <value>alert</value> + <value>crit</value> + <value>debug</value> + <value>emerg</value> + <value>err</value> + <value>info</value> + <value>notice</value> + <value>warning</value> + </choice> + </attribute> + </optional><optional> + <attribute name="tags"> + + <text/> + </attribute> + </optional><optional> + <attribute name="to_logfile" a:defaultValue="no"> + <a:documentation>This specifies whether to use +the respective destination of logging output. + +Please note, if you are using *to_logfile* and want to rotate the file, +use `logrotate(8)` with the option `copytruncate`, e.g. + +---- +/var/log/corosync.log { + missingok + compress + notifempty + daily + rotate 7 + copytruncate +} +----</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional><optional> + <attribute name="to_stderr" a:defaultValue="yes"> + <a:documentation>This specifies whether to use +the respective destination of logging output.</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional><optional> + <attribute name="to_syslog" a:defaultValue="yes"> + <a:documentation>This specifies whether to use +the respective destination of logging output.</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional> + </group> + </element> + </define> + <define name="nodelist"> + <element name="nodelist"> + <a:documentation>In this configuration section, one can +adjust nodes in the cluster.</a:documentation> + <zeroOrMore> + <ref name="node"/> + </zeroOrMore> + </element> + </define> + <define name="node"> + <element name="node"> + <group> + <optional> + <attribute name="nodeid"> + <a:documentation>This configuration option is optional when +using IPv4 and required when using IPv6. This is a 32bit value +specifying the node identifier delivered to the cluster membership +service. If this is not specified with IPv4, *nodeid* will be +determined from the 32bit IP address the system to which the system +is bound with ring identifier of 0. The node identifier value of zero +is reserved and should not be used.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="quorum_votes"> + + <data type="unsignedInt"/> + </attribute> + </optional> + <attribute name="ring0_addr"> + <a:documentation>This specifies IP address of one of the nodes for particular ring +as denoted by its number (instead 0, there can be higher numbers).</a:documentation> + + <text/> + </attribute> + <optional> + <attribute name="ring1_addr"/> + </optional> + <optional> + <attribute name="ring2_addr"/> + </optional> + <optional> + <attribute name="ring3_addr"/> + </optional> + <optional> + <attribute name="ring4_addr"/> + </optional> + <optional> + <attribute name="ring5_addr"/> + </optional> + <optional> + <attribute name="ring6_addr"/> + </optional> + <optional> + <attribute name="ring7_addr"/> + </optional> + <optional> + <attribute name="ring8_addr"/> + </optional> + <optional> + <attribute name="ring9_addr"/> + </optional> + </group> + + </element> + </define> + <define name="quorum"> + <element name="quorum"> + <a:documentation>In this configuration section, one can +adjust quorum.</a:documentation> + <group> + <optional> + <attribute name="allow_downscale" a:defaultValue="0"> + <a:documentation>This enables Downscale feature +(see `votequorum(5)`).</a:documentation> + + <choice> + <value>0</value> + <value>1</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="auto_tie_breaker" a:defaultValue="0"> + <a:documentation>This enables Auto Tie Breaker feature +(see `votequorum(5)`).</a:documentation> + + <choice> + <value>0</value> + <value>1</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="expected_votes"> + <a:documentation>This specifies the number of expected votes, overriding the number +implied by the number of *node* items within *nodes*.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="last_man_standing" a:defaultValue="0"> + <a:documentation>This enables Last Man Standing feature +(see `votequorum(5)`).</a:documentation> + + <choice> + <value>0</value> + <value>1</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="last_man_standing_window" a:defaultValue="0"> + <a:documentation>This specifies the tunable for Last Man +Standing feature (see `votequorum(5)`).</a:documentation> + + <data type="nonNegativeInteger"/> + </attribute> + </optional> + <optional> + <attribute name="provider"> + <a:documentation>This specifies the quorum algorithm to use. +As of now, only 'corosync_votequorum' is supported.</a:documentation> + + <value>corosync_votequorum</value> + </attribute> + </optional> + <optional> + <attribute name="two_node" a:defaultValue="0"> + <a:documentation>This enables two node cluster operations +(see `votequorum(5)`).</a:documentation> + + <choice> + <value>0</value> + <value>1</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="votes"> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="wait_for_all" a:defaultValue="0"> + <a:documentation>This enables Wait For All feature +(see `votequorum(5)`).</a:documentation> + + <choice> + <value>0</value> + <value>1</value> + </choice> + </attribute> + </optional> + </group> + </element> + </define> + + <define name="load_15min"> + <element name="load_15min"> + <optional> + <optional> + <attribute name="max"> + + <data type="decimal"/> + </attribute> + </optional><optional> + <attribute name="poll_period"> + + <data type="unsignedInt"/> + </attribute> + </optional><optional> + <attribute name="recovery"> + + <choice> + <value>reboot</value> + <value>shutdown</value> + <value>watchdog</value> + <value>none</value> + </choice> + </attribute> + </optional> + </optional> + </element> + </define> + <define name="memory_used"> + <element name="memory_used"> + <optional> + <optional> + <attribute name="max"> + + <data type="decimal"/> + </attribute> + </optional><optional> + <attribute name="poll_period"> + + <data type="unsignedInt"/> + </attribute> + </optional><optional> + <attribute name="recovery"> + + <choice> + <value>reboot</value> + <value>shutdown</value> + <value>watchdog</value> + <value>none</value> + </choice> + </attribute> + </optional> + </optional> + </element> + </define> + <define name="system"> + <element name="system"> + <interleave> + <optional> + <ref name="load_15min"/> + </optional> + <optional> + <ref name="memory_used"/> + </optional> + </interleave> + </element> + </define> + <define name="resources"> + <element name="resources"> + <optional> + <ref name="system"/> + </optional> + </element> + </define> + <define name="totem"> + <element name="totem"> + <a:documentation>In this configuration section, one can +adjust totem protocol.</a:documentation> + <group> + <optional> + <attribute name="clear_node_high_bit" a:defaultValue="no" a4doc:discretion-hint="The clusters behavior is undefined if this option is enabled on only a subset of the cluster (for example during a rolling upgrade)."> + <a:documentation>This configuration option is only relevant +when no *nodeid* option within *nodelist* section is specified. Some +corosync clients require a signed 32bit nodeid that is greater than +zero however, by default, corosync uses all 32 bits of the IPv4 address +space when generating a nodeid. +Set this option to 'yes' to force the high bit to be zero and therefor +ensure the nodeid is a positive signed 32bit integer.</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="cluster_name"> + <a:documentation>This specifies the name of cluster and it's +used for automatic generating of multicast address.</a:documentation> + + <text/> + </attribute> + </optional> + <optional> + <attribute name="consensus" a:defaultValue="1200"> + <a:documentation>This timeout specifies in milliseconds how +long to wait for consensus to be achieved before starting a new round +of membership configuration. The minimum value for *consensus* must be +1.2 x *token*. + +This value will be automatically calculated at 1.2 x *token* if +the user doesn't specify a *consensus* value. + +For two node clusters, a *consensus* larger than the *join* timeout but +less than *token* is safe. For three-node or larger clusters, +*consensus* should be larger than *token*. There is an increasing risk +of odd membership changes, which still guarantee virtual synchrony, +as node count grows if *consensus* is less than *token*.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="crypto_cipher" a:defaultValue="aes256"> + <a:documentation>This specifies which cipher should be used +to encrypt all messages.</a:documentation> + + <choice> + <value>3des</value> + <value>aes128</value> + <value>aes192</value> + <value>aes256</value> + <value>none</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="crypto_compat"> + + <choice> + <value>2.0</value> + <value>2.2</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="crypto_hash" a:defaultValue="sha1"> + <a:documentation>This specifies which HMAC authentication +should be used to authenticate all messages.</a:documentation> + + <choice> + <value>none</value> + <value>md5</value> + <value>sha1</value> + <value>sha256</value> + <value>sha384</value> + <value>sha512</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="crypto_type"> + + <choice> + <value>3des</value> + <value>aes128</value> + <value>aes192</value> + <value>aes256</value> + <value>nss</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="downcheck" a:defaultValue="1000"> + <a:documentation>This timeout specifies in milliseconds how +long to wait before checking that a network interface is back up after +it has been downed.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="fail_recv_const" a:defaultValue="2500"> + <a:documentation>This constant specifies how many rotations +of the token without receiving any of the messages when messages should +be received may occur before a new configuration is formed.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="heartbeat_failures_allowed" a:defaultValue="0"> + <a:documentation>Configures the optional HeartBeating +mechanism for faster failure detection. Keep in mind that engaging this +mechanism in lossy networks could cause faulty loss declaration as +the mechanism relies on the network for heartbeating. + +So as a rule of thumb use this mechanism if you require improved +failure in low to medium utilized networks. + +This constant specifies the number of heartbeat failures the system +should tolerate before declaring heartbeat failure, e.g., 3. +Also if this value is not set or is 0, the heartbeat mechanism is +not engaged in the system and token rotation is the method of failure +detection. Zero disables the mechanism.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="hold" a:defaultValue="180" a4doc:danger-hint="It is not recommended to override this value without guidance from the corosync community."> + <a:documentation>This timeout specifies in milliseconds +how long the token should be held by the representative when +the protocol is under low utilization.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="join" a:defaultValue="50"> + <a:documentation>This timeout specifies in milliseconds how +long to wait for join messages in the membership protocol.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="max_messages" a:defaultValue="17"> + <a:documentation>This constant specifies the maximum number +of messages that may be sent by one processor on receipt of the token. +The *max_messages* parameter is limited to 256000 / *netmtu* to prevent +overflow of the kernel transmit buffers.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="max_network_delay" a:defaultValue="50"> + <a:documentation>This constant specifies in milliseconds +the approximate delay that your network takes to transport one packet +from one machine to another. This value is to be set by system engineers +and please don't change it if not sure as this effects the failure +detection mechanism using heartbeat.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="merge" a:defaultValue="200"> + <a:documentation>This timeout specifies in milliseconds how +long to wait before checking for a partition when no multicast traffic +is being sent. If multicast traffic is being sent, the merge detection +happens automatically as a function of the protocol.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="miss_count_const" a:defaultValue="5"> + <a:documentation>This constant defines the maximum number +of times on receipt of a token a message is checked for retransmission +before a retransmission occurs. This parameter is useful to modify for +switches that delay multicast packets compared to unicast packets. +The default setting works well for nearly all modern switches.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="netmtu" a:defaultValue="1500"> + <a:documentation>This specifies the network maximum transmit +unit. To set this value beyond 1500, the regular frame MTU, requires +ethernet devices that support large, or also called jumbo, frames. +If any device in the network doesn't support large frames, the protocol +will not operate properly. The hosts must also have their mtu size set +from 1500 to whatever frame size is specified here. + +Please note that while some NICs or +switches claim large frame support, they support '9000' MTU as +the maximum frame size including the IP header. Setting the *netmtu* +and host MTUs to '9000' will cause totem to use the full 9000 bytes +of the frame. Then Linux will add an 18byte header moving the full +frame size to 9018. As a result some hardware will not operate properly +with this size of data. A *netmtu* of '8982' seems to work for the few +large frame devices that have been tested. Some manufacturers claim +large frame support when in fact they support frame sizes of 4500 bytes. + +When sending multicast traffic, if the network frequently reconfigures, +chances are that some device in the network doesn't support large frames. + +Choose hardware carefully if intending to use large frame support.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="nodeid"> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="rrp_autorecovery_check_timeout" a:defaultValue="1000"> + <a:documentation>This specifies the time in milliseconds +to check if the failed ring can be auto-recovered.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="rrp_mode"> + <a:documentation>This specifies the mode of redundant ring. +Active replication ('active') offers slightly lower latency from +transmit to delivery in faulty network environments but with less +performance. Passive replication ('passive') may nearly double +the speed of the totem protocol if it doesn't become CPU bound. +The remaining option is 'none', in which case only one network +interface will be used to operate the totem protocol. + +If only one *interface* section is specified, 'none' is automatically +chosen. If multiple *interface* sections are specified, only 'active' +or 'passive' may be chosen. + +The maximum number of *interface* sections that is allowed for either +mode ('active' or 'passive') is 2.</a:documentation> + + <choice> + <value>active</value> + <value>none</value> + <value>passive</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="rrp_problem_count_mcast_threshold"> + <a:documentation>This specifies the number of times +a problem is detected with multicast before setting the link faulty for +'passive' *rrp_mode*. This variable is unused in 'active' *rrp_mode*. + +The default is 10 x *rrp_problem_count_threshold*.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="rrp_problem_count_threshold" a:defaultValue="10"> + <a:documentation>This specifies the number of times +a problem is detected with a link before setting the link faulty. +Once a link is set faulty, no more data is transmitted upon it. Also, +the problem counter is no longer decremented when the problem count +timeout expires. + +A problem is detected whenever all tokens from the proceeding +processor have not been received within the *rrp_token_expired_timeout*. +The *rrp_problem_count_threshold* x *rrp_token_expired_timeout* should be +at least 50 milliseconds less than the *token* timeout, or a complete +reconfiguration may occur.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="rrp_problem_count_timeout" a:defaultValue="2000"> + <a:documentation>This specifies the time in milliseconds +to wait before decrementing the problem count by 1 for a particular ring +to ensure a link is not marked faulty for transient network failures.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="rrp_token_expired_timeout" a:defaultValue="47" a4doc:danger-hint="It is not recommended to override this value without guidance from the corosync community."> + <a:documentation>This specifies the time in milliseconds +to increment the problem counter for the redundant ring protocol after +not having received a token from all rings for a particular processor. + +This value will automatically be calculated from the *token* timeout +and *problem_count_threshold* but may be overridden.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="secauth" a:defaultValue="on" a4doc:deprecation-hint="It's recomended to use combination of *crypto_cipher* and *crypto_hash*."> + <a:documentation>This specifies that HMAC/SHA1 authentication should be used +to authenticate all messages. It further specifies that all data +should be encrypted with the nss library and aes256 encryption +algorithm to protect data from eavesdropping. + +Enabling this option adds a encryption header to every message sent +by totem which reduces total throughput. Also encryption and +authentication consume extra CPU cycles in corosync.</a:documentation> + + <choice> + <value>off</value> + <value>on</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="send_join" a:defaultValue="0" a4doc:danger-hint="Seek advice from the corosync mailing list if trying to run larger configurations."> + <a:documentation>This timeout specifies in milliseconds +an upper range between 0 and *send_join* to wait before sending a join +message. For configurations with less than 32 nodes, this parameter +is not necessary. For larger rings, this parameter is necessary +to ensure the NIC is not overflowed with join messages on formation of +a new ring. A reasonable value for large rings (128 nodes) would be +__80__msec. Other timer values must also change if this value +is changed.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="seqno_unchanged_const" a:defaultValue="30"> + <a:documentation>This constant specifies how many rotations +of the token without any multicast traffic should occur before the hold +timer is started.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="threads"> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="token" a:defaultValue="1000"> + <a:documentation>This timeout specifies a period in +milliseconds until a token loss is declared after not receiving +a token. This is the time spent detecting a failure of a processor +in the current configuration. Reforming a new configuration takes +about 50 milliseconds in addition to this timeout.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="token_retransmit" a:defaultValue="238" a4doc:danger-hint="It is not recommended to override this value without guidance from the corosync community."> + <a:documentation>This timeout specifies a period in +milliseconds without receiving a token after which the token is +retransmitted. This will be automatically calculated if *token* is +modified.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="token_retransmits_before_loss_const" a:defaultValue="4"> + <a:documentation>This value identifies how many token +retransmits should be attempted before forming a new configuration. +If this value is set, retransmit and hold will be automatically +calculated from *retransmits_before_loss* and *token*.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <optional> + <attribute name="transport" a:defaultValue="udp"> + <a:documentation>This option controls the transport +mechanism used. If the interface to which corosync is binding is +an RDMA interface such as RoCEE or Infiniband, the 'iba' parameter +may be specified. To avoid the use of multicast entirely, a unicast +transport parameter 'udpu' can be specified. This requires specifying +the list of members that could potentially make up the membership +in *nodelist* section before deployment.</a:documentation> + + <choice> + <value>iba</value> + <value>udp</value> + <value>udpu</value> + </choice> + </attribute> + </optional> + <attribute name="version"> + <a:documentation>This specifies the version of +the configuration file. Currently the only valid value for this +option is '2'.</a:documentation> + + <data type="boolean"/> + </attribute> + <optional> + <attribute name="vsftype" a:defaultValue="ykd"> + <a:documentation>This option controls the virtual +synchrony filter type used to identify a primary component. +The preferred choice is YKD dynamic linear voting ('ykd'), however, for +clusters larger than 32 nodes YKD consumes a lot of memory. For large +scale clusters that are created by changing the MAX_PROCESSORS_COUNT +#define in the C code totem.h file, the virtual synchrony filter 'none' +is recommended but then AMF and DLCK services (which are currently +experimental) are not safe for use.</a:documentation> + + <choice> + <value>none</value> + <value>ykd</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="window_size" a:defaultValue="50"> + <a:documentation>This constant specifies the maximum number +of messages that may be sent on one token rotation. If all processors +perform equally well, this value could be large ('300'), which would +introduce higher latency from origination to delivery for very large +rings. To reduce latency in large rings (16+), the default is a safe +compromise. If 1 or more slow processor(s) are present among fast +processors, *window_size* should be no larger than 256000 / *netmtu* +to avoid overflow of the kernel receive buffers. The user is notified +of this by the display of a retransmit list in the notification logs. +There is no loss of data, but performance is reduced when these errors +occur.</a:documentation> + + <data type="unsignedInt"/> + </attribute> + </optional> + <zeroOrMore> + <ref name="interface"/> + </zeroOrMore> + </group> + </element> + </define> + <define name="interface"> + <element name="interface"> + <group> + <optional> + <attribute name="bindnetaddr"> + <a:documentation>This specifies the network address +the corosync executive should bind to. +*bindnetaddr* should be an IP address configured on the system, or +a network address. + +For example, if the local interface is `192.168.5.92` with netmask +`255.255.255.0`, you should set *bindnetaddr* to `192.168.5.92` or +`192.168.5.0`. If the local interface is `192.168.5.92` with netmask +`255.255.255.192`, set *bindnetaddr* to `192.168.5.92` or `192.168.5.64`, +and so forth. + +This may also be an IPv6 address, in which case IPv6 networking will be +used. In this case, the exact address must be specified and there is no +automatic selection of the network interface within a specific subnet +as with IPv4. + +If IPv6 networking is used, *nodeid* options within *nodelist* section +must be specified.</a:documentation> + + <text/> + </attribute> + </optional> + <optional> + <attribute name="broadcast" a:defaultValue="no"> + <a:documentation>If this is set to 'yes', the broadcast +address will be used for communication. If this option is set, +*mcastaddr* should not be set.</a:documentation> + + <choice> + <value>no</value> + <value>yes</value> + </choice> + </attribute> + </optional> + <optional> + <attribute name="mcastaddr"> + <a:documentation>This is the multicast address used +by corosync executive. The default should work for most networks, but +the network administrator should be queried about a multicast address +to use. Avoid `224.x.x.x` because this is a "config" multicast address. + +This may also be an IPv6 multicast address, in which case IPv6 networking +will be used. If IPv6 networking is used, *nodeid* options within +*nodelist* section must be specified. + +It's not needed to use this option if *cluster_name* option in +*totem* section is used. If both options are used, *mcastaddr* has +higher priority.</a:documentation> + + <text/> + </attribute> + </optional> + <optional> + <attribute name="mcastport"> + <a:documentation>This specifies the UDP port number. +It is possible to use the same multicast address on a network with +the corosync services configured for different UDP ports. Please note +corosync uses two UDP ports *mcastport* (for mcast receives) and +*mcastport* - 1 (for mcast sends). If you have multiple clusters +on the same network using the same *mcastaddr*, please configure +the **mcastport**s with a gap.</a:documentation> + + <data type="unsignedShort"/> + </attribute> + </optional> + <optional> + <attribute name="ringnumber"> + <a:documentation>This specifies the ring number for +the interface. When using the redundant ring protocol, each interface +should specify separate ring numbers to uniquely identify to +the membership protocol which interface to use for which redundant ring. +The *ringnumber* must start at '0'.</a:documentation> + + <data type="unsignedByte"/> + </attribute> + </optional> + <optional> + <attribute name="ttl" a:defaultValue="1"> + <a:documentation>This specifies the Time To Live (TTL). +If you run your cluster on a routed network, the default of '1' will +be too small. This option provides a way to increase this up to '255'. +The valid range is '0..255'. Note that this is only valid on multicast +transport types.</a:documentation> + + <data type="unsignedByte"/> + </attribute> + </optional> + </group> + </element> + </define> + <define name="uidgid"> + <element name="uidgid"> + <group> + <optional> + <attribute name="uid"> + + <text/> + </attribute> + </optional> + <optional> + <attribute name="gid"> + + <text/> + </attribute> + </optional> + </group> + </element> + </define> + +</grammar> diff --git a/formats/pcs.py b/formats/pcs.py new file mode 100644 index 0000000..77b92fd --- /dev/null +++ b/formats/pcs.py @@ -0,0 +1,17 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Pacemaker configuration system (pcs) format""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +from ..format import XML + + +class pcs(XML): + """Cman-based cluster stack configuration (cluster.conf) + + Also known as Pacemaker Configuration System (pcs). + """ + # XML + root = 'cluster' diff --git a/formats/simpleconfig.py b/formats/simpleconfig.py new file mode 100644 index 0000000..8de4060 --- /dev/null +++ b/formats/simpleconfig.py @@ -0,0 +1,29 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Structured configuration formats such as corosync.conf""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +from ..format import Format, producing + + +class simpleconfig(Format): + """"Structured configuration formats such as corosync.conf""" + # yacc-based parser in fence-virt + native_protocol = 'struct' + + @producing('bytestring') + def get_bytestring(self, protocol): + ret = super(Format, self).get_bytestring(self) + if ret is not None: + return ret + + # fallback + # XXX TODO self('struct') + raise NotImplementedError + + @producing('struct', protect=True) + def get_struct(self, protocol): + #return etree.fromstring(self('bytestring')).getroottree() + pass @@ -0,0 +1,26 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Machinery entry point""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import sys +from .format_manager import FormatManager +from .filter_manager import FilterManager +from .command_manager import CommandManager +from .utils import EC +from . import metadata + + +def main(argv): + ec = EC.SUCCESS + if len(argv) and argv[0] in ('-v', '--version'): + print '\n'.join(metadata) + else: + cm = CommandManager(FilterManager(FormatManager())) + ec = cm(argv) + return ec + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/misc/index.html b/misc/index.html new file mode 100644 index 0000000..9dd1f17 --- /dev/null +++ b/misc/index.html @@ -0,0 +1,22 @@ +<!DOCTYPE html PUBLIC "-//XML-DEV//DTD XHTML RDDL 1.0//EN" + "http://www.w3.org/2001/rddl/rddl-xhtml.dtd" > +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:rddl="http://www.rddl.org/" xml:lang="en"> +<head> + <title>clufter Namespace</title> +</head> +<body> + <h1>clufter Namespace</h1> + <div> + <p>Resources will be published once they exist.</p> + </div> + <div> + <p> + contact: <code>jpokorny at redhat dot com</code> + <br/> + last change: <code>2013-01-08</code> + </p> + </div> +</body> +</html> diff --git a/misc/ns-a4doc.html b/misc/ns-a4doc.html new file mode 100644 index 0000000..e302ff7 --- /dev/null +++ b/misc/ns-a4doc.html @@ -0,0 +1,60 @@ +<!DOCTYPE html PUBLIC "-//XML-DEV//DTD XHTML RDDL 1.0//EN" + "http://www.w3.org/2001/rddl/rddl-xhtml.dtd" > +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:rddl="http://www.rddl.org/" xml:lang="en"> +<head> + <title>a4doc Namespace</title> +</head> +<body> + <h1><code>a4doc</code> Namespace</h1> + <p>Note: resources are being published as they (continually) emerge.</p> + <p> + <code>a4doc</code> namespace serves a purpose of annotating specific + <em>notable semantic properties accompanied with (optional) + textual details or just standalone hints</em> connected with + annotated entity. + </p> + <h2>Common attributes</h2> + <div> + <dl> + <dt><code>a4doc:hint</code></dt> + <dd> + <p> + standalone side notes for the item + </p> + </dd> + <dt><code>a4doc:deprecation-hint</code></dt> + <dd> + <p> + details about item deprecation (when used, even if with an empty + string, marks the current item as <em>deprecated</em>) + </p> + </dd> + <dt><code>a4doc:danger-hint</code></dt> + <dd> + <p> + explaining what is dangerous about touching this item (when used, + even if with an empty string, marks the current item as + <em>dangerous</em>/experimental) + </p> + </dd> + <dt><code>a4doc:discretion-hint</code></dt> + <dd> + <p> + explaining why extra caution about touching this item (when used, + even if with an empty string, marks the current item as + <em>requiring discretion</em>) + </p> + </dd> + </dl> + </div> + <div> + <p> + contact: <code>jpokorny at redhat dot com</code> + <br/> + last change: <code>2013-02-19</code> + </p> + </div> +</body> +</html> diff --git a/misc/ns-clufter.html b/misc/ns-clufter.html new file mode 100644 index 0000000..92c3487 --- /dev/null +++ b/misc/ns-clufter.html @@ -0,0 +1,74 @@ +<!DOCTYPE html PUBLIC "-//XML-DEV//DTD XHTML RDDL 1.0//EN" + "http://www.w3.org/2001/rddl/rddl-xhtml.dtd" > +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:rddl="http://www.rddl.org/" xml:lang="en"> +<head> + <title>clufter Namespace</title> +</head> +<body> + <h1><code>clufter</code> Namespace</h1> + <p>Note: resources are being published as they (continually) emerge.</p> + <h2>Common tags</h2> + <div> + <dl> + <dt><code>clufter:snippet</code></dt> + <dd> + <dl> + <dt><em>content</em></dt> + <dd> + either application-specific, <code>clufter</code> or top-level/special XSL tags + or their combination + </dd> + </dl> + <p> + used in the internal <code>clufter</code> logic, not intended + for direct usage + </p> + </dd> + <dt><code>clufter:recursion</code></dt> + <dd> + <dl> + <dt><code>at</code></dt> + <dd> + the target of the recursion + (optional, defaults to <code>*</code> under the hood) + </dd> + <dt><em>content</em></dt> + <dd> + (empty) + </dd> + </dl> + <p> + triggers the recursion (in the XSLT sense) + </p> + </dd> + <dt><code>clufter:message</code></dt> + <dd> + <dl> + <dt><code>prefix</code></dt> + <dd> + additional prefix to enable fine-grained filtering of the messages later on + (optional, defaults to <code>''</code> under the hood) + </dd> + <dt><em>content</em></dt> + <dd> + the actual message + </dd> + </dl> + <p> + custom wrapper for <code>xsl:message</code> so that correct messages + can be picked up later on (from presumably chaotic soup of messages) + </p> + </dd> + </dl> + </div> + <div> + <p> + contact: <code>jpokorny at redhat dot com</code> + <br/> + last change: <code>2013-01-10</code> + </p> + </div> +</body> +</html> diff --git a/plugin_registry.py b/plugin_registry.py new file mode 100644 index 0000000..05369a5 --- /dev/null +++ b/plugin_registry.py @@ -0,0 +1,194 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Easy (at least for usage) plugin framework""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import imp +import logging +from os import extsep, walk +from os.path import abspath, dirname, join, splitext +from contextlib import contextmanager +from sys import modules + +from .utils import classproperty, hybridproperty + +log = logging.getLogger(__name__) + + +class PluginRegistry(type): + """Core of simple plugin framework + + This ``superclass'' serves two roles: + + (1) metaclass for plugins (and its class hierarchy) + - only a tiny wrapper on top of `type` + + (2) base class for particular registries + + Both roles are combined due to a very tight connection. + + Inspired by http://eli.thegreenplace.net/2012/08/07/ (Fundamental...). + """ + _registries = set() # dynamic tracking of specific registries + + # + # these are relevant for use case (1) + # + + # non-API + + def __new__(registry, name, bases, attrs): + if '__metaclass__' not in attrs and (registry.use_local + or not registry.__module__.startswith(attrs['__module__'])): + # alleged end-use plugin + ret = registry.probe(name, bases, attrs) + else: + if registry not in PluginRegistry._registries: + log.debug("Registering `{0}' as registry" + .format(registry.registry)) + # specific registry not yet tracked + # (e.g., specific plugin was imported natively) + registry.setup() + PluginRegistry._registries.add(registry) + if registry.namespace not in modules: + # XXX hack to keep going in the test suite + __import__(registry.namespace) + + ret = super(PluginRegistry, registry).__new__(registry, name, + bases, attrs) + + return ret + + # + # these are relevant for both (1) + (2) + # + + @classmethod + def probe(registry, name, bases, attrs): + """Meta-magic to register plugin""" + try: + ret = registry._plugins[name] + log.info("Probe `{0}' plugin under `{1}' registry: already tracked" + .format(name, registry.registry)) + except KeyError: + log.debug("Probe `{0}' plugin under `{1}' registry: yet untracked" + .format(name, registry.registry)) + ret = super(PluginRegistry, registry).__new__(registry, name, + bases, attrs) + registry._plugins[name] = ret + finally: + if registry._path_context is not None: + registry._path_mapping[registry._path_context].add(name) + + return ret + + @hybridproperty + def name(this): + """Nicer access to __name__""" + return this.__name__ + + @classproperty + def registry(registry): + """Registry identifier""" + return registry.__name__ + + @classproperty + def namespace(registry): + """Absolute namespace possessed by the particular plugin registry + + For a plugin, this denotes to which registry/namespace it belongs. + """ + try: + return registry._namespace + except AttributeError: + registry._namespace = '.'.join((__package__, registry.__name__)) + return registry._namespace + + # + # these are relevant for use case (2) + # + + # non-API + + @classmethod + @contextmanager + def _path(registry, path): + """Temporary path context setup enabling safe sideways use""" + assert registry._path_context is None + registry._path_context = path + yield path, registry._path_mapping.setdefault(path, set()) + assert registry._path_context is not None + registry._path_context = None + + @classmethod + def _context(registry, paths): + """Iterate through the paths yielding context along + + Context is a pair `(path, list_of_per_path_tracked_plugins_so_far)`. + """ + if not isinstance(paths, (list, tuple)): + if paths is None: + return # explictly asked not to use even implicit path + paths = (paths, ) + + # inject implicit one + implicit = join(dirname(abspath(__file__)), registry.__name__) + paths = (lambda *x: x)(implicit, *paths) + + for path in paths: + with registry._path(path) as context: + yield context + + # API + + use_local = False + + @classmethod + def setup(registry, reset=False): + """Implicit setup upon first registry involvement or external reset""" + attrs = ('_path_context', None), ('_path_mapping', {}), ('_plugins', {}) + if reset: + map(lambda (a, d): setattr(registry, a, d), attrs) + else: + map(lambda (a, d): setattr(registry, a, getattr(registry, a, d)), + attrs) + if reset: + PluginRegistry._registries.discard(registry) + + @classmethod + def discover(registry, paths): + """Find relevant plugins available at the specified path(s) + + Returns `{plugin_name: plugin_cls}` mapping of plugins found. + """ + ret = {} + for path, path_plugins in registry._context(paths): + # skip if path already discovered (considered final) + if not len(path_plugins): + # visit *.py files within (and under) the path and probe them + for root, dirs, files in walk(path): + for f in files: + name, ext = splitext(f) + if not name.startswith('_') and ext == extsep + 'py': + mfile, mpath, mdesc = imp.find_module(name, [root]) + if not mfile: + log.debug("Omitting `{0}' at `{1}'" + .format(name, root)) + continue + mname = '.'.join((registry.namespace, name)) + try: + imp.load_module(mname, mfile, mpath, mdesc) + finally: + mfile.close() + path_plugins = registry._path_mapping[path] + + # filled as a side-effect of meta-magic, see `__new__` + ret.update({n: registry._plugins[n] for n in path_plugins}) + if registry.use_local: + # add "built-in" ones + ret.update({n: p for n, p in registry._plugins.iteritems() + if p.__module__ == registry.__module__}) + + return ret diff --git a/runtests.sh b/runtests.sh new file mode 100755 index 0000000..14ccfd6 --- /dev/null +++ b/runtests.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +# Specific tests can be specified as per [1], e.g.: +# tests.filter.XMLTraverse.testXSLTTemplate +# If nothing specified, auto-discovery is in place -> all the tests. +# --- +# [1] http://docs.python.org/2/library/unittest.html#command-line-interface + +stop=$(printf "\033\[0m") +blue=$(printf "\033[01;34m") red=$(printf "\033[01;31m") green=$(printf "\033[01;32m") +cyan=$(printf "\033[01;36m") magenta=$(printf "\033[01;35m") +COLORIZE='|& sed \ + -e "s/\(^\|[^A-Za-z]\)\(OK\)/\1${blue}\2${stop}/" \ + -e "s/\(^\|[^A-Za-z]\)\(FAILED.*\)/\1${red}\2${stop}/" \ + -e "s/\(^\|[^A-Za-z]\)\(ok\)/\1${green}\2${stop}/" \ + -e "s/^\(\(FAIL\|ERROR\):.*\)/${magenta}\1${stop}/" \ + -e "s/\(^\|[^A-Za-z]\)\(FAIL\)/\1${red}\2${stop}/" \ + -e "s/\(^\|[^A-Za-z]\)\(ERROR\)/\1${red}\2${stop}/" \ + -e "s/^\(Ran [1-9][0-9]*.*\)/${cyan}\1${stop}/"' + +CMD="python -m unittest" +DEBUG="|& sed 's|\(.*\)DEBUG:.*|\1|' | grep -v '^[ ]*$'" +VERBOSE=1 +ACC= +while [ $# -gt 0 ]; do + case "$1" in + "-d") + DEBUG= + ;; + "-q") + VERBOSE=0 + ;; + *) + ACC+=" $1" + ;; + esac + shift +done +if [ -z ${ACC} ]; then + ACC=" discover -s tests -p '*.py'" + if [ "$VERBOSE" -eq 1 ]; then + ACC+=" --verbose" + fi +fi +if [ ! -t 0 ]; then + COLORIZE= +fi +eval "$CMD $ACC $DEBUG $COLORIZE" diff --git a/tests/XMLFormat-walk/cluster/__init__.py b/tests/XMLFormat-walk/cluster/__init__.py new file mode 100644 index 0000000..03a6057 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/__init__.py @@ -0,0 +1,6 @@ +full = "cluster-full" +direct_xslt_test = '''\ + <test version="{@config_version}"> + <clufter:recursion at="clusternodes"/> + </test> +''' diff --git a/tests/XMLFormat-walk/cluster/clusternodes/__init__.py b/tests/XMLFormat-walk/cluster/clusternodes/__init__.py new file mode 100644 index 0000000..90a1601 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/clusternodes/__init__.py @@ -0,0 +1,2 @@ +full = "clusternodes-full" +traverse_test = "clusternodes-traverse_test" diff --git a/tests/XMLFormat-walk/cluster/clusternodes/clusternode/__init__.py b/tests/XMLFormat-walk/cluster/clusternodes/clusternode/__init__.py new file mode 100644 index 0000000..c61e735 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/clusternodes/clusternode/__init__.py @@ -0,0 +1,5 @@ +full = "clusternode-full" +traverse_test = "clusternode-traverse_test" +direct_xslt_test = '''\ + <node><xsl:value-of select="@name" /></node> +''' diff --git a/tests/XMLFormat-walk/cluster/cman/__init__.py b/tests/XMLFormat-walk/cluster/cman/__init__.py new file mode 100644 index 0000000..d5cc994 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/cman/__init__.py @@ -0,0 +1 @@ +full = "cman-full" diff --git a/tests/XMLFormat-walk/cluster/dlm/__init__.py b/tests/XMLFormat-walk/cluster/dlm/__init__.py new file mode 100644 index 0000000..fb9f351 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/dlm/__init__.py @@ -0,0 +1 @@ +full = "dlm-full" diff --git a/tests/XMLFormat-walk/cluster/quorumd/__init__.py b/tests/XMLFormat-walk/cluster/quorumd/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/quorumd/__init__.py diff --git a/tests/XMLFormat-walk/cluster/quorumd/heuristic.py b/tests/XMLFormat-walk/cluster/quorumd/heuristic.py new file mode 100644 index 0000000..c64c158 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/quorumd/heuristic.py @@ -0,0 +1 @@ +sparse = "heuristic-sparse" diff --git a/tests/XMLFormat-walk/cluster/rm/__init__.py b/tests/XMLFormat-walk/cluster/rm/__init__.py new file mode 100644 index 0000000..de7b2f9 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/rm/__init__.py @@ -0,0 +1 @@ +full = "rm-full" diff --git a/tests/XMLFormat-walk/cluster/rm/failoverdomains/__init__.py b/tests/XMLFormat-walk/cluster/rm/failoverdomains/__init__.py new file mode 100644 index 0000000..e06d06c --- /dev/null +++ b/tests/XMLFormat-walk/cluster/rm/failoverdomains/__init__.py @@ -0,0 +1 @@ +full = "failoverdomains-full" diff --git a/tests/XMLFormat-walk/cluster/rm/failoverdomains/failoverdomain.py b/tests/XMLFormat-walk/cluster/rm/failoverdomains/failoverdomain.py new file mode 100644 index 0000000..418be05 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/rm/failoverdomains/failoverdomain.py @@ -0,0 +1,2 @@ +full = "failoverdomain-full" +sparse = "failoverdomain-sparse" diff --git a/tests/XMLFormat-walk/cluster/rm/service/__init__.py b/tests/XMLFormat-walk/cluster/rm/service/__init__.py new file mode 100644 index 0000000..97f1689 --- /dev/null +++ b/tests/XMLFormat-walk/cluster/rm/service/__init__.py @@ -0,0 +1 @@ +full = "service-full" diff --git a/tests/_bootstrap.py b/tests/_bootstrap.py new file mode 100644 index 0000000..5a4e3b2 --- /dev/null +++ b/tests/_bootstrap.py @@ -0,0 +1,13 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Bootstrap the environment for testing""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import logging +logging.basicConfig(level=logging.DEBUG) + +from sys import path +from os.path import dirname, abspath +path.insert(0, reduce(lambda x, y: dirname(x), xrange(3), abspath(__file__))) diff --git a/tests/_common.py b/tests/_common.py new file mode 100644 index 0000000..b866c97 --- /dev/null +++ b/tests/_common.py @@ -0,0 +1,22 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Testing filter manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import unittest + +import _bootstrap # known W402, required + +from clufter.format_manager import FormatManager +from clufter.filter_manager import FilterManager + + +class CommonFilterTestCase(unittest.TestCase): + def setUp(self): + self.flt_mgr = FilterManager(FormatManager()) + + #def tearDown(self): + # self.flt_mgr.registry.setup(True) # start from scratch + # self.flt_mgr = None diff --git a/tests/ccs2coroxml.py b/tests/ccs2coroxml.py new file mode 100644 index 0000000..a307eeb --- /dev/null +++ b/tests/ccs2coroxml.py @@ -0,0 +1,21 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Testing filter manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +from os.path import dirname, join + +from _common import CommonFilterTestCase + + +class Main(CommonFilterTestCase): + def test01(self): + # using ./filled.conf + testfile = join(dirname(__file__), 'filled.conf') + out_obj = self.flt_mgr('ccs2ccsflat', ('file', testfile)) + out_obj = self.flt_mgr('ccs2coroxml', ('etree', out_obj('etree'))) + #out_obj = self.flt_mgr('ccs_obfuscate_credentials', ('etree', out_obj('etree'))) + #out_obj = self.flt_mgr('ccs_obfuscate_identifiers', ('etree', out_obj('etree'))) + print out_obj('bytestring') diff --git a/tests/empty.conf b/tests/empty.conf new file mode 100644 index 0000000..9c76670 --- /dev/null +++ b/tests/empty.conf @@ -0,0 +1,14 @@ +<?xml version="1.0"?> +<cluster name="test" config_version="1"> + + <clusternodes> + </clusternodes> + + <fencedevices> + </fencedevices> + + <rm> + <failoverdomains/> + <resources/> + </rm> +</cluster> diff --git a/tests/filled.conf b/tests/filled.conf new file mode 100644 index 0000000..6d75402 --- /dev/null +++ b/tests/filled.conf @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<cluster name="test" config_version="1"> + + <clusternodes> + <clusternode nodeid="1" name="ju" /> + <clusternode nodeid="2" name="hele" /> + </clusternodes> + + <cman two_node="1" expected_votes="3"/> + + <totem consensus="200" join="100" token="5000" token_retransmits_before_loss_const="4"> + <interface ttl="3"/> + </totem> + + <logging> + <logging_daemon debug="on" name="corosync" subsys="CONFDB"/> + </logging> + + <fencedevices> + <fencedevice name="foo" passwd="mysecret" testarg="ble"/> + </fencedevices> + + <rm> + <failoverdomains/> + <resources/> + </rm> +</cluster> diff --git a/tests/filled.conf.obfuscation b/tests/filled.conf.obfuscation new file mode 100644 index 0000000..102ca0f --- /dev/null +++ b/tests/filled.conf.obfuscation @@ -0,0 +1,84 @@ +<?xml version="1.0"?> +<cluster config_version="101" name="lhcms"> + <fence_daemon clean_start="0" post_fail_delay="0" post_join_delay="3"/> + <clusternodes> + <clusternode name="graz.got.jeppesensystems.com" nodeid="1" votes="1"> + <fence> + <method name="1"> + <device name="fence_graz" nodename="graz.got.jeppesensystems.com"/> + </method> + </fence> + </clusternode> + <clusternode name="innsbruck.got.jeppesensystems.com" nodeid="2" votes="1"> + <fence> + <method name="1"> + <device name="fence_innsbruck" nodename="innsbruck.got.jeppesensystems.com"/> + </method> + </fence> + </clusternode> + </clusternodes> + <cman expected_votes="1" two_node="1"/> + <fencedevices> + <fencedevice agent="fence_manual" name="fence_graz"/> + <fencedevice agent="fence_manual" name="fence_innsbruck"/> + <fencedevice agent="fence_drac" ipaddr="10.100.2.142" login="fence" name="gotrc-graz" passwd="***"/> + <fencedevice agent="fence_apc" ipaddr="10.100.2.143" login="fence" name="gotrc-innsbruck" passwd="***"/> + </fencedevices> + <rm> + <failoverdomains> + <failoverdomain name="NFS" nofailback="0" ordered="1" restricted="1"> + <failoverdomainnode name="graz.got.jeppesensystems.com" priority="10"/> + <failoverdomainnode name="innsbruck.got.jeppesensystems.com" priority="1"/> + </failoverdomain> + <failoverdomain name="qpidd" nofailback="1" ordered="1" restricted="1"> + <failoverdomainnode name="graz.got.jeppesensystems.com" priority="1"/> + <failoverdomainnode name="innsbruck.got.jeppesensystems.com" priority="1"/> + </failoverdomain> + <failoverdomain name="qpidd2" nofailback="1" ordered="0" restricted="1"> + <failoverdomainnode name="graz.got.jeppesensystems.com" priority="1"/> + <failoverdomainnode name="innsbruck.got.jeppesensystems.com" priority="1"/> + </failoverdomain> + <failoverdomain name="qpidd-3" nofailback="1" ordered="1" restricted="1"> + <failoverdomainnode name="graz.got.jeppesensystems.com" priority="1"/> + <failoverdomainnode name="innsbruck.got.jeppesensystems.com" priority="1"/> + </failoverdomain> + </failoverdomains> + <resources> + <ip address="10.67.128.10" monitor_link="1"/> + <ip address="10.67.128.17" monitor_link="1"/> + <ip address="10.67.128.41" monitor_link="0"/> + <fs device="/dev/vg1/carmlog" force_fsck="0" force_unmount="0" fsid="30815" fstype="ext3" mountpoint="/opt/carmlog-test" name="carmlog" self_fence="0"/> + <nfsexport name="carmlog nfs-export"/> + <nfsclient allow_recover="0" name="nfs client carmlog" target="*"/> + </resources> + <service autostart="1" domain="qpidd-3" exclusive="0" name="qpidd-3" recovery="relocate"> + <ip ref="10.67.128.17"> + <script file="/etc/init.d/qpidd-3" name="qpidd-3 init script"/> + </ip> + </service> + <service autostart="1" domain="NFS" exclusive="0" name="klagenfurt" recovery="relocate"> + <nfsexport name="/opt/Carmen"/> + <ip ref="10.67.128.10"/> + </service> + <service autostart="1" domain="qpidd2" exclusive="0" max_restarts="0" name="qpidd-2" recovery="restart" restart_expire_time="0"> + <ip address="10.67.128.15" monitor_link="1"> + <script file="/etc/init.d/qpidd-2" name="qpidd-2"/> + </ip> + </service> + <service autostart="1" domain="qpidd" exclusive="0" max_restarts="0" name="qpidd-1" recovery="restart" restart_expire_time="0"> + <ip address="10.67.128.11" monitor_link="1"> + <script file="/etc/init.d/qpidd-1" name="qpidd-1"/> + </ip> + </service> + <service autostart="1" domain="NFS" exclusive="0" name="lh-int-log" recovery="relocate"> + <ip ref="10.67.128.41"> + <fs ref="carmlog"> + <nfsexport ref="carmlog nfs-export"> + <nfsclient ref="nfs client carmlog"/> + </nfsexport> + </fs> + </ip> + <!--script file="/etc/init.d/sysmond" name="sysmond"/--> + </service> + </rm> +</cluster> diff --git a/tests/filled.conf.orig b/tests/filled.conf.orig new file mode 100644 index 0000000..6d75402 --- /dev/null +++ b/tests/filled.conf.orig @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<cluster name="test" config_version="1"> + + <clusternodes> + <clusternode nodeid="1" name="ju" /> + <clusternode nodeid="2" name="hele" /> + </clusternodes> + + <cman two_node="1" expected_votes="3"/> + + <totem consensus="200" join="100" token="5000" token_retransmits_before_loss_const="4"> + <interface ttl="3"/> + </totem> + + <logging> + <logging_daemon debug="on" name="corosync" subsys="CONFDB"/> + </logging> + + <fencedevices> + <fencedevice name="foo" passwd="mysecret" testarg="ble"/> + </fencedevices> + + <rm> + <failoverdomains/> + <resources/> + </rm> +</cluster> diff --git a/tests/filter.py b/tests/filter.py new file mode 100644 index 0000000..ba2354b --- /dev/null +++ b/tests/filter.py @@ -0,0 +1,72 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Testing filter manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import unittest +from os.path import dirname, join +#from pprint import pprint + +from lxml import etree + +import _bootstrap # known W402, required + +from clufter.formats.ccs import ccs +from clufter.filter import XMLFilter + +WALK_DIR = join(dirname(__file__), 'XMLFormat-walk') + +RESULT_TRAVERSE = \ + ('clusternodes-traverse_test', 'element: clusternodes', [ + ('clusternode-traverse_test', 'element: clusternode', [ + ]), + ('clusternode-traverse_test', 'element: clusternode', [ + ]) + ]) + +RESULT_DIRECT_XSLT = \ + '<test version="1"><node>ju</node><node>hele</node></test>' + + +def fnc(symbol, elem, children): + return symbol, "element: " + elem.tag, children.values() + + +class XMLTraverse(unittest.TestCase): + def testDirectXSLT(self): + flt = XMLFilter(ccs, ccs) + in_obj = ccs('file', join(dirname(__file__), 'filled.conf')) + r = flt.proceed_xslt(in_obj, symbol='direct_xslt_test', + root_dir=join(dirname(__file__), 'XMLFormat-walk')) + if isinstance(r, list): + ret = [etree.tostring(i) for i in r] + else: + ret = etree.tostring(r) + #print ret # --> expected + self.assertTrue(ret == RESULT_DIRECT_XSLT) + + def testXSLTTemplate(self): + flt = XMLFilter(ccs, ccs) + in_obj = ccs('file', join(dirname(__file__), 'filled.conf')) + r = flt.get_template(in_obj, symbol='direct_xslt_test', + root_dir=join(dirname(__file__), 'XMLFormat-walk')) + #if isinstance(r, list): + # ret = [etree.tostring(i) for i in r] + #else: + # ret = etree.tostring(r) + #print "|", ret, "|" # --> expected + + assert not isinstance(r, list) + et = in_obj('etree') + #print ">>>", etree.tostring(r) + #print ">>>", etree.tostring(r) + modified = et.xslt(r) + ret = etree.tostring(modified) + #print "<<<", ret + self.assertTrue(ret == RESULT_DIRECT_XSLT) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/filter_manager.py b/tests/filter_manager.py new file mode 100644 index 0000000..8b070dc --- /dev/null +++ b/tests/filter_manager.py @@ -0,0 +1,45 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Testing filter manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import unittest +from os.path import dirname, join + +import _bootstrap # known W402, required + +from clufter.format_manager import FormatManager +from clufter.filter_manager import FilterManager +from clufter.filters.ccs2ccsflat import ccs2ccsflat +from clufter.filters.ccsflat2pcs import ccsflat2pcs + + +class FilterManagerTestCase(unittest.TestCase): + def setUp(self): + self.flt_mgr = FilterManager(FormatManager()) + + def tearDown(self): + self.flt_mgr.registry.setup(True) # start from scratch + self.flt_mgr = None + + +class Default(FilterManagerTestCase): + def test_default(self): + filters = self.flt_mgr.filters + #print filters + for cls in ccs2ccsflat, ccsflat2pcs: + self.assertTrue(cls.name in filters) + self.assertEqual(cls, type(filters[cls.name])) + + def test_run_ccs2ccsflat(self): + # using ./empty.conf + testfile = join(dirname(__file__), 'empty.conf') + self.assertTrue('ccs2ccsflat' in self.flt_mgr.filters) + out_obj = self.flt_mgr('ccs2ccsflat', ('file', testfile)) + # XXX print out_obj('bytestring') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/format.py b/tests/format.py new file mode 100644 index 0000000..ea9a75d --- /dev/null +++ b/tests/format.py @@ -0,0 +1,60 @@ +# -*- coding: UTF-8 -*- +# Copyright 2013 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Testing filter manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import unittest +from os.path import dirname, join +#from pprint import pprint + +import _bootstrap # known W402, required + +from clufter.formats.ccs import ccs + +WALK_DIR = join(dirname(__file__), 'XMLFormat-walk') + +RESULT_WALK_FULL = { + 'cluster': ('cluster-full', { + 'clusternodes': ('clusternodes-full', { + 'clusternode': ('clusternode-full', { + }) + }), + 'cman': ('cman-full', { + }), + 'dlm': ('dlm-full', { + }), + 'rm': ('rm-full', { + 'failoverdomains': ('failoverdomains-full', { + 'failoverdomain': ('failoverdomain-full', { + }) + }), + 'service': ('service-full', { + }) + }) + }) +} + +RESULT_WALK_SPARSE = { + 'failoverdomain': ('failoverdomain-sparse', { + }), + 'heuristic': ('heuristic-sparse', { + }) +} + + +class XMLFormatWalkTestCase(unittest.TestCase): + def testWalkFull(self): + r = ccs.walk_schema(WALK_DIR, 'full') + #pprint(r, width=8) # --> expected + self.assertTrue(r == RESULT_WALK_FULL) + + def testWalkSparse(self): + r = ccs.walk_schema(WALK_DIR, 'sparse') + #pprint(r, width=8) # --> expected + self.assertTrue(r == RESULT_WALK_SPARSE) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/format_manager.py b/tests/format_manager.py new file mode 100644 index 0000000..3db5bf6 --- /dev/null +++ b/tests/format_manager.py @@ -0,0 +1,55 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) +"""Testing format manager""" +__author__ = "Jan Pokorný <jpokorny at redhat dot com>" + +import unittest + +import _bootstrap # known W402, required + +from clufter.format_manager import FormatManager +from clufter.formats.ccs import ccs +from clufter.formats.ccs import ccsflat +from clufter.formats.pcs import pcs + + +class FormatManagerTestCase(unittest.TestCase): + def setUp(self): + self.fmt_mgr = FormatManager() + + def tearDown(self): + self.fmt_mgr.registry.setup(True) # start from scratch + self.fmt_mgr = None + + +class Default(FormatManagerTestCase): + def test_default(self): + formats = self.fmt_mgr.formats + #print formats + for cls in ccs, ccsflat, pcs: + self.assertTrue(cls.__name__ in formats) + # the first was needed in the past, but now the more restrictive + # one is ok + self.assertEqual(type(cls), type(formats[cls.__name__])) + self.assertEqual(cls, formats[cls.__name__]) + + +class Injection(FormatManagerTestCase): + formats = {'frobniccs': ccs} + + def setUp(self): + self.fmt_mgr = FormatManager(paths=None, formats=self.formats) + + def test_injection(self): + formats = self.fmt_mgr.formats + #print formats + self.assertTrue(len(formats) == len(self.formats)) + for fmt_id, fmt_cls in self.formats.iteritems(): + self.assertTrue(fmt_id in formats) + self.assertEqual(fmt_cls, formats[fmt_id]) + + +if __name__ == '__main__': + unittest.main() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..57f4d89 --- /dev/null +++ b/utils.py @@ -0,0 +1,65 @@ +# -*- coding: UTF-8 -*- +# Copyright 2012 Red Hat, Inc. +# Part of clufter project +# Licensed under GPLv2 (a copy included | http://gnu.org/licenses/gpl-2.0.txt) + +import os + + +# name the exitcodes +ecodes = 'SUCCESS', 'FAILURE' +EC = type('EC', (), {n: v for v, n in enumerate('EXIT_' + i for i in ecodes)}) + + +head_tail = lambda x=None, *y: (x, x if x is None else y) +filtervars = lambda src,which: {x: src[x] for x in which if x in src} +filtervarsdef = lambda src,which: {x: src[x] for x in which if src.get(x, None)} +filtervarspop = lambda src,which: {x: src.pop(x) for x in which if x in src} + + +def which(name, *where): + where = tuple(os.path.abspath(i) for i in where) + if 'PATH' in os.environ: + path = tuple(i for i in os.environ['PATH'].split(os.pathsep) + if len(i.strip())) + else: + path = () + for p in where + path: + check = os.path.join(p, name) + if os.path.exists(check): + return check + else: + return None + + +class ClufterError(Exception): + def __init__(self, ctx_self, msg, *args): + self.ctx_self = ctx_self + self.msg = msg + + def __str__(self): + ret = getattr(self.ctx_self, '__name__', + self.ctx_self.__class__.__name__) + return ret + ': ' + self.msg.format(*self.args) + + +class ClufterPlainError(ClufterError): + def __init__(self, msg, *args): + super(ClufterPlainError, self).__init__(self, None, msg, *args) + + +# Inspired by http://stackoverflow.com/a/1383402 +class classproperty(property): + def __init__(self, fnc): + property.__init__(self, classmethod(fnc)) + + def __get__(self, this, owner): + return self.fget.__get__(None, owner)() + + +class hybridproperty(property): + def __init__(self, fnc): + property.__init__(self, classmethod(fnc)) + + def __get__(self, this, owner): + return self.fget.__get__(None, this if this else owner)() |