# -*- coding: UTF-8 -*- # Copyright 2015 Red Hat, Inc. # Part of clufter project # Licensed under GPLv2+ (a copy included | http://gnu.org/licenses/gpl-2.0.txt) """Utility functions wrt. cluster systems in general""" __author__ = "Jan Pokorný " from logging import getLogger from .utils_func import apply_intercalate log = getLogger(__name__) # # EXECUTABLE KNOWLEDGE ABOUT THE CLUSTER PACKAGES PER SYSTEM/DISTROS # # or a bit of a logic programming paradigm with Python... # # ... first, define some facts (in suitable data structures): # # core hierarchical (sparse!) map: # system -> distro -> release -> package -> version # # The notation should be self-explanatory, perhaps except 'component[extra]' # (borrowed from the very similar construct of setuptools, there can be more # "extra traits" delimited with comma) ... simply 'pacemaker[cman]' means that # such pacemaker is somehow associated with "cman" (more specificially, it is # intended [compilation flags, etc.] to be used with cman), which apparently # doesn't match 'pacemaker[coro]' component specification in the input query. cluster_map = { 'linux': { 'debian': ( ((6, ), { # https://packages.debian.org/squeeze/{corosync,pacemaker} 'corosync': (1, 2), 'pacemaker[coro,hb]': (1, 0, 9), }), ((7, ), { # https://packages.debian.org/wheezy/{corosync,pacemaker} 'corosync': (1, 4), 'pacemaker[coro,hb]': (1, 1, 7), }), ((8, ), { # https://packages.debian.org/jessie/corosync 'corosync': (1, 4, 6), }), ((9, ), { # https://packages.debian.org/stretch/corosync 'corosync': (2, 3, 5), }), ), 'fedora': ( ((13, ), { 'corosync': (1, 3), 'pacemaker[cman]': (1, 1, 4), #--- 'pkg::mysql': 'mysql-server', #--- 'cmd::pkg-install': 'yum install -y {packages}', }), ((14, ), { 'corosync': (1, 4), 'pacemaker[cman]': (1, 1, 6), }), ((17, ), { 'corosync': (2, 3), 'pacemaker[coro]': (1, 1, 7), 'pcs': (0, 9, 1), }), ((18, ), { 'pacemaker[coro]': (1, 1, 8), 'pcs': (0, 9, 27), }), ((19, ), { 'pacemaker[coro]': (1, 1, 9), 'pcs': (0, 9, 34), #--- # https://fedoraproject.org/wiki/Features/ReplaceMySQLwithMariaDB 'pkg::mysql': 'mariadb-server', }), ((20, ), { 'pcs': (0, 9, 44), }), ((21, ), { 'pacemaker[coro]': (1, 1, 11), 'pcs': (0, 9, 115), }), ((22, ), { 'pcs': (0, 9, 139), #--- # https://fedoraproject.org/wiki/Changes/ReplaceYumWithDNF 'cmd::pkg-install': 'dnf install -y {packages}', }), ((23, ), { 'pacemaker[coro]': (1, 1, 13), }), ), 'redhat': ( ((6, 0), { 'corosync': (1, 2), #--- 'pkg::lvm': 'lvm2', 'pkg::mysql': 'mysql-server', 'pkg::postgresql': 'postgresql-server', 'pkg::virsh': 'libvirt-client', #--- 'cmd::pkg-install': 'yum install -y {packages}', }), ((6, 2), { 'corosync': (1, 4), }), ((6, 4), { 'pcs': (0, 9, 26), }), ((6, 5), { 'pacemaker[cman]': (1, 1, 10), 'pcs': (0, 9, 90), }), ((6, 6), { 'pacemaker[acls,cman]': (1, 1, 11), 'pcs': (0, 9, 123), }), ((6, 7), { 'pacemaker[acls,cman]': (1, 1, 12), 'pcs': (0, 9, 139), }), ((6, 8), { # u9n := utilization 'pcs[u9n]': (0, 9, 148), # XXX guess }), ((7, 0), { 'corosync': (2, 3), 'pacemaker[coro]': (1, 1, 10), 'pcs': (0, 9, 115), #--- 'pkg::mysql': 'mariadb-server', }), ((7, 1), { 'pacemaker[acls,coro]': (1, 1, 12), 'pcs': (0, 9, 137), }), ((7, 2), { 'pacemaker[acls,coro]': (1, 1, 13), 'pcs': (0, 9, 143), }), ), 'ubuntu': ( ((13, 04), { # https://packages.ubuntu.com/raring/{corosync,pacemaker} 'corosync': (1, 4), 'pacemaker[coro,hb]': (1, 1, 7), }), ((13, 10), { # https://packages.ubuntu.com/saucy/{corosync,pacemaker} 'corosync': (2, 3), 'pacemaker[coro,hb]': (1, 1, 10), }), ((15, 04), { # https://packages.ubuntu.com/vivid/{corosync,pacemaker} 'pacemaker[coro,hb]': (1, 1, 12), }), ), }, } # mere aliases of the distributions (packages remain the same), # i.e., downstream rebuilders; # values (and keys when making "alias" association) in this dict should # correspond to `platform.linux_distribution(full_distribution_name=0)` output # and the dict can contain also associated keys obtained as # `platform.linux_distribution(full_distribution_name=1).lower()` # or, on top of previous, what's expected to be entered by the user aliases_dist = { # aliases # XXX platform.linux_distribution(full_distribution_name=0), i.e., # short distro names of Scientific Linux et al. needed 'centos': 'redhat', # full_distribution_name=1 (lower-cased) -> full_distribution_name=0 'red hat enterprise linux server': 'redhat', # convenience/choice of common sense or intuition 'rhel': 'redhat', } # in the queries, one can use following aliases to wildcard versions # of particular packages aliases_releases = { 'corosync': { 'flatiron': '1', 'needle': '2', }, 'debian': { # because of http://bugs.python.org/issue9514 @ 2.6 ? 'squeeze': '6', 'wheezy': '7', 'wheezy/sid': '7.999', # XXX ? 'jessie': '8', }, 'ubuntu': { 'raring': '13.04', # Raring Ringtail 'saucy': '13.10', # Saucy Salamander 'trusty': '14.04', # Trusty Tahr 'utopic': '14.10', # Utopic Unicorn 'vivid': '15.04', # Vivid Vervet 'wily': '15.10', # Wily Werewolf } } # # ...then, define some executable inference rules, starting with their helpers: # def _parse_ver(s): name, ver = (lambda a, b=None: (a, b))(*s.split('=', 1)) if ver: try: ver = aliases_releases[name][ver] except KeyError: pass ver = tuple(map(int, ver.split('.'))) return name, ver def _cmp_ver(v1, v2): if v1 and v2: v1, v2 = list(reversed(v1)), list(reversed(v2)) while v1 and v2: ret = cmp(v1.pop(), v2.pop()) if ret: return ret return 0 def _parse_extra(s): name, extra = (lambda a, b=None: (a, b))(*s.split('[', 1)) extra = extra.strip(']').split(',') if extra and extra.endswith(']') else [] return name, extra def infer_error(smth, branches): raise RuntimeError("This should not be called") def infer_sys(sys, branches=None): log.debug("infer_sys: {0}: {1}".format(sys, branches)) # lists of system-level dicts # -> list of dist-level dicts pertaining the specified system(s) if branches is None: branches = [cluster_map] if sys == "*": return apply_intercalate([b.values() for b in branches]) return [b[sys] for b in branches if sys in b] def infer_dist(dist, branches=None): # list of dist-level dicts # -> list of component-level dicts pertaining the specified dist(s) # incl. dist alias resolution log.debug("infer_dist: {0}: {1}".format(dist, branches)) if branches is None: branches = infer_sys('*') # alt.: branches = [cluster_map.values()] if dist == '*': return apply_intercalate([c[1] for b in branches for c in b.itervalues()]) ret = [] dist, dist_ver = _parse_ver(dist) try: dist = aliases_dist[dist] except KeyError: pass cur_acc = {} for b in branches: for d, d_branches in b.iteritems(): if d == dist: # first time, we (also) traverse whole sequence of per-distro # releases, in-situ de-sparsifying particular packages releases; # to avoid needlessly repeated de-sparsifying, we are using # '__proceeded__' mark to denote already proceeded dicts if '__proceeded__' not in d_branches or dist_ver: for i, (dver, dver_branches) in enumerate(d_branches): if dist_ver: if (_cmp_ver(dist_ver, dver) >= 0 and (i == len(d_branches) - 1 or _cmp_ver(dist_ver, d_branches[i+1][0]) < 0)): ret.append(dver_branches) if '__proceeded__' in dver_branches: break else: dist_ver = None # only desparsify since now if '__proceeded__' in dver_branches: continue # only searching, no hit yet for k, v in dver_branches.iteritems(): kk, k_extra = _parse_extra(k) prev_extra = cur_acc.get(kk, '') cur_acc.pop("{0}{1}".format(kk, prev_extra), None) if k_extra: cur_acc[kk] = "[{0}]".format(','.join(k_extra)) cur_acc[k] = v for k, v in cur_acc.iteritems(): dver_branches[k] = v dver_branches['__proceeded__'] = '[true]' if dist_ver is None and not ret: # alt. above: dist_ver = '' ret.extend(dver_branches for _, dver_branches in d_branches) return ret def infer_comp(comp, branches=None): log.debug("infer_comp: {0}: {1}".format(comp, branches)) # list of component-level dicts # -> list of component-level dicts pertaining the specified comp(s) # incl. component version alias resolution if branches is None: branches = infer_dist('*') # alt.: branches = [cluster_map.values()] if comp == '*': return branches ret = [] comp, comp_ver = _parse_ver(comp) comp, comp_extra = _parse_extra(comp) for b in branches: for c, c_ver in b.iteritems(): c, c_extra = _parse_extra(c) if (c == comp and (not comp_extra or not set(comp_extra).difference(c_extra)) and _cmp_ver(comp_ver, c_ver) == 0): ret.append(b) break return ret rule_error = (0, infer_error) inference_rules = { # type (of clause): (handling priority, handler) 'error': rule_error, 'sys': (1, infer_sys), 'dist': (2, infer_dist), 'comp': (3, infer_comp), } # # ...and application-specific inference engine: # def infer(query, system=None, system_extra=None): """Get/infer answers for system-distro-release-package-version queries Query grammar is (currently = least resistance, generalization needed): QUERY ::= TERM | TERM WS* '+' WS* QUERY TERM ::= TYPE:SPEC | comp:COMPSPEC # comp~component WS ::= [ ] # whitespace TYPE ::= sys | dist # sys~system, dist~distro SPEC ::= NAME | NAME=SUBSPEC COMPSPEC ::= NAME | NAME-EXTRA | NAME=COMPSUBSPEC | NAME-EXTRA=COMPSUBSPEC NAME ::= [a-zA-Z_-]+ # generally anything reasonable SUBSPEC ::= NUMBER '.' NUMBER | '*' # arbitrary version as such NAME-EXTRA ::= NAME '[' EXTRAS ']' EXTRAS ::= NAME | NAME ',' EXTRAS COMPSUBSPEC::= NUMBER '.' NUMBER | NUMBER ',' '*' # arbitrary minor version NUMBER ::= [0-9]+ with notes: - (COMP)SUBSPEC can be defined as a single number, minor version is assumed to be 0 in that case, but it's more like a syntactic sugar thant the full-fledged grammar case - for simplier expressions, several alias resolutions are in place: . distro (almost same set of packages known under different names) . component version codename (usually to denote whole major releases) - for working examples, see the define predicates below """ # XXX trace_back=comp/dist/sys to anchor the result back in the higher sets # XXX only AND is supported via '+', also OR ('/'? even though in some uses # '+' denotes this) together with priority control (parentheses) would # be cool :) prev, ret = None, None q = [p.strip().split(':', 1) for p in query.split('+')] if system: q.append(('sys', system.lower())) if system_extra: q.append(('dist', '='.join(system_extra[:2]).lower())) q.sort(key=lambda t: inference_rules.get(t[0], rule_error)[0]) level = '' for q_type, q_content in q: inference_rule = inference_rules.get(q_type, rule_error)[1] if q_type != level: prev = ret inferred = inference_rule(q_content, prev) log.debug("inferred: {0}".format(inferred)) ret = [i for i in ret if i in inferred] if q_type == level else inferred level = q_type return ret # # and finally, some wrapping predicates: # def cluster_pcs_flatiron(*sys_id): """Whether particular `sys_id` corresponds to pcs-flatiron cluster system""" return bool(infer("comp:corosync=flatiron + comp:pacemaker[cman]", *sys_id)) def cluster_pcs_needle(*sys_id): """Whether particular `sys_id` corresponds to pcs-needle cluster system""" return bool(infer("comp:corosync=needle + comp:pacemaker[coro]", *sys_id)) def cluster_pcs_1_2(*sys_id): """Whether particular `sys_id` corresponds to pacemaker with 1.2+ schema""" return not any(( infer("comp:pacemaker=1.1.0", *sys_id), infer("comp:pacemaker=1.0", *sys_id), infer("comp:pacemaker=0", *sys_id), )) def _find_meta(meta, which, *sys_id, **kwargs): meta_comp = '::'.join((meta, which)) res = infer(':'.join(('comp', meta_comp)), *sys_id) for i in res: if meta_comp in i: return i[meta_comp] else: return kwargs.get('default') def package(which, *sys_id): return _find_meta('pkg', which, *sys_id, default=which) def cmd_pkg_install(pkgs, *sys_id): # ready to deal with pkgs being a generator cmd = _find_meta('cmd', 'pkg-install', *sys_id) packages = ' '.join(pkgs) return cmd.format(packages=packages) if cmd and packages else '' cluster_systems = (cluster_pcs_flatiron, cluster_pcs_needle) def cluster_unknown(*sys_id): return not(any(cluster_sys(*sys_id) for cluster_sys in cluster_systems))