path: root/bin/metabuild

#!/usr/bin/python

# metabuild: Generic build system wrapper
# Copyright 2010, 2011 Colin Walters <walters@verbum.org>
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)

# metabuild currently just wraps autotools (configure+make).
# To use it, you must first use the "inroot" tool to enter an alternative
# buildroot.
#
# $ inroot /path/to/buildroot bash
#
# Next, just type:
# $ metabuild
# This will:
#  1) Run ./configure if necessary
#  2) Run make
#
# The build output is automatically logged to $TMPDIR/build-$(PWD).log.
# For example, invoking metabuild in a directory named "foo" will log
# to /tmp/build-foo.log
#
# You can pass arguments to metabuild; if they start with '--', they're
# given to configure.  Otherwise, they're passed to make.
#
# $ metabuild --enable-libfoo  # passed to configure
# $ metabuild -j 1             # passed to make

import os,sys,subprocess,tempfile,re
from multiprocessing import cpu_count
import glib,gio

if 'INROOT_DIR' not in os.environ:
    print "INROOT_DIR not set; run under inroot"
    sys.exit(1)
root = os.environ['INROOT_DIR']
if os.path.isdir('/lib64'):
    libdir=os.path.join(root, 'lib64')
else:
    libdir=os.path.join(root, 'lib')

# "Constants" (well, some are derived from the environment)
subprocess_nice_args = ['nice', 'ionice', '-c', '3', '-t']

# In the future we should test for this better; possibly implement a
# custom fork handler
if os.uname()[0] == 'Linux':
    subprocess_nice_args = ['chrt', '--idle', '0'] + subprocess_nice_args

output_print_timeout_mseconds = 5000

warning_re = re.compile(r': ((warning)|(error)|(fatal error)): ')
output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(metabuild: )')
default_make_parallel = ['-j', '%d' % (cpu_count() * 6, )]
configargs = ['--prefix=' + root, '--libdir=' + libdir]
makeargs = ['make']

target_phase = 'build'

for arg in sys.argv[1:]:
    if arg.startswith('--'):
        configargs.append(arg)
    elif arg == 'install':
        target_phase = 'install'
    else:
        makeargs.append(arg)

loop = glib.MainLoop()

class OutputFilter(object):
    def __init__(self, filename, output):
        self.filename = filename
        self.output = output

        # inherit globals
        self._warning_re = warning_re
        self._nonfilter_re = output_whitelist_re
        self._timeout_write_status_msec = output_print_timeout_mseconds

        self._buf = ''
        self._warning_count = 0
        self._filtered_line_count = 0
        self._timeout_write_status_id = 0
        self._gfile = gio.File(path=filename)
        self._mon = self._gfile.monitor(gio.FILE_MONITOR_NONE)
        self._instream = self._gfile.read()
        self._read_queued = False
        self._final_read = False
        self._quit_data = None
        self._mon.connect('changed', self._on_changed)

    def _do_read(self):
        if self._read_queued:
            return
        self._read_queued = True
        self._instream.read_async(8192, self._on_read)

    def _timeout_write_status(self):
        self.output.write("metabuild: %d lines of output filtered\n" % (self._filtered_line_count, ))
        self._filtered_line_count = 0
        self._timeout_write_status_id = 0
        return False

    def _write_last_log_lines(self):
        f = open(logfile_path)
        lines = []
        for line in f:
            if line.startswith('metabuild: '):
                continue
            lines.append(line)
            if len(lines) > 10:
                lines.pop(0)
        f.close()
        for line in lines:
            self.output.write(line)

    def _final_output(self, successful):
        if not successful:
            # disabled temporarily
            #self._write_last_log_lines()
            pass
        self.output.write("metabuild: %s %s: %d warnings\n" % (target_phase,
                                                               'success' if successful else 'failed',
                                                               self._warning_count, ))
        self.output.write("metabuild: full log path: %s\n" % (logfile_path, ))
        sys.exit(0 if successful else 1)

    def _flush(self):
        while True:
            p = self._buf.find('\n')
            if p < 0:
                break
            line = self._buf[0:p]
            self._buf = self._buf[p+1:]
            match = self._warning_re.search(line)
            if match:
                self._warning_count += 1
                self.output.write(line + '\n')
            else:    
                match = self._nonfilter_re.search(line)
                if match:
                    self.output.write(line + '\n')
                else:
                    self._filtered_line_count += 1
        if self._timeout_write_status_id == 0:
            self._timeout_write_status_id = glib.timeout_add(self._timeout_write_status_msec, self._timeout_write_status)

    def _on_read(self, src, result):
        self._read_queued = False
        buf = src.read_finish(result)
        if buf != '':
            self._buf += buf
            self._flush()
            self._do_read()
        elif self._quit_data:
            if self._final_read:
                if self._timeout_write_status_id > 0:
                    glib.source_remove(self._timeout_write_status_id)
                    self._timeout_write_status_id = 0
                self._final_output(self._quit_data[1])
                self._quit_data[0].quit()
            else:
                self._final_read = True
                self._do_read()

    def _on_changed(self, mon, gfile, other, event):
        self._do_read()

    def start(self):
        self._do_read()

    def finish(self, loop, success):
        self._quit_data = (loop, success)
        self._do_read()

tempdir = os.environ.get('TMPDIR', '/tmp')
logfile_path = os.path.join(tempdir, 'build-%s.log' % (os.path.basename(os.getcwd()), ))
try:
    os.unlink(logfile_path)
except OSError, e:
    pass
logfile_write_fd = os.open(logfile_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
logfile_f = os.fdopen(logfile_write_fd, "w")
sys.stdout.write('metabuild: logging to %r\n' % (logfile_path, ))
sys.stdout.flush()

loop = glib.MainLoop()

tail = OutputFilter(logfile_path, sys.stdout)
tail.start()

def log(msg):
    fullmsg = 'metabuild: ' + msg + '\n'
    logfile_f.write(fullmsg)
    logfile_f.flush()

def global_failure_handler():
    tail.finish(loop, False)

def fatal(msg):
    log(msg)
    global_failure_handler()

class BuildProcess(object):
    def __init__(self, args, cwd=None, nice=True, env=None):
        if env is not None:
            srcenv = env
        else:
            srcenv = os.environ
        self.env = []
        for k, v in srcenv.iteritems():
            self.env.append('%s=%s' % (k, v))
        if nice:
            self.args = list(subprocess_nice_args)
            self.args.extend(args)
        else:
            self.args = args
        self.pid = None
        self.next_callback = None

    def _child_setup(self, *args):
        nullfd = os.open('/dev/null', os.O_RDONLY)
        os.dup2(nullfd, 0)
        os.close(nullfd)
        os.dup2(logfile_write_fd, 1)

        os.dup2(logfile_write_fd, 2)

    def _exit_callback(self, pid, condition):
        log("pid %d exited with condition %d" % (pid, condition))
        if condition != 0:
            global_failure_handler()
        else:
            glib.idle_add(self.next_callback)

    def run_async(self, exit_callback):
        log("Running: %r" % (self.args, ))
        (pid, stdin_fd, stdout_fd, stderr_fd) = \
            glib.spawn_async(self.args,
                             envp=self.env,
                             flags=(glib.SPAWN_DO_NOT_REAP_CHILD | glib.SPAWN_SEARCH_PATH),
                             child_setup=self._child_setup)
        self.pid = pid
        self.next_callback = exit_callback
        glib.child_watch_add(pid, self._exit_callback)

class BuildSystemScanner(object):
    @classmethod
    def _find_file(cls, names):
        for name in names:
            if os.path.exists(name):
                return name
        return None

    @classmethod
    def get_configure_source_script(cls):
        return cls._find_file(('./configure.ac', './configure.in'))

    @classmethod
    def get_configure_script(cls):
        return cls._find_file(('./configure', ))

    @classmethod
    def get_bootstrap_script(cls):
        return cls._find_file(('./autogen.sh', ))

    @classmethod
    def get_makefile(cls):
        return cls._find_file(('Makefile', ))

    @classmethod
    def get_make_requires_v1(cls):
        src = cls.get_configure_source_script()
        if not src:
            return False
        f = open(src)
        for line in f:
            if line.find('AM_SILENT_RULES') >= 0:
                f.close()
                return True
        f.close()
        return False

def phase_bootstrap():        
    have_configure = BuildSystemScanner.get_configure_script() 
    have_configure_source = BuildSystemScanner.get_configure_source_script()
    if not (have_configure or have_configure_source):
        fatal("No configure or bootstrap script detected; unknown buildsystem")
        return
    if have_configure:
        phase_configure()
    else:
        bootstrap = BuildSystemScanner.get_bootstrap_script()
        if bootstrap:
            log("Detected bootstrap script: %s, using it" % (bootstrap, ))
            args = [bootstrap]
            args.extend(configargs)
            # Add NOCONFIGURE; GNOME style scripts use this
            env = dict(os.environ)
            env['NOCONFIGURE'] = '1'
            bootstrap_proc= BuildProcess(args, env=env)
            bootstrap_proc.run_async(phase_configure)
        else:
            log("No bootstrap script found; using generic autoreconf")
            bootstrap_proc = BuildProcess(['autoreconf', '-f', '-i'])
            bootstrap_proc.run_async(phase_configure)
            phase_configure()

def phase_configure():
    prefix_matches=True
    configure = BuildSystemScanner.get_configure_script()
    if configure and os.path.exists('config.log'):
        previous_prefix = None
        f = open('config.log')
        for line in f:
            if line.startswith('prefix=\''):
                previous_prefix = line[8:-2]
                break
        f.close()
        if previous_prefix != root:
            log("Reruning configure due to prefix change (%r -> %r)" % (root, previous_prefix))
            prefix_matches=False
    if configure and (not os.path.exists('Makefile') or not prefix_matches):
        log("Detected configure script, using it")
        args = ['./configure']
        args.extend(configargs)
        configure = BuildProcess(args)
        configure.run_async(phase_build)
    else:
        phase_build()

build_status = False

def _phase_build_makefile():
    log("Detected Makefile, using it")
    args = makeargs
    user_specified_jobs = False
    for arg in args:
        if arg == '-j':
            user_specified_jobs = True

    need_v1 = BuildSystemScanner.get_make_requires_v1()
    if need_v1:
        log("Detected AM_SILENT_RULES, overriding it with V=1")
        args.append('V=1')

    if not user_specified_jobs:
        log("No jobs specified; overriding to be default parallel")
        args.extend(default_make_parallel)

    make = BuildProcess(args)
    make.run_async(phase_install)

def phase_build():
    if os.path.exists('Makefile'):
        _phase_build_makefile()
    else:
        log("Couldn't find supported build system")
        log("Known systems:")
        log("    Makefile: make")

def _phase_install_makefile():
    log("Doing install")
    make = BuildProcess(['make', 'install'])
    make.run_async(phase_complete)

def phase_install():
    if target_phase != 'install':
        phase_complete()
        return
    if os.path.exists('Makefile'):
        _phase_install_makefile()

def phase_complete():
    tail.finish(loop, True)

# Start off the process
phase_bootstrap()
loop.run()