diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/log_picker/__init__.py | 146 | ||||
| -rw-r--r-- | utils/log_picker/archiving.py | 100 | ||||
| -rw-r--r-- | utils/log_picker/argparser.py | 279 | ||||
| -rw-r--r-- | utils/log_picker/logmining.py | 255 | ||||
| -rw-r--r-- | utils/log_picker/sending/__init__.py | 42 | ||||
| -rw-r--r-- | utils/log_picker/sending/bugzillasender.py | 62 | ||||
| -rw-r--r-- | utils/log_picker/sending/emailsender.py | 71 | ||||
| -rw-r--r-- | utils/log_picker/sending/ftpsender.py | 47 | ||||
| -rw-r--r-- | utils/log_picker/sending/localsender.py | 39 | ||||
| -rw-r--r-- | utils/log_picker/sending/scpsender.py | 62 | ||||
| -rw-r--r-- | utils/log_picker/sending/senderbaseclass.py | 25 | ||||
| -rw-r--r-- | utils/log_picker/sending/stratasender.py | 55 | ||||
| -rwxr-xr-x | utils/logpicker | 136 |
13 files changed, 1319 insertions, 0 deletions
diff --git a/utils/log_picker/__init__.py b/utils/log_picker/__init__.py new file mode 100644 index 000000000..61bdaa175 --- /dev/null +++ b/utils/log_picker/__init__.py @@ -0,0 +1,146 @@ +#!/usr/bin/python + +import os +import sys +import tempfile + +import log_picker.archiving as archiving +from log_picker.archiving import ArchivationError +from log_picker.archiving import NoFilesArchivationError +import log_picker.sending as sending +from log_picker.sending import SenderError +import log_picker.logmining as logmining +from log_picker.logmining import LogMinerError + + +class LogPickerError(Exception): + pass + + +class LogPicker(object): + + def __init__(self, archive_obj=None, sender_obj=None, miners=[], + use_one_file=False): + self.sender_obj = sender_obj + self.archive_obj = archive_obj + self.miners = miners + + self.archive = None + self.tmpdir = None + self.files = [] + self.filename = self._get_tmp_file("completelog") if use_one_file else None + + + def _errprint(self, msg): + """Print message on stderr.""" + sys.stderr.write('%s\n' % msg) + + + def _get_tmp_file(self, name, suffix="", register=True): + """Create temp file.""" + if not self.tmpdir: + self.tmpdir = tempfile.mkdtemp(prefix="lp-logs-", dir="/tmp") + + name += suffix + filename = os.path.join(self.tmpdir, name) + open(filename, 'w') # Create empty file + + if register: + self.files.append(filename) + return filename + + + def create_archive(self, name=""): + """Create archive (one file) containing multiple log files.""" + name = name or self.tmpdir or "logs" + self.archive = self._get_tmp_file(name, + suffix=self.archive_obj.file_ext, register=False) + try: + self.archive_obj.create_archive(self.archive, self.files) + except (ArchivationError): + os.remove(self.archive) + raise + + + def send(self): + """Send log/archive with logs via sender object.""" + + if not len(self.files): + return + + if not self.archive and len(self.files) > 1: + raise LogPickerError('More than one file to send. ' + \ + 'You have to create archive. Use create_archive() method.') + + file = self.files[0] + contenttype = "text/plain" + if self.archive: + file = self.archive + contenttype = self.archive_obj.mimetype + + self.sender_obj.sendfile(file, contenttype) + + + def getlogs(self): + """Collect logs generated by miners passed to the constructor.""" + + # self.filename != None means that we should put all logs into one file. + # self.filename == None means that every log should have its own file. + if self.filename: + f = open(self.filename, 'w') + + for miner in self.miners: + if not self.filename: + tmpfilename = self._get_tmp_file(miner.get_filename()) + f = open(tmpfilename, 'w') + + desc = "%s\n\n" % (miner.get_description()) + f.write(desc) + try: + miner.set_logfile(f) + miner.getlog() + except (LogMinerError) as e: + self._errprint("Warning: %s - %s" % (miner._name, e)) + f.write("\n%s\n\n\n" % e) + + if not self.filename: + f.close() + # XXX Cut our anaconda dump into pieces. + if isinstance(miner, logmining.AnacondaLogMiner): + self._cut_to_pieces(tmpfilename) + + if self.filename: + f.close() + + + def _cut_to_pieces(self, filename): + """Create multiple log files from Anaconda dump. + Attention: Anaconda dump file on input will be used and overwritten! + @filename file with Anaconda dump""" + actual_file = os.path.basename(filename) + files = {actual_file: []} + empty_lines = 0 + + # Split file into memmory + for line in open(filename): + striped = line.strip() + + if not striped: + empty_lines += 1 + elif empty_lines > 1 and striped.startswith('/') \ + and striped.endswith(':') and len(line) > 2: + actual_file = striped[:-1].rsplit('/', 1)[-1]#.replace('.', '-') + files[actual_file] = [] + empty_lines = 0 + + files[actual_file].append(line) + + # Overwrite original file + actual_file = os.path.basename(filename) + open(filename, 'w').writelines(files[actual_file]) + del files[actual_file] + + # Write other individual files + for file in files: + open(self._get_tmp_file(file), 'w').writelines(files[file]) + diff --git a/utils/log_picker/archiving.py b/utils/log_picker/archiving.py new file mode 100644 index 000000000..fb3da9c39 --- /dev/null +++ b/utils/log_picker/archiving.py @@ -0,0 +1,100 @@ +import os +import tempfile +import tarfile +import bz2 + + +class ArchivationError(Exception): + pass + +class NoFilesArchivationError(ArchivationError): + pass + +class ArchiveBaseClass(object): + """Base class for archive classes.""" + + _compression = False + _ext = ".ext" + _mimetype = "" + + def __init__(self, *args, **kwargs): + self._tar_ext = ".tar" + + @property + def support_compression(self): + """Return True if compression is supported/used.""" + return self._compression + + @property + def file_ext(self): + """Return extension for output file.""" + return self._ext + + @property + def mimetype(self): + """Return archive mime type.""" + return self._mimetype + + def _create_tmp_tar(self, filelist): + _, tmpfile = tempfile.mkstemp(suffix=self._tar_ext) + tar = tarfile.open(tmpfile, "w") + for name in filelist: + pieces = name.rsplit('/', 2) + arcname = "%s/%s" % (pieces[-2], pieces[-1]) + tar.add(name, arcname=arcname) + tar.close() + return tmpfile + + def create_archive(self, outfilename, filelist): + raise NotImplementedError() + + +class Bzip2Archive(ArchiveBaseClass): + """Class for bzip2 compression.""" + + _compression = True + _ext = ".bz2" + _mimetype = "application/x-bzip2" + + def __init__(self, usetar=True, *args, **kwargs): + ArchiveBaseClass.__init__(self, args, kwargs) + self.usetar = usetar + + @property + def file_ext(self): + """Return extension for output file.""" + if self.usetar: + return "%s%s" % (self._tar_ext, self._ext) + return self._ext + + def create_archive(self, outfilename, filelist): + """Create compressed archive containing files listed in filelist.""" + if not filelist: + raise NoFilesArchivationError("No files to archive.") + + size = 0 + for file in filelist: + size += os.path.getsize(file) + if size <= 0: + raise NoFilesArchivationError("No files to archive.") + + if not self.usetar and len(filelist) > 1: + raise ArchivationError( \ + "Bzip2 cannot archive multiple files without tar.") + + if self.usetar: + f_in_path = self._create_tmp_tar(filelist) + else: + f_in_path = filelist[0] + + f_in = open(f_in_path, 'rb') + f_out = bz2.BZ2File(outfilename, 'w') + f_out.writelines(f_in) + f_out.close() + f_in.close() + + if self.usetar: + os.remove(f_in_path) + + return outfilename + diff --git a/utils/log_picker/argparser.py b/utils/log_picker/argparser.py new file mode 100644 index 000000000..7bc8a0bfa --- /dev/null +++ b/utils/log_picker/argparser.py @@ -0,0 +1,279 @@ +import optparse +import log_picker.sending as sending + + +class ArgError(Exception): + pass + + +class SimpleOptionGroup (optparse.OptionGroup): + + def _group_info_helper(self, formatter): + res = "" + lines = self.description.split('\n') + for line in lines: + res += formatter._format_text(line) + res += '\n' + return res + + def format_help(self, formatter): + result = formatter.format_heading(self.title) + formatter.indent() + result += self._group_info_helper(formatter) + formatter.dedent() + return result + + +class _OptionParserWithRaise(optparse.OptionParser): + def error(self, msg): + raise ArgError(msg) + + +class ArgParser(object): + + def __init__(self): + self.options = None + self.parser = None + + + def _generate_bz_it_group(self): + if sending.RHBZ in sending.NOT_AVAILABLE and \ + sending.STRATA in sending.NOT_AVAILABLE: + return None + + if sending.RHBZ in sending.NOT_AVAILABLE: + title = "Send the report to the Red Hat Ticketing system | options" + params_info = \ + "-r, --rhel \n"\ + " Send the report to the Red Hat Ticketing \n"\ + " system. \n"\ + "-i ID, --idbug=ID \n"\ + " The case number in the Red Hat Ticketing \n"\ + " system. \n"\ + "-l USERNAME, --login=USERNAME \n"\ + " Set the Red Hat Customer Portal username. \n" + + bzg = SimpleOptionGroup(self.parser, title, params_info) + bzg.add_option("-r", "--rhel", action="store_true", dest="strata") + + elif sending.STRATA in sending.NOT_AVAILABLE: + title = "Send the report to the Bugzilla | options" + params_info = \ + "-b, --bugzilla \n"\ + " Send the report to the Bugzilla. \n"\ + "-i ID, --idbug=ID \n"\ + " Set the bug id in the Bugzilla. \n"\ + "-l USERNAME, --login=USERNAME \n"\ + " Set the bugzilla username. \n" + + bzg = SimpleOptionGroup(self.parser, title, params_info) + bzg.add_option("-b", "--bugzilla", action="store_true", + dest="bugzilla") + + else: + title = "Send the report to the Bugzilla or the Red Hat Ticketing" \ + " system | options" + params_info = \ + "-b, --bugzilla \n"\ + " Send the report to the Bugzilla. \n"\ + "-r, --rhel \n"\ + " Send the report to the Red Hat Ticketing \n"\ + " system. \n"\ + "-i ID, --idbug=ID \n"\ + " Set the bug id in the Bugzilla/Case number \n"\ + " in the Red Hat Ticketing system. \n"\ + "-l USERNAME, --login=USERNAME \n"\ + " Set the Bugzilla/Red Hat Cutomer Portal \n"\ + " username. \n" + + bzg = SimpleOptionGroup(self.parser, title, params_info) + bzg.add_option("-b", "--bugzilla", action="store_true", + dest="bugzilla") + bzg.add_option("-r", "--rhel", action="store_true", dest="strata") + + bzg.add_option("-i", "--idbug", dest="bug_id", metavar="ID") + bzg.add_option("-l", "--login", dest="login", metavar="USERNAME") + + return bzg + + + def _generate_email_group(self): + if sending.EMAIL in sending.NOT_AVAILABLE: + return None + + title = "Send the report to an email | options" + params_info = \ + "-e, --email \n"\ + " Send the report to an email address. \n"\ + "-s ADDRESS, --server=ADDRESS \n"\ + " Set the SMTP server address. \n"\ + "-f EMAIL, --from=EMAIL \n"\ + " Set your email address. \n"\ + "-t EMAIL, --to=EMAIL \n"\ + " Set the destination email address. \n" + + + emailg = SimpleOptionGroup(self.parser, title, params_info) + emailg.add_option("-e", "--email", action="store_true", dest="email") + emailg.add_option("-s", "--server", dest="smtp_addr", metavar="ADDRESS") + emailg.add_option("-f", "--from", dest="from_addr", metavar="EMAIL") + emailg.add_option("-t", "--to", dest="to_addr", metavar="EMAIL") + return emailg + + + def _generate_scp_group(self): + if sending.SCP in sending.NOT_AVAILABLE: + return None + + title = "Send the report via secure copy (scp) | options" + params_info = \ + "-o, --scp \n"\ + " Send the report to the remote computer via scp.\n"\ + "-l USERNAME, --login=USERNAME \n"\ + " Set the remote username. \n"\ + "-a HOST, --host=HOST \n"\ + " Set the remote host address. \n"\ + "-p PATH, --path=PATH \n"\ + " Set the file path on the remote host. \n" + + scpg = SimpleOptionGroup(self.parser, title, params_info) + scpg.add_option("-o", "--scp", action="store_true", dest="scp") + scpg.add_option("-l", "--login", dest="login", metavar="USERNAME") + scpg.add_option("-a", "--host", dest="host", metavar="HOST") + scpg.add_option("-p", "--path", dest="path", metavar="PATH") + return scpg + + + def _generate_ftp_group(self): + if sending.FTP in sending.NOT_AVAILABLE: + return None + + title = "Upload the report via FTP | options" + params_info = \ + "-q, --ftp \n"\ + " Upload the report via ftp. \n"\ + "-l USERNAME, --login=USERNAME \n"\ + " Set the ftp username. \n"\ + " Note: For anonymous login don't use this option\n"\ + "-a HOST, --host=HOST \n"\ + " Set the remote host address. \n"\ + " Address syntax: [ftp://]host[:port][path] \n"\ + " Examples of host addresses: \n"\ + " host.com, ftp://host.com:21/path/on/the/host \n" + + ftpg = SimpleOptionGroup(self.parser, title, params_info) + ftpg.add_option("-q", "--ftp", action="store_true", dest="ftp") + ftpg.add_option("-l", "--login", dest="login", metavar="USERNAME") + ftpg.add_option("-a", "--host", dest="host", metavar="HOST") + return ftpg + + + def _generate_local_group(self): + if sending.LOCAL in sending.NOT_AVAILABLE: + return None + + title = "Save the report to the local computer | options" + params_info = \ + "-m, --local \n"\ + " Save the report to a directory on the computer.\n"\ + "-p DIRECTORY, --path=DIRECTORY \n"\ + " Set the local directory. \n" + + localg = SimpleOptionGroup(self.parser, title, params_info) + localg.add_option("-m", "--local", action="store_true", dest="local") + localg.add_option("-p", "--path", dest="path", metavar="PATH") + return localg + + + def _create_parser(self): + self.parser = _OptionParserWithRaise(conflict_handler="resolve") + self.parser.add_option("-c", "--comment", dest="bug_comment", + default=None, help="Report comment.", metavar="COMMENT") + + # Bugzilla and Red Hat Ticketing system options + group = self._generate_bz_it_group() + if group: self.parser.add_option_group(group) + + # Email options + group = self._generate_email_group() + if group: self.parser.add_option_group(group) + + # Scp options + group = self._generate_scp_group() + if group: self.parser.add_option_group(group) + + # Ftp options + group = self._generate_ftp_group() + if group: self.parser.add_option_group(group) + + # Local options + group = self._generate_local_group() + if group: self.parser.add_option_group(group) + + def _parse(self): + (self.options, _) = self.parser.parse_args() + + # Set sender attribute + if self.options.ensure_value('email', None): + self.options.sender = sending.EMAIL + elif self.options.ensure_value('strata', None): + self.options.sender = sending.STRATA + elif self.options.ensure_value('bugzilla', None): + self.options.sender = sending.RHBZ + elif self.options.ensure_value('scp', None): + self.options.sender = sending.SCP + elif self.options.ensure_value('ftp', None): + self.options.sender = sending.FTP + elif self.options.ensure_value('local', None): + self.options.sender = sending.LOCAL + else: + self.options.sender = None + + def _validate(self): + cnt = 0 + if self.options.ensure_value('email', None): cnt += 1 + if self.options.ensure_value('bugzilla', None): cnt += 1 + if self.options.ensure_value('strata', None): cnt += 1 + if self.options.ensure_value('scp', None): cnt += 1 + if self.options.ensure_value('ftp', None): cnt += 1 + if self.options.ensure_value('local', None): cnt += 1 + + if not cnt: + raise ArgError("No send method selected.") + elif cnt > 1: + raise ArgError("Options -b, -r, -e, -o, -q and -m" \ + " are mutually exclusive.") + + missing = [] + if self.options.ensure_value('email', None): + if not self.options.smtp_addr: missing.append('-s') + if not self.options.from_addr: missing.append('-f') + if not self.options.to_addr: missing.append('-t') + elif self.options.ensure_value('bugzilla', None): + if not self.options.bug_id: missing.append('-i') + if not self.options.login: missing.append('-l') + elif self.options.ensure_value('strata', None): + if not self.options.bug_id: missing.append('-i') + if not self.options.login: missing.append('-l') + elif self.options.ensure_value('scp', None): + if not self.options.login: missing.append('-l') + if not self.options.host: missing.append('-a') + elif self.options.ensure_value('ftp', None): + if not self.options.host: missing.append('-a') + elif self.options.ensure_value('local', None): + if not self.options.path: missing.append('-p') + + if missing: + msg = "" + for arg in missing: + msg += '\nArgument "%s" is missing!' % arg + raise ArgError(msg) + + + def parse(self): + """Parse and validate command line arguments.""" + self._create_parser() + self._parse() + self._validate() + return self.options + diff --git a/utils/log_picker/logmining.py b/utils/log_picker/logmining.py new file mode 100644 index 000000000..2bbf1c3cc --- /dev/null +++ b/utils/log_picker/logmining.py @@ -0,0 +1,255 @@ +import os +import stat +import shlex +import time +import subprocess + + +class LogMinerError(Exception): + pass + + +class LogMinerBaseClass(object): + """Base class for LogMiner classes. + LogMiner object represents one file/command/function + to get useful information (log).""" + + _name = "name" + _description = "Description" + _filename = "filename" + _prefer_separate_file = True + + def __init__(self, logfile=None, *args, **kwargs): + """@logfile open file object. This open file object will be used for + output generated during getlog() call.""" + self.logfile = logfile + self._used = False + + @classmethod + def get_filename(cls): + """Suggested log filename.""" + return cls._filename + + @classmethod + def get_description(cls): + """Log description.""" + return cls._description + + def set_logfile(self, logfile): + self.logfile = logfile + + def _write_separator(self): + self.logfile.write('\n\n') + + def _write_files(self, files): + if not isinstance(files, list): + files = [files] + + if self._used: + self._write_separator() + self._used = True + + for filename in files: + self.logfile.write('%s:\n' % filename) + try: + with open(filename, 'r') as f: + self.logfile.writelines(f) + self.logfile.write('\n') + except (IOError) as e: + self.logfile.write("Exception while opening: %s\n" % e) + continue + + def _run_command(self, command): + if self._used: + self._write_separator() + self._used = True + + if isinstance(command, basestring): + command = shlex.split(command) + + proc = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + self.logfile.write('STDOUT:\n%s\n' % out) + self.logfile.write('STDERR:\n%s\n' % err) + self.logfile.write('RETURN CODE: %s\n' % proc.returncode) + + def getlog(self): + """Create log and write it to a file object + recieved in the constructor.""" + self._action() + self._write_separator() + + def _action(self): + raise NotImplementedError() + + + +class AnacondaLogMiner(LogMinerBaseClass): + """Class represents way to get Anaconda dump.""" + + _name = "anaconda_log" + _description = "Log dumped from Anaconda." + _filename = "anaconda-dump" + _prefer_separate_file = True + + def _action(self): + # Actual state of /tmp + old_state = set(os.listdir('/tmp')) + + # Tell Anaconda to dump itself + try: + anaconda_pid = open('/var/run/anaconda.pid').read().strip() + except (IOError): + raise LogMinerError("Anaconda pid file doesn't exists") + + proc = subprocess.Popen(shlex.split("kill -s USR2 %s" % anaconda_pid), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc.communicate() + if proc.returncode: + raise LogMinerError('Error while sending signal to Anaconda') + + time.sleep(5) + + # Check if new traceback file exists + new_state = set(os.listdir('/tmp')) + tbpfiles = list(new_state - old_state) + + if not len(tbpfiles): + raise LogMinerError('Error: No anaconda traceback file exist') + + for file in tbpfiles: + if file.startswith('anaconda-tb-'): + tbpfile_name = file + break + else: + raise LogMinerError('Error: No anaconda traceback file exist') + + # Copy anaconda traceback log + self._write_files('/tmp/%s' % tbpfile_name) + + + +class FileSystemLogMiner(LogMinerBaseClass): + """Class represents way to get image of filesystem structure.""" + + _name = "filesystem" + _description = "Image of disc structure." + _filename = "filesystem" + _prefer_separate_file = True + + FSTREE_FORMAT = "%1s %6s%1s %s" # Format example: "d 1023.9K somedir" + DADPOINT = 1 # Number of Digits After the Decimal POINT + + def _action(self): + self._get_tree_structure() + + def _size_conversion(self, size): + """Converts bytes into KB, MB or GB""" + if size >= 1073741824: # Gigabytes + size = round(size / 1073741824.0, self.DADPOINT) + unit = "G" + elif size >= 1048576: # Megabytes + size = round(size / 1048576.0, self.DADPOINT) + unit = "M" + elif size >= 1024: # Kilobytes + size = round(size / 1024.0, self.DADPOINT) + unit = "K" + else: + size = size + unit = "" + return size, unit + + + def _get_tree_structure(self, human_readable=True): + """Creates filesystem structure image.""" + white_list = ['/sys'] + + logfile = self.logfile + + for path, dirs, files in os.walk('/'): + line = "\n%s:" % (path) + logfile.write('%s\n' % line) + + # List dirs + dirs.sort() + for directory in dirs: + fullpath = os.path.join(path, directory) + size = os.path.getsize(fullpath) + unit = "" + if human_readable: + size, unit = self._size_conversion(size) + line = self.FSTREE_FORMAT % ("d", size, unit, directory) + logfile.write('%s\n' % line) + + # Skip mounted directories + original_dirs = dirs[:] + for directory in original_dirs: + dirpath = os.path.join(path, directory) + if os.path.ismount(dirpath) and not dirpath in white_list: + dirs.remove(directory) + + # List files + files.sort() + for filename in files: + fullpath = os.path.join(path, filename) + if os.path.islink(fullpath): + line = self.FSTREE_FORMAT % ("l", "0", "", filename) + line += " -> %s" % os.path.realpath(fullpath) + if not os.path.isfile(fullpath): + # Broken symlink + line += " (Broken)" + else: + stat_res = os.stat(fullpath)[stat.ST_MODE] + if stat.S_ISREG(stat_res): + filetype = "-" + elif stat.S_ISCHR(stat_res): + filetype = "c" + elif stat.S_ISBLK(stat_res): + filetype = "b" + elif stat.S_ISFIFO(stat_res): + filetype = "p" + elif stat.S_ISSOCK(stat_res): + filetype = "s" + else: + filetype = "-" + + size = os.path.getsize(fullpath) + unit = "" + if human_readable: + size, unit = self._size_conversion(size) + line = self.FSTREE_FORMAT % (filetype, size, unit, filename) + logfile.write('%s\n' % line) + + + +class DmSetupLsLogMiner(LogMinerBaseClass): + """Class represents way to get 'dmsetup ls --tree' output.""" + + _name = "dmsetup ls" + _description = "Output from \"dmsetup ls --tree\"." + _filename = "dmsetup-ls" + _prefer_separate_file = True + + def _action(self): + self._run_command("dmsetup ls --tree") + + +class DmSetupInfoLogMiner(LogMinerBaseClass): + """Class represents way to get 'dmsetup info' output.""" + + _name = "dmsetup info" + _description = "Output from \"dmsetup info -c\"." + _filename = "dmsetup-info" + _prefer_separate_file = True + + def _action(self): + self._run_command("dmsetup info -c") + + +ALL_MINERS = [AnacondaLogMiner(), + FileSystemLogMiner(), + DmSetupLsLogMiner(), + DmSetupInfoLogMiner(), + ] + diff --git a/utils/log_picker/sending/__init__.py b/utils/log_picker/sending/__init__.py new file mode 100644 index 000000000..9571af4d7 --- /dev/null +++ b/utils/log_picker/sending/__init__.py @@ -0,0 +1,42 @@ +from log_picker.sending.senderbaseclass import SenderError + +RHBZ = 0 # RedHat Bugzilla +EMAIL = 1 # Email +STRATA = 2 # Red Hat ticketing system +SCP = 3 # Scp +FTP = 4 # Ftp +LOCAL = 5 # Local + +NOT_AVAILABLE = [] + + +try: + from log_picker.sending.bugzillasender import RedHatBugzilla +except (ImportError): + NOT_AVAILABLE.append(RHBZ) + +try: + from log_picker.sending.emailsender import EmailSender +except (ImportError): + NOT_AVAILABLE.append(EMAIL) + +try: + from log_picker.sending.stratasender import StrataSender +except (ImportError): + NOT_AVAILABLE.append(STRATA) + +try: + from log_picker.sending.scpsender import ScpSender +except (ImportError): + NOT_AVAILABLE.append(SCP) + +try: + from log_picker.sending.ftpsender import FtpSender +except (ImportError): + NOT_AVAILABLE.append(FTP) + +try: + from log_picker.sending.localsender import LocalSender +except (ImportError): + NOT_AVAILABLE.append(LOCAL) + diff --git a/utils/log_picker/sending/bugzillasender.py b/utils/log_picker/sending/bugzillasender.py new file mode 100644 index 000000000..1fc4d5915 --- /dev/null +++ b/utils/log_picker/sending/bugzillasender.py @@ -0,0 +1,62 @@ +import os +from log_picker.sending.senderbaseclass import SenderBaseClass +from log_picker.sending.senderbaseclass import SenderError +from report.plugins.bugzilla import filer +from report.plugins.bugzilla.filer import CommunicationError +from report.plugins.bugzilla.filer import LoginError + + +class BugzillaBaseClass(SenderBaseClass): + + _bz_address = "" + _bz_xmlrpc = "" + _description = "" + + def __init__(self, *args, **kwargs): + SenderBaseClass.__init__(self, args, kwargs) + self.bzfiler = None + self.bug_id = None + self.comment = None + + def connect_and_login(self, username, password): + try: + self.bzfiler = filer.BugzillaFiler(self._bz_xmlrpc, self._bz_address, + filer.getVersion(), filer.getProduct()) + self.bzfiler.login(username, password) + except (CommunicationError, LoginError) as e: + raise SenderError("%s. Bad username or password?" % e) + except (ValueError) as e: + raise SenderError("%s" % e) + + def set_bug(self, bug_id): + self.bug_id = bug_id + + def set_comment(self, comment): + self.comment = comment + + def sendfile(self, filename, contenttype): + description = self._get_description(self._description) + + dict_args = {'isprivate': False, + 'filename': os.path.basename(filename), + 'contenttype': contenttype} + + if self.comment: + dict_args['comment'] = self.comment + + try: + bug = self.bzfiler.getbug(self.bug_id) + bug.attachfile(filename, description, **dict_args) + except (CommunicationError, ValueError) as e: + raise SenderError(e) + + +class RedHatBugzilla(BugzillaBaseClass): + + _bz_address = "http://bugzilla.redhat.com" + _bz_xmlrpc = "https://bugzilla.redhat.com/xmlrpc.cgi" + _description = "LogPicker" + + def __init__(self, *args, **kwargs): + BugzillaBaseClass.__init__(self, args, kwargs) + diff --git a/utils/log_picker/sending/emailsender.py b/utils/log_picker/sending/emailsender.py new file mode 100644 index 000000000..03c94ca62 --- /dev/null +++ b/utils/log_picker/sending/emailsender.py @@ -0,0 +1,71 @@ +import os +import smtplib +import email +import email.encoders +import email.mime.multipart +from log_picker.sending.senderbaseclass import SenderBaseClass +from log_picker.sending.senderbaseclass import SenderError + + +class EmailSender(SenderBaseClass): + + _description = "Email from LogPicker" + + def __init__(self, sendby, addresses, smtp_server, *args, **kwargs): + """Send file by email. + @sendby - Sender email address. (string) + @addresses - List of destination email addresses. (list) + @smtp_server - SMTP server address. (string)""" + + SenderBaseClass.__init__(self, args, kwargs) + self.smtp_server = smtp_server + self.sendby = sendby + self.addresses = addresses + self.comment = "" + + def set_comment(self, comment): + self.comment = comment + + def sendfile(self, filename, contenttype): + # Create email message + msg = email.mime.multipart.MIMEMultipart() + msg['Subject'] = self._get_description(self._description) + msg['From'] = self.sendby + msg['To'] = ', '.join(self.addresses) + msg.preamble = 'This is a multi-part message in MIME format.' + + # Add message text + msgtext = email.mime.base.MIMEBase("text", "plain") + msgtext.set_payload(self.comment) + msg.attach(msgtext) + + # Add message attachment + cont_type = contenttype.split('/', 1) + if len(cont_type) == 1: + cont_type.append("") + elif not cont_type: + cont_type = ["application", "octet-stream"] + + attach_data = open(filename, 'rb').read() + + msgattach = email.mime.base.MIMEBase(cont_type[0], cont_type[1]) + msgattach.set_payload(attach_data) + email.encoders.encode_base64(msgattach) + msgattach.add_header('Content-Disposition', 'attachment', + filename=os.path.basename(filename)) + msg.attach(msgattach) + + # Send message + try: + s = smtplib.SMTP(self.smtp_server) + except(smtplib.socket.gaierror, smtplib.SMTPServerDisconnected): + raise SenderError("Email cannot be send. " +\ + "Error while connecting to smtp server.") + + try: + s.sendmail(self.sendby, self.addresses, msg.as_string()) + except(smtplib.SMTPRecipientsRefused) as e: + raise SenderError("Email cannot be send. Wrong destination " +\ + "email address?\nErr message: %s" % e) + s.quit() + diff --git a/utils/log_picker/sending/ftpsender.py b/utils/log_picker/sending/ftpsender.py new file mode 100644 index 000000000..92f7184cb --- /dev/null +++ b/utils/log_picker/sending/ftpsender.py @@ -0,0 +1,47 @@ +import os +import urlparse +import ftplib +from log_picker.sending.senderbaseclass import SenderBaseClass +from log_picker.sending.senderbaseclass import SenderError + +# This class uses code from module report.plugins.scp + +class FtpSender(SenderBaseClass): + + def __init__(self, *args, **kwargs): + SenderBaseClass.__init__(self, args, kwargs) + self.host = None + self.username = None + self.password = None + + def set_host(self, host): + if not host.startswith('ftp://'): + host = 'ftp://' + host + self.host = host + + def set_login(self, username, password): + self.username = username + self.password = password + + def sendfile(self, filename, contenttype): + _, netloc, path, _, _, _ = urlparse.urlparse(self.host) + if netloc.find(':') > 0: + netloc, port = netloc.split(':') + else: + port = 21 + + try: + ftp = ftplib.FTP() + ftp.connect(netloc, port) + if self.username: + ftp.login(self.username, self.password) + else: + ftp.login() + ftp.cwd(path) + ftp.set_pasv(True) + ftp.storbinary('STOR %s' % os.path.basename(filename), \ + file(filename)) + ftp.quit() + except ftplib.all_errors, e: + raise SenderError("FTP upload failed: %(error)s" % {'error':e}) + diff --git a/utils/log_picker/sending/localsender.py b/utils/log_picker/sending/localsender.py new file mode 100644 index 000000000..24fe9ec88 --- /dev/null +++ b/utils/log_picker/sending/localsender.py @@ -0,0 +1,39 @@ +import os +import shutil +from log_picker.sending.senderbaseclass import SenderBaseClass +from log_picker.sending.senderbaseclass import SenderError + + +class LocalSender(SenderBaseClass): + + def __init__(self, *args, **kwargs): + SenderBaseClass.__init__(self, args, kwargs) + self.path = None + + def set_path(self, directory): + self.path = directory + + if os.path.exists(self.path) and not os.path.isdir(self.path): + raise SenderError('Cannot create "%s" directory. A file of ' + 'the same name already exists.' % self.path) + + + def sendfile(self, filename, contenttype): + + # Another check due possibility of race condition + if os.path.exists(self.path): + if not os.path.isdir(self.path): + raise SenderError('Cannot create "%s" directory. A file of ' + 'the same name already exists.' % self.path) + else: + try: + os.makedirs(self.path) + except Exception as e: + raise SenderError('Cannot create "%s" directory: %s' % \ + (self.path, e)) + try: + shutil.copy(filename, self.path) + except Exception as e: + raise SenderError('Could not save "%s" to "%s": %s' % \ + (os.path.basename(filename), self.path, e)) + diff --git a/utils/log_picker/sending/scpsender.py b/utils/log_picker/sending/scpsender.py new file mode 100644 index 000000000..a3842edf9 --- /dev/null +++ b/utils/log_picker/sending/scpsender.py @@ -0,0 +1,62 @@ +import os +import subprocess +from log_picker.sending.senderbaseclass import SenderBaseClass +from log_picker.sending.senderbaseclass import SenderError + + +class ScpSender(SenderBaseClass): + + def __init__(self, *args, **kwargs): + SenderBaseClass.__init__(self, args, kwargs) + self.host = None + self.port = None + self.username = None + self.path = "." + + + def set_host(self, host): + if host.find(":") != -1: + (self.host, port) = host.split(":") + try: + self.port = int(port) + except ValueError: + self.port = None + else: + self.host = host + + + def set_login(self, username): + self.username = username + + + def set_path(self, path): + if path: self.path = path + + + def sendfile(self, filename, contenttype): + port_args = [] + if self.port: + port_args = ["-P", self.port] + + target = "%s@%s:%s" % (self.username, self.host, self.path) + + command = ["scp", + "-q", + "-oGSSAPIAuthentication=no", + "-oHostbasedAuthentication=no", + "-oPubkeyAuthentication=no", + "-oChallengeResponseAuthentication=no", + "-oPasswordAuthentication=yes", + "-oNumberOfPasswordPrompts=1", + "-oStrictHostKeyChecking=no", + "-oUserKnownHostsFile=/dev/null", + ] + port_args + [filename, target] + + p = subprocess.Popen(command, stdin=subprocess.PIPE) + p.communicate() + + if p.returncode: + raise SenderError("Scp sending failed.\n" + \ + "Possible causes: Bad hostname, bad username, "\ + "bad password, host is down.") + diff --git a/utils/log_picker/sending/senderbaseclass.py b/utils/log_picker/sending/senderbaseclass.py new file mode 100644 index 000000000..30aec026d --- /dev/null +++ b/utils/log_picker/sending/senderbaseclass.py @@ -0,0 +1,25 @@ +import datetime +from socket import gethostname + + +class SenderError(Exception): + pass + + +class SenderBaseClass(object): + + def __init__(self, *args, **kwargs): + pass + + def sendfile(self, filename, contenttype): + raise NotImplementedError() + + def _get_description(self, prefix=""): + try: + hostname = gethostname() + except: + hostname = "" + date_str = datetime.datetime.now().strftime("%Y-%m-%d") + description = "%s (%s) %s" % (prefix, hostname, date_str) + return description + diff --git a/utils/log_picker/sending/stratasender.py b/utils/log_picker/sending/stratasender.py new file mode 100644 index 000000000..4f242da1a --- /dev/null +++ b/utils/log_picker/sending/stratasender.py @@ -0,0 +1,55 @@ +import xml.dom.minidom +from log_picker.sending.senderbaseclass import SenderBaseClass +from log_picker.sending.senderbaseclass import SenderError +from report.plugins.strata import send_report_to_existing_case +from report.plugins.strata import strata_client_strerror + + +class StrataSender(SenderBaseClass): + + _URL = "https://api.access.redhat.com/rs" + _CERT_DATA = "INSECURE" + + def __init__(self, *args, **kwargs): + SenderBaseClass.__init__(self, args, kwargs) + self.username = None + self.password = None + self.case_number = None + + def set_login(self, username, password): + self.username = username + self.password = password + + def set_case_number(self, case_num): + self.case_number = case_num + + def sendfile(self, filename, contenttype): + response = send_report_to_existing_case(self._URL, + self._CERT_DATA, + self.username, + self.password, + self.case_number, + filename) + + if not response: + raise SenderError("Sending log to the Red Hat Ticket System fail" +\ + " - %s" % strata_client_strerror()) + + # Try parse response + try: + dom = xml.dom.minidom.parseString(response) + mnode = dom.getElementsByTagName("response")[0] + title = mnode.getElementsByTagName("title")[0].childNodes[0].data + body = mnode.getElementsByTagName("body")[0].childNodes[0].data + except Exception as e: + raise SenderError("Sending log to the Red Hat Ticket System fail.") + + if title == "File Attachment Failed": + if body.startswith("401 Unauthorized"): + raise SenderError("Bad login or password.") + elif body.startswith("Error : CASE_NOT_FOUND"): + raise SenderError("Selected case doesn't exist.") + else: + raise SenderError("Sending log to the " +\ + "Red Hat Ticket System fail - %s" % body.strip()) + diff --git a/utils/logpicker b/utils/logpicker new file mode 100755 index 000000000..03d166995 --- /dev/null +++ b/utils/logpicker @@ -0,0 +1,136 @@ +#!/usr/bin/python + +import sys +import getpass +import log_picker + +import log_picker.argparser as argparser +from log_picker.argparser import ArgError +import log_picker.archiving as archiving +from log_picker.archiving import NoFilesArchivationError +import log_picker.sending as sending +from log_picker.sending import SenderError +import log_picker.logmining as logmining + + +class ApplicationScope(object): + """Application configuration class.""" + + def __init__(self, parser_options): + self.bug_comment = parser_options.ensure_value('bug_comment', None) + + self.bug_id = parser_options.ensure_value('bug_id', None) + self.login = parser_options.ensure_value('login', None) + self.password = None + + self.smtp_server = parser_options.ensure_value('smtp_addr', None) + self.from_addr = parser_options.ensure_value('from_addr', None) + self.to_addr = [] + if parser_options.ensure_value('to_addr', None): + self.to_addr = [parser_options.to_addr] + self.host = parser_options.ensure_value('host', None) + self.path = parser_options.ensure_value('path', None) + + # sender + self.sender = parser_options.ensure_value('sender', None) + + # miners + self.miners = logmining.ALL_MINERS + + +class Injector(object): + """Main factory class.""" + + @staticmethod + def inject_main_helper(scope): + logpicker = Injector.inject_logpicker(scope) + return MainHelper(logpicker) + + @staticmethod + def inject_logpicker(scope): + sender = Injector.inject_sender(scope) + archivator = Injector.inject_archivator(scope) + return log_picker.LogPicker(archive_obj=archivator, sender_obj=sender, + miners=scope.miners) + + @staticmethod + def inject_sender(scope): + if scope.sender == sending.RHBZ: + sender = sending.RedHatBugzilla() + sender.set_bug(scope.bug_id) + sender.set_comment(scope.bug_comment) + sender.connect_and_login(scope.login, scope.password) + return sender + if scope.sender == sending.EMAIL: + sender = sending.EmailSender(scope.from_addr, scope.to_addr, + scope.smtp_server) + sender.set_comment(scope.bug_comment) + return sender + if scope.sender == sending.STRATA: + sender = sending.StrataSender() + sender.set_login(scope.login, scope.password) + sender.set_case_number(scope.bug_id) + return sender + if scope.sender == sending.SCP: + sender = sending.ScpSender() + sender.set_login(scope.login) + sender.set_path(scope.path) + sender.set_host(scope.host) + return sender + if scope.sender == sending.FTP: + sender = sending.FtpSender() + sender.set_login(scope.login, scope.password) + sender.set_host(scope.host) + return sender + if scope.sender == sending.LOCAL: + sender = sending.LocalSender() + sender.set_path(scope.path) + return sender + + raise Exception("Unknown sender type.") + + @staticmethod + def inject_archivator(scope): + return archiving.Bzip2Archive() + + +class MainHelper(object): + """Main application class.""" + + def __init__(self, logpicker): + self.picker = logpicker + + def run(self): + self.picker.getlogs() + self.picker.create_archive() + self.picker.send() + print "Successfully completed!" + + + +if __name__ == "__main__": + + # Argument parsing + try: + options = argparser.ArgParser().parse() + except (ArgError) as e: + sys.stderr.write("Argument error: %s\n" % e) + sys.exit(1) + + # Application scope + scope = ApplicationScope(options) + if scope.sender == sending.RHBZ or scope.sender == sending.STRATA or \ + (scope.sender == sending.FTP and scope.login): + scope.password = getpass.getpass("Password: ") + + # Application + try: + app = Injector.inject_main_helper(scope) + app.run() + except (NoFilesArchivationError): + sys.stderr.write("Nothing to report.\n") + sys.exit(0) + except (SenderError) as e: + sys.stderr.write("Error: %s\n" % e) + sys.exit(1) + |