#!/bin/sh

# Source the common test script helpers
. /usr/bin/rhts_environment.sh
. ../../kernel/include/runtest.sh

CUR_TIME=$(date --date="$(date --utc)" +%s)
# control where to log debug messages to:
# devnull = 1 : log to /dev/null
# devnull = 0 : log to file specified in ${DEBUGLOG}
devnull=0

# Create debug log
DEBUGLOG=`mktemp -p /mnt/testarea -t DeBug.XXXXXX`

# locking to avoid races
lck=$OUTPUTDIR/$(basename $0).lck
  
# Functions

# Log a message to the ${DEBUGLOG} or to /dev/null
function DeBug ()
{
    local msg="$1"
    local timestamp=$(date '+%F %T')
    if [ "$devnull" = "0" ]; then
        (
            flock -x 200 2>/dev/null
            echo -n "${timestamp}: " >>$DEBUGLOG 2>&1
            echo "${msg}" >>$DEBUGLOG 2>&1
        ) 200>$lck
    else
        echo "${msg}" >/dev/null 2>&1
    fi
}

function RHTSAbort ()
{
    # Abort the rhts recipe if we are running the wrong kernel
    DeBug "Abort recipe"
    rhts-abort -t recipe
}

function SysReport ()
{
    DeBug "Enter SysReport"
    OUTPUTFILE=`mktemp /tmp/tmp.XXXXXX`
    grep -q "release 3 " /etc/redhat-release 
    if [ $? -eq 0 ]; then
	modarg=-d
	modarg2=
    else
	modarg="-F description"
	modarg2="-F version"
    fi
    sysnode=$(/bin/uname -n)
    syskernel=$(/bin/uname -r)
    sysmachine=$(/bin/uname -m)
    sysprocess=$(/bin/uname -p)
    sysuname=$(/bin/uname -a)
    sysswap=$(/usr/bin/free -m | /bin/awk '{if($1=="Swap:") {print $2,"MB"}}')
    sysmem=$(/usr/bin/free -m | /bin/awk '{if($1=="Mem:") {print $2,"MB"}}')
    syscpu=$(/bin/cat /proc/cpuinfo | /bin/grep processor | wc -l)
# Bios Info
    biosVendor=$(dmidecode --type=0 | /bin/grep -i vendor | /bin/awk -F: '{print $2}')
    biosVersion=$(dmidecode --type=0 | /bin/grep -i version | /bin/awk -F: '{print $2}')
    biosRelease=$(dmidecode --type=0 | /bin/grep -i release | /bin/awk -F: '{print $2}')
    biosRevision=$(dmidecode --type=0 | /bin/grep -i revision | /bin/awk -F: '{print $2}')
#
    syslspci=$(/sbin/lspci > $OUTPUTDIR/lspci.$kernbase)
    if [ -f /etc/fedora-release ]; then
	sysrelease=$(/bin/cat /etc/fedora-release)
    else
	sysrelease=$(/bin/cat /etc/redhat-release)
    fi
    syscmdline=$(/bin/cat /proc/cmdline)
    sysnmiint=$(/bin/cat /proc/interrupts | /bin/grep -i nmi)
    sysmodprobe=$(/bin/cat /etc/modprobe.conf > $OUTPUTDIR/modprobe.$kernbase)
    for x in $(/sbin/lsmod | /bin/cut -f1 -d" " 2>/dev/null | /bin/grep -v Module 2>/dev/null ); do
	echo "Checking module information $x:" >> $OUTPUTDIR/modinfo.$kernbase
	/sbin/modinfo $modarg $x >> $OUTPUTDIR/modinfo.$kernbase
	if [ -n "$modarg2" ]; then
	    /sbin/modinfo $modarg2 $x >> $OUTPUTDIR/modinfo.$kernbase
	fi
    done
    if [ -x /usr/sbin/sestatus ]; then
	syssestatus=$(/usr/sbin/sestatus >> $OUTPUTDIR/selinux.$kernbase)
    fi
    if [ -x /usr/sbin/xm ]; then
	syshypervisor=$(/usr/sbin/xm info >> $OUTPUTDIR/hypervisor.$kernbase)
    fi
    if [ -x /usr/sbin/semodule ]; then
	echo "********* SELinux Module list **********" >> $OUTPUTDIR/selinux.$kernbase
	syssemodulelist=$(/usr/sbin/semodule -l >> $OUTPUTDIR/selinux.$kernbase)
    fi

    sysderror=$(/bin/cat $OUTPUTDIR/boot.$kernbase | grep -i error | grep -v BIOS >> $OUTPUTDIR/derror.$kernbase)
    sysderror1=$(/bin/grep -i collision $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror2=$(/bin/grep -i fail $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror3=$(/bin/grep -i temperature $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror4=$(/bin/grep BUG: $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror5=$(/bin/grep INFO: $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror6=$(/bin/grep FATAL: $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror7=$(/bin/grep WARNING: $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror8=$(/bin/grep -i "command not found" $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/derror.$kernbase)
    sysderror9=$(/bin/cat $OUTPUTDIR/boot.$kernbase | grep avc: | grep -v granted >> $OUTPUTDIR/avcerror.$kernbase)
    sysderror10=$(/bin/grep -i "Unknown symbol" $OUTPUTDIR/boot.$kernbase >> $OUTPUTDIR/serror.$kernbase)

    echo "********** System Information **********" >> $OUTPUTFILE
    echo "Hostname                = $sysnode"       >> $OUTPUTFILE
    echo "Kernel Version          = $syskernel"     >> $OUTPUTFILE
    echo "Machine Hardware Name   = $sysmachine"    >> $OUTPUTFILE
    echo "Processor Type          = $sysprocess"    >> $OUTPUTFILE
    echo "uname -a output         = $sysuname"      >> $OUTPUTFILE
    echo "Swap Size               = $sysswap"       >> $OUTPUTFILE
    echo "Mem Size                = $sysmem"        >> $OUTPUTFILE
    echo "Number of Processors    = $syscpu"        >> $OUTPUTFILE
    echo "System Release          = $sysrelease"    >> $OUTPUTFILE
    echo "Command Line            = $syscmdline"    >> $OUTPUTFILE
    echo "System NMI Interrupts   = $sysnmiint"     >> $OUTPUTFILE
    echo "*********** BIOS Information ***********" >> $OUTPUTFILE
    echo "Vendor                  = $biosVendor"    >> $OUTPUTFILE
    echo "Version                 = $biosVersion"   >> $OUTPUTFILE
    echo "Release                 = $biosRelease"   >> $OUTPUTFILE
    echo "Revision                = $biosRevision"  >> $OUTPUTFILE
    echo "**************** LSPCI *****************" >> $OUTPUTFILE
    /bin/cat $OUTPUTDIR/lspci.$kernbase             >> $OUTPUTFILE
    echo "*************** Modprob ****************" >> $OUTPUTFILE
    /bin/cat $OUTPUTDIR/modprobe.$kernbase          >> $OUTPUTFILE
    echo "********** Module Information **********" >> $OUTPUTFILE
    /bin/cat $OUTPUTDIR/modinfo.$kernbase           >> $OUTPUTFILE
    if [ -x /usr/sbin/sestatus ]; then
	echo "************ SELinux Status ************" >> $OUTPUTFILE
	/bin/cat $OUTPUTDIR/selinux.$kernbase       >> $OUTPUTFILE
    fi
    echo "********** Interfaces Information **********" > ifcinfo
    echo "-- /etc/resolv.conf --" >> ifcinfo
    cat /etc/resolv.conf >> ifcinfo
    echo "-- END of /etc/resolv.conf --" >> ifcinfo
    echo "ip a" >> ifcinfo
    ip a 2>&1 >> ifcinfo
    echo "ip route" >> ifcinfo
    ip route 2>&1 >> ifcinfo
    local interfaces=`ip link show | grep '^[0-9]\+' | sed 's/^[0-9]\+: \([a-zA-Z0-9]\+\):.*/\1/'`
    for i in $interfaces; do
        echo "ethtool -i $i" >> ifcinfo
        ethtool -i $i 2>&1 >> ifcinfo
    done
    cat ifcinfo >> $OUTPUTFILE
    cat ifcinfo > /dev/console
    if [ -x /usr/sbin/xm ]; then
	echo "*********** Hypervisor info ************" >> $OUTPUTFILE
	/bin/cat $OUTPUTDIR/hypervisor.$kernbase     >> $OUTPUTFILE
    fi
    FAILURE=FALSE
    # Check dmesg log for issues
    dresult_count=0
    if [ -s $OUTPUTDIR/derror.$kernbase ]; then
	dresult_count=$(/usr/bin/wc -l $OUTPUTDIR/derror.$kernbase | awk '{print $1}')
	echo "******** Potential Issues dmesg ********" >> $OUTPUTFILE
	/bin/cat $OUTPUTDIR/derror.$kernbase        >> $OUTPUTFILE
    fi
    # Check dmesg log for failures
    if [ -s $OUTPUTDIR/serror.$kernbase ]; then
	dresult_count=$(/usr/bin/wc -l $OUTPUTDIR/serror.$kernbase | awk '{print $1}')
	echo "********** Failures in dmesg ***********" >> $OUTPUTFILE
	/bin/cat $OUTPUTDIR/serror.$kernbase        >> $OUTPUTFILE
	FAILURE=TRUE
    fi
    # Check dmesg log for avc failures
    if [ -s $FILEAREA/avcerror.$kernbase ]; then
	echo "********* SElinux AVC Failures *********" >> $OUTPUTFILE
	/bin/cat $FILEAREA/avcerror.$kernbase       >> $OUTPUTFILE
	FAILURE=TRUE
    fi
    echo "******** End System Information ********" >> $OUTPUTFILE
    if [ -s $OUTPUTDIR/derror.$kernbase -o -s $OUTPUTDIR/serror.$kernbase -o -s $OUTPUTDIR/avcerror.$kernbase ]; then
	result_count=$(/usr/bin/printf "%03d%03d%03d\n" $dresult_count 0 0)
	if [ $FAILURE = TRUE ]; then
	    report_result $TEST/Sysinfo FAIL $result_count
	else
	    report_result $TEST/Sysinfo PASS $result_count
	fi
    else
	report_result $TEST/Sysinfo PASS 0
    fi
    DeBug "Exit SysReport"
}
                
function DiffDmesg ()
{
    DeBug "Enter DiffDmesg"
    DMESGDIFFLOG=`mktemp -p /mnt/testarea -t DMSGDIFF.XXXXXX`
    filelist=`ls $OUTPUTDIR/boot.*`
    hit=0 
    for l in $filelist ; do
	hit=`expr $hit + 1`
	export FILE$hit=$l 
	DeBug "$FILE$l"
    done
    echo "
===================================================================
diff -u $FILE1 $FILE2
===================================================================" | tee -a $DMESGDIFFLOG
    /usr/bin/diff -u $FILE1 $FILE2 | tee -a $DMESGDIFFLOG
    DeBug "/usr/bin/diff -u $FILE1 $FILE2"
    sleep 3
    SubmitLog $DMESGDIFFLOG
    DeBug "Exit DiffDmesg"
}

function DiffLspci ()
{
    DeBug "Enter DiffLspci"
    LSPCIDIFFLOG=`mktemp -p /mnt/testarea -t LSPCIDIFF.XXXXXX`
    filelist=`ls $OUTPUTDIR/lspci.*`
    hit=0
    for l in $filelist ; do
	hit=`expr $hit + 1`
	export FILE$hit=$l
	DeBug "$FILE$l"
    done
    echo "
===================================================================
diff -u $FILE1 $FILE2
===================================================================" | tee -a $LSPCIDIFFLOG
    /usr/bin/diff -u $FILE1 $FILE2 | tee -a $LSPCIDIFFLOG
    DeBug "/usr/bin/diff -u $FILE1 $FILE2"
    sleep 3
    SubmitLog $LSPCIDIFFLOG
    DeBug "Exit DiffLspci"
}

function RprtRslt ()
{
    ONE=$1
    TWO=$2
    THREE=$3

    # File the results in the database
    report_result $ONE $TWO $THREE

    if [ "$TWO" == "FAIL" ]; then
	SubmitLog $DEBUGLOG
    fi
}

function SubmitLog ()
{
    LOG=$1
    rhts_submit_log -l $LOG
}

function XendLogging ()
{
    XENDCONF=/etc/sysconfig/xend
    DeBug "Enter XendLogging"
    if [ -e $XENDCONF ]; then
	DeBug "$XENDCONF exists"
	sed -i 's/#XENCONSOLED_LOG_HYPERVISOR=no/XENCONSOLED_LOG_HYPERVISOR=yes/g' $XENDCONF
	sed -i 's/#XENCONSOLED_LOG_GUESTS=no/XENCONSOLED_LOG_GUESTS=yes/g' $XENDCONF
	sed -i 's/#XENCONSOLED_LOG_DIR/XENCONSOLED_LOG_DIR/g' $XENDCONF
    fi
    DeBug "Exit XendLogging"
}

function LogBootloaderConfig ()
{
    conf_files="/boot/grub2/grub.cfg /boot/grub/grub.conf /boot/efi/efi/redhat/elilo.conf
    /boot/etc/yaboot.conf /etc/yaboot.conf /etc/zipl.conf"

    for conf_file in $conf_files; do
        if [ -f $conf_file ]; then
            echo "----- $conf_file -----" | tee -a $DEBUGLOG
            cat $conf_file | tee -a $DEBUGLOG
            echo "----- $conf_file END -----" | tee -a $DEBUGLOG
        fi
    done
}

function SelectKernel ()
{
    DeBug "Enter SelectKernel"
    VR=$1
    EXTRA=$2
    DeBug "VR=$VR EXTRA=$EXTRA"

    # If not version or Extra selected then choose the latest installed version
    if [ -z "$EXTRA" -a -z "$VR" ]; then
	DeBug "ERROR: missing args"
	return 1
    fi

    # Workaround for broken RT kernel spec file, part 1
    if [ "$EXTRA" = "rt" ]; then
	DeBug "EXTRA=$EXTRA"
	EXTRA=""
    fi

    # Workaround for broken RT kernel spec file, part 2
    if [ "$EXTRA" = "rt-vanilla" -o "$EXTRA" = "rt-debug" -o "$EXTRA" = "rt-trace" ]; then
	DeBug "Before EXTRA=$EXTRA"
	# if it is MRG-2.x RHEL6
	if $(rpm -qf /etc/redhat-release | grep -q "redhat-release-server-6"); then
	    if [ "${KVER::2}" = "3." ]; then
		if [ "${KVER::3}" = "3.0" ]; then
		    # MRG-2.1
		    EXTRA=$(echo $EXTRA | awk -F- '{print $2}')
		else
		    # MRG-2.2 or newer
		    EXTRA=".x86_64.$EXTRA"
		fi
	    else
		# MRG-2.0
		EXTRA=$(echo $EXTRA | awk -F- '{print $1 $2}')
	    fi
	else
	    # MRG-1.x on RHEL 5
	    EXTRA=$(echo $EXTRA | awk -F- '{print $2}')
	fi
	DeBug "After EXTRA=$EXTRA"
    fi

    # Workaround for UP kernel spec file
    if [ "$EXTRA" = "up" ]; then
	DeBug "EXTRA=$EXTRA"
	EXTRA=""
    fi

    echo "***** Attempting to switch boot kernel to ($VR$EXTRA) *****" | tee -a $OUTPUTFILE
    DeBug "Attempting to switch boot kernel to ($VR$EXTRA)"

    # if we have grubby, then let's use that
    if command -v grubby >/dev/null 2>&1; then
        SelectKernelGrubby $VR $EXTRA
        ret=$?
    else
        SelectKernelLegacy $VR $EXTRA
        ret=$?
    fi

    sync
    sleep 5
    DeBug "Exit SelectKernel"
    return $ret
}

# Bug 798577 - ppc64: grubby/new-kernel-pkg fails to install new kernel
# There seem to be 2 issues, one in anaconda, one in grubby
function WorkaroundBug798577 ()
{
    local VR=$1
    local EXTRA=$2
    local vmlinuz=$3

    DeBug "WorkaroundBug798577: start <$VR> <$EXTRA> <$vmlinuz>"
    uname -r | grep ppc64
    if [ $? -ne 0 ]; then
        DeBug "WorkaroundBug798577: this is not ppc64"
        return 0
    fi

    cat /etc/redhat-release | grep "release 7"
    if [ $? -ne 0 ]; then
        DeBug "WorkaroundBug798577: this is not RHEL7"
        return 0
    fi

    local initrd=$(rpm -ql $testkernbase.$kernarch | grep -e /initramfs-$VR | grep "$EXTRA" | awk '{print length"\t"$0}' | sort -n | cut -f2- | head -1)
    DeBug "WorkaroundBug798577: vmlinuz=$vmlinuz"
    DeBug "WorkaroundBug798577: initrd=$initrd"

    if [ -n "$vmlinuz" -a -n "$initrd" ]; then
        sed -i 's/image=vmlinu/image=\/vmlinu/' /etc/yaboot.conf
        DeBug "/sbin/new-kernel-pkg --package kernel --install $VR.ppc64 --initrdfile=$initrd"
        /sbin/new-kernel-pkg --package kernel --install $VR.ppc64 --initrdfile=$initrd

        grubby --set-default "$vmlinuz"
        cp -f /etc/yaboot.conf /boot/etc/yaboot.conf
        ybin -v

        default_vmlinuz=$(grubby --default-kernel)
        DeBug "WorkaroundBug798577: grubby --default-kernel: $default_vmlinuz"
    fi

    DeBug "WorkaroundBug798577: end"
}

function SelectKernelGrubby ()
{
    DeBug "Enter SelectKernelGrubby"
    VR=$1
    EXTRA=$2
    DeBug "VR=$VR EXTRA=$EXTRA"

    # We can only have one kernel on arm right now and grubby is busted
    # until bz 751608 is fixed.
    if [ "${ARCH}" = "armhfp" -o "${ARCH}" = "arm" ]; then
        return 0
    fi

    # match vmlinuz with $VR and $EXTRA, take first shortest match
    vmlinuz=$(rpm -ql $testkernbase.$kernarch | grep -e /vmlinu.-$VR | grep "$EXTRA" | awk '{print length"\t"$0}' | sort -n | cut -f2- | head -1)

    if [ -n "$vmlinuz" ]; then
        grubby --set-default "$vmlinuz"
        ret=$?
        DeBug "grubby --set-default $vmlinuz returned: $ret"
        default_vmlinuz=$(grubby --default-kernel)
        DeBug "grubby --default-kernel: $default_vmlinuz"
        LogBootloaderConfig

        if [ "$default_vmlinuz" == "$vmlinuz" ]; then
            DeBug "grubby reports correct kernel as default: $default_vmlinuz"
        else
            DeBug "grubby reports default kernel as: $default_vmlinuz, expected: $vmlinuz"
            DeBug "trying WorkaroundBug798577"
            WorkaroundBug798577 "$VR" "$EXTRA" "$vmlinuz"
            LogBootloaderConfig
        fi

        if [ $ret -eq 0 ]; then
            test "${ARCH}" = "s390" -o "${ARCH}" = "s390x" && zipl
        fi
        return $ret
    fi
    DeBug "ERROR: failed to find vmlinuz in rpm: $testkernbase.$kernarch"
    return 1
}

function SelectKernelLegacy ()
{
    DeBug "Enter SelectKernelLegacy"
    VR=$1
    EXTRA=$2
    DeBug "VR=$VR EXTRA=$EXTRA"

    grub_file=/boot/grub/grub.conf

    if [ -f $grub_file ]; then
	DeBug "Using: $grub_file"
	COUNT=0
	DEFAULT=undefined
	for i in $(grep '^title' $grub_file | sed -e 's/.*(\(.*\)).*/\1/' -e "s/\.$(uname -m)*.//g"); do
	    DeBug "COUNT=$COUNT VR=$VR EXTRA=$EXTRA i=$i"
	    if [ "$VR$EXTRA" = "$i" ]; then
		DEFAULT=$COUNT;
	    fi
	    COUNT=$(expr $COUNT + 1)
	done
	if [ $DEFAULT != "undefined" ]; then
	    DeBug "DEFAULT=$DEFAULT"
	    /bin/ed -s $grub_file <<EOF
/default/
d
i
default=$DEFAULT
.
w
q
EOF
	fi
	DeBug "$grub_file"
	cat $grub_file | tee -a $DEBUGLOG
    fi

    elilo_file=/boot/efi/efi/redhat/elilo.conf

    if [ -f $elilo_file ]; then
	DeBug "Using: $elilo_file"
	DEFAULT=$(grep -A 2 "image=vmlinuz-$VR$EXTRA$" $elilo_file | awk -F= '/label=/ {print $2}')
	DeBug "DEFAULT=$DEFAULT"
	if [ -n "$DEFAULT" ]; then
	    DeBug "DEFAULT=$DEFAULT"
	    /bin/ed -s $elilo_file <<EOF
/default/
d
i
default=$DEFAULT
.
w
q
EOF
	fi
	DeBug "$elilo_file"
	cat $elilo_file | tee -a $DEBUGLOG
    fi

    yaboot_file=/boot/etc/yaboot.conf
 
    if [ -f $yaboot_file ] ; then
	DeBug "Using: $yaboot_file"
	grep vmlinuz $yaboot_file
	if [ $? -eq 0 ] ; then
	    VM=z
	else
	    VM=x
	fi
	DeBug "VM=$VM"
	DEFAULT=$(grep -A 1 "image=/vmlinu$VM-$VR.*$EXTRA" $yaboot_file | awk -F= '/label=/ {print $2}')
	DeBug "DEFAULT=$DEFAULT"
	if [ -n "$DEFAULT" ] ; then
	    sed -i 's/label=linux/label=orig-linux/g' $yaboot_file
	    sed -i 's/label='$DEFAULT'/label=linux/g' $yaboot_file
	    DeBug "DEFAULT=$DEFAULT"
	    grep -q label=linux $yaboot_file
	    if [ $? -ne 0 ] ; then
		sed -i 's/label=orig-linux/label=linux/g' $yaboot_file
		DeBug "Reverted back to original kernel"
	    fi
	fi
	DeBug "$yaboot_file"
	cat $yaboot_file | tee -a $DEBUGLOG
    fi

    zipl_file=/etc/zipl.conf

    if [ -f $zipl_file ] ; then
	DeBug "Using: $zipl_file"
	DEFAULT=$(grep "image=/boot/vmlinuz-$VR.*$EXTRA" $zipl_file | awk -Fvmlinuz- '/vmlinuz/ {printf "%.15s\n",$2}')
	DeBug "DEFAULT=$DEFAULT"
	if [ -n "$DEFAULT" ] ; then
	    DeBug "$VR$EXTRA"
	    tag=$(grep "\[$DEFAULT\]" $zipl_file)
	    DeBug "tag=$tag"
	    if [ -z "$tag" ] ; then
		# This was added because BZ 426992 was fixed
		DEFAULT=$(grep "image=/boot/vmlinuz-$VR.*$EXTRA" $zipl_file | awk -Fvmlinuz- '/vmlinuz/ {printf "%s\n",$2}')
		DeBug "Second DEFAULT=$DEFAULT"
		tag=$(grep "\[$DEFAULT\]" $zipl_file)
		DeBug "Second tag=$tag"
		if [ -z "$tag" ] ; then
		    DeBug "Setting it back to default"
		    DEFAULT=linux
		fi
	    fi
	    /bin/ed -s $zipl_file <<EOF
/default=/
d
i
default=$DEFAULT
.
w
q
EOF
	    zipl
	fi
	DeBug "$zipl_file"
	cat $zipl_file | tee -a $DEBUGLOG
    fi
    return 0
}

function CheckKernel ()
{
    DeBug "Enter CheckKernel"
    KVER=$1
    KVAR=$2
    DeBug "Before KVER=$KVER KVAR=$KVAR"

    # Workaround for broken RT kernel spec file, part 1
    if [ "$KVAR" = "rt" ]; then
	DeBug "KVAR=$KVAR"
	KVAR=""
    fi

    # Workaround for broken RT kernel spec file, part 2
    if [ "$KVAR" = "rt-vanilla" -o "$KVAR" = "rt-debug" -o "$KVAR" = "rt-trace" ]; then
	DeBug "Before KVAR=$KVAR"
	# if it is MRG-2.x RHEL6
	if $(rpm -qf /etc/redhat-release | grep -q "redhat-release-server-6"); then
	    if [ "${KVER::2}" = "3." ]; then
		if [ "${KVER::3}" = "3.0" ]; then
		    # MRG-2.1
		    KVAR=$(echo $KVAR | awk -F- '{print $2}')
		else
		    # MRG-2.2 or newer
		    KVAR=".x86_64.$KVAR"
		fi
	    else
		# MRG-2.0
		KVAR=$(echo $KVAR | awk -F- '{print $1 $2}')
	    fi
	else
	    # MRG-1.x o RHEL 5
	    KVAR=$(echo $KVAR | awk -F- '{print $2}')
	fi
	DeBug "After KVAR=$KVAR"
    fi

    # Workaround for UP kernel spec file
    if [ "$KVAR" = "up" ]; then
	DeBug "KVAR=$KVAR"
	KVAR=""
    fi

    DeBug "After KVER=$KVER KVAR=$KVAR"
    if [ "$KVER$KVAR" != "$runkernel" ]; then
	DeBug "Requested kernel != Running kernel"
	DeBug "   $KVER$KVAR != $runkernel"
 	return 1
    else
	DeBug "Requested kernel = Running kernel"
	DeBug "   $KVER$KVAR = $runkernel"
	return 0
    fi
    DeBug "Exit CheckKernel"
}

function YumInstallKernel ()
{
    # Check and see if the kernel available for install from a repo
    # yum list available checks nvr, and will refuse to show older one by default
    # so let's use yum list all instead
    DeBug "Enter YumInstallPackage"
    echo "***** Install kernel via yum $testkernbase.$kernarch *****" | tee -a $OUTPUTFILE
    DeBug "Yum install $testkernbase.$kernarch"
    $yumcmd list all $testkernbase.$kernarch | grep -q $testkername.$kernarch
    if [ "$?" -eq "0" ]; then
        # Install the kernel from yum repo
	$yumcmd -y install $testkernbase.$kernarch
	if [ "$?" -ne "0" ]; then
	    echo "***** Yum returned an error while trying to install $testkernbase.$kernarch *****" | tee -a $OUTPUTFILE
	    DeBug "Exit YumInstallPackage FAIL 3 (YUM exited with a failure)"
	    return 3
	else
	    # Check to see if the kernel is now installed, using yum
	    $yumcmd list installed $testkernbase.$kernarch | grep -q installed
	    if [ "$?" -ne "0" ]; then
		# Double check, this time using rpm
		rpm -qa --queryformat '%{name}-%{version}-%{release}.%{arch}\n' | grep -q $testkernbase.$kernarch
		if [ "$?" -ne "0" ]; then
		    echo "***** Failed to find $testkernbase.$kernarch in installed *****" | tee -a $OUTPUTFILE
		    DeBug "Exit YumInstallPackage FAIL 4 (Thought we installed but can't find it)"
		    return 4
		fi
	    fi
	fi
	$yumcmd -y install $testkerndevel.$kernarch
    else
	DeBug "Exit YumInstallPackage FAIL 5 (Can't find kernel in repo)"
	return 5
    fi
    DeBug "Exit YumInstallPackage SUCCESS"
    return 0
}

function BrewInstallKernel ()
{
    # Is the kernel available for install from a BREW
    DeBug "Enter BrewInstallPackage"
    # install kernel-firmware first if there is one.
    if curl -s $httpbase/noarch/$testkernfirmware.noarch.rpm -o /dev/null -f; then
        kernfwprpm="$httpbase/noarch/$testkernfirmware.noarch.rpm"
    elif curl -s $archbase/noarch/$testkernfirmware.noarch.rpm -o /dev/null -f; then
        kernfwprpm="$archbase/noarch/$testkernfirmware.noarch.rpm"
    fi
    yum -y localinstall --nogpgcheck $kernfwprpm
    DeBug "$httpbase/$kernarch/$testkernbase.$kernarch.rpm"
    echo "***** Install kernel via BREW $testkernbase.$kernarch.rpm *****" | tee -a $OUTPUTFILE
    if curl -s $httpbase/$kernarch/$testkernbase.$kernarch.rpm -o /dev/null -f; then
       rpm -ivh $httpbase/$kernarch/$testkernbase.$kernarch.rpm
       if [ "$?" -ne "0" ]; then
          DeBug "Forcing the rpm command --force"
          rpm -ivh --force $httpbase/$kernarch/$testkernbase.$kernarch.rpm
          if [ "$?" -ne "0" ]; then
             DeBug "Exit BrewInstallPackage FAIL 6"
             return 6
          fi
       fi
       rpm -ivh $httpbase/$kernarch/$testkerndevel.$kernarch.rpm
       DeBug "Exit BrewInstallPackage SUCCESS"
    elif curl -s $archbase/$kernarch/$testkernbase.$kernarch.rpm -o /dev/null -f; then 
       DeBug "Installing the rpm from the kernelarchive."
       rpm -ivh $archbase/$kernarch/$testkernbase.$kernarch.rpm
       if [ "$?" -ne "0" ]; then
          DeBug "Forcing the rpm command --force"
          rpm -ivh --force $archbase/$kernarch/$testkernbase.$kernarch.rpm
          if [ "$?" -ne "0" ]; then
             DeBug "Exit BrewInstallPackage FAIL 6"
             return 6
          fi
       fi
       rpm -ivh $archbase/$kernarch/$testkerndevel.$kernarch.rpm
       DeBug "Exit BrewInstallPackage SUCCESS"
    else 
       DeBug "can't find the package in brew ... FAIL"
       return 1
    fi
    return 0
}

function YumUpgradeKernelHeaders ()
{
    KERNELHEADERS=kernel-headers-$testkernver-$testkernrel

    # Check and see if the kernel-headers are available for install from a repo
    # yum list available checks nvr, and will refuse to show older one by default,
    # so let's use yum list all instead
    DeBug "Enter YumUpgradeKernelHeaders"
    echo "***** Upgrade $KERNELHEADERS via yum *****" | tee -a $OUTPUTFILE
    DeBug "Yum upgrade $KERNELHEADERS"
    $yumcmd list all $KERNELHEADERS | grep -q $testkernver-$testkernrel
    if [ "$?" -eq "0" ]; then
	# Install the kernel-headers from yum repo
	$yumcmd -y upgrade $KERNELHEADERS
	if [ "$?" -ne "0" ]; then
	    echo "***** Yum returned an error while trying to upgrade $KERNELHEADERS *****" | tee -a $OUTPUTFILE
	    DeBug "Exit YumUpgradeKernelHeaders FAIL 3 (YUM exited with a failure)"
	    return 3
	else
	    # Check to see if the kernel-headers is now installed, using yum
	    $yumcmd list installed $KERNELHEADERS | grep -q installed
	    if [ "$?" -ne "0" ]; then
		# Double check, this time using rpm
		rpm -qa --queryformat '%{name}-%{version}-%{release}\n' | grep -q $KERNELHEADERS
		if [ "$?" -ne "0" ]; then
		    echo "***** Failed to find $KERNELHEADERS in installed *****" | tee -a $OUTPUTFILE
		    DeBug "Exit YumInstallPackage FAIL 4 (Thought we installed but can't find it)"
		    return 4
		fi
	    fi
	fi
    else
	DeBug "Exit YumUpgradeKernelHeaders FAIL 5 (Can't find kernel in repo)"
	return 5
    fi
    DeBug "Exit YumUpgradeKernelHeaders SUCCESS"
    return 0
}

function DepmodChk ()
{
    DeBug "Enter DepmodChk"
    DEPCHKFILE=`mktemp /tmp/tmp.XXXXXX`
    DeBug "DEPCHKFILE=$DEPCHKFILE"
    /sbin/depmod -ae -F /boot/System.map-`uname -r` `uname -r` > $DEPCHKFILE 2>&1
    if [ -s $DEPCHKFILE ] ; then
	DeBug "$DEPCHKFILE > 0"
        total=`cat $DEPCHKFILE | wc -l`
        OUTPUTFILE=`mktemp /tmp/tmp.XXXXXX`
        echo "***** List of Warnings/Errors reported by depmod *****" | tee -a $OUTPUTFILE
        cat $DEPCHKFILE | tee -a $OUTPUTFILE
        echo "***** End of list *****" | tee -a $OUTPUTFILE
        report_result $TEST/depmod FAIL $total
    fi
    DeBug "Exit DepmodChk"
}

function NukeRepo ()
{
    DeBug "Enter NukeRepo"
    if [ -e /etc/yum.repos.d/rhel-beta.repo ] ; then
	DeBug "beta repo existed, moving it to tmp"
	mv -f /etc/yum.repos.d/rhel-beta.repo /tmp
	yum clean all
    fi
    DeBug "Exit NukeRepo"
}

fix_bootif ()
{
    local macaddr=${1}
    local IFS='-'
    macaddr=$(for i in ${macaddr} ; do echo -n $i:; done)
    macaddr=${macaddr%:}
    # strip hardware type field from pxelinux
    [ -n "${macaddr%??:??:??:??:??:??}" ] && macaddr=${macaddr#??:}
    # return macaddr with lowercase alpha characters expected by udev
    echo $macaddr | sed 'y/ABCDEF/abcdef/'
}

function workaround_bug905918 ()
{
    echo "Enter workaround_bug905918" >> $OUTPUTFILE

    cat /etc/redhat-release | grep "release 7"
    if [ $? -ne 0 ]; then
        echo "This is not RHEL7, exiting"
        return
    fi

    local cmdline=`grep "Command line" /var/log/anaconda/syslog | head -1`
    local BOOTIF=`echo $cmdline | grep -o "BOOTIF=[0-9a-zA-Z:-]\+" | sed 's/BOOTIF=//'`

    # command line
    echo $cmdline | grep "ip=" 2>&1 >/dev/null
    if [ $? -eq 0 ]; then
        echo "ip= present on command line, exiting" >> $OUTPUTFILE
        return
    fi

    # kickstart
    if [ ! -f /root/anaconda-ks.cfg ]; then
        echo "Could not find /root/anaconda-ks.cfg, exiting" >> $OUTPUTFILE
        return
    fi
    grep "^network" /root/anaconda-ks.cfg | grep device
    if [ $? -eq 0 ]; then
        echo "device= present in kickstart network option, exiting" >> $OUTPUTFILE
        return
    fi

    if [ -z "$BOOTIF" ]; then
        echo "Could not find BOOTIF in /var/log/anaconda/syslog" >> $OUTPUTFILE
        return
    fi
    BOOTIF=$(fix_bootif "$BOOTIF")
    echo "Fixed up BOOTIF: $BOOTIF" >> $OUTPUTFILE

    ip a s up | grep $BOOTIF 2>&1 >/dev/null
    if [ $? -ne 0 ]; then
        echo "Could not find any interface with this MAC, exiting" >> $OUTPUTFILE
        return
    fi

    local ifcfg_mask="/etc/sysconfig/network-scripts/ifcfg-"
    for ifcfg in $ifcfg_mask*; do
        local ifc=`echo $ifcfg | sed "s|$ifcfg_mask||"`
        if [ "$ifc" == "lo" ]; then
            continue
        fi
        grep -l -i "$BOOTIF" $ifcfg 2>&1 >/dev/null
        if [ $? -ne 0 ]; then
            echo "Setting ONBOOT=no for $ifcfg" >> $OUTPUTFILE
            sed -i 's/ONBOOT=.*/ONBOOT=no/' $ifcfg
            restorecon $ifcfg
            echo "Turning off interface (ip link set down): $ifc" >> $OUTPUTFILE
            ip link set down $ifc
        fi
    done
    #wait for NetworkManager to process new config
    sleep 60
    date >> $OUTPUTFILE
    ls -la /etc/resolv.conf >> $OUTPUTFILE
    echo "-- /etc/resolv.conf --"
    cat /etc/resolv.conf >> $OUTPUTFILE
    echo "-- END of /etc/resolv.conf --"
    echo "Exiting workaround_bug905918" >> $OUTPUTFILE
}

function Main ()
{
    DeBug "Enter Main"

    # Seed the log with an initial entry
    echo "***** Start of kernel install test *****" | tee -a $OUTPUTFILE

    # RHEL6-Beta repo workaround
    NukeRepo

    # Enable the Hypervisor and Console logging for xen tests
    XendLogging

    # workaround Bug 905918 - set ONBOOT=yes only for interfaces used during installation
    workaround_bug905918

    # Check to see if the kernel we want to test is already running
    CheckKernel $KERNELARGVERSION $KERNELARGVARIANT 
    if [ "$?" = "1" ]; then
        # Check to see if the kernel we want to test is already installed
	rpm -qa --queryformat '%{name}-%{version}-%{release}.%{arch}\n' | grep -q $testkernbase.$kernarch
	if [ "$?" -ne "0" ]; then
            # Install kernel variant from yum repo
	    YumInstallKernel
	    if [ "$?" -ne "0" ]; then
		echo "***** Could not install from yum repo trying rpm -ivh from BREW *****" | tee -a $OUTPUTFILE
		# Install from yum failed lets try direct from brew
		BrewInstallKernel
		if [ "$?" -ne "0" ]; then
		    RprtRslt $TEST/BrewInstallkernel FAIL $?
		    RHTSAbort
		fi
	    fi
	fi
        # Lets make it our default boot kernel the kernel we want to test
	SelectKernel $KERNELARGVERSION $KERNELARGVARIANT
	if [ "$?" -ne "0" ]; then
	    RprtRslt $TEST/SelectKernel FAIL $?
	else
            # Now that the kernel is our default... Let's reboot
	    echo "***** End of kernel install test *****" | tee -a $OUTPUTFILE
	    if [ -f $OUTPUTDIR/boot.$kernbase ]; then
		SubmitLog $OUTPUTDIR/boot.$kernbase
	    fi
	    SubmitLog $DEBUGLOG
	    RprtRslt $TEST/rhts-reboot PASS 0
	    date --date="$(date --utc)" +%s > /mnt/testarea/kernelinstall_reboottime.log
	    rhts-reboot
	fi
    else
	echo "***** The running kernel is the kernel we want to test *****" | tee -a $OUTPUTFILE
	echo "***** End of kernel install test *****" | tee -a $OUTPUTFILE
	if [ -f $OUTPUTDIR/boot.$kernbase ]; then
	    SubmitLog $OUTPUTDIR/boot.$kernbase
	fi
	RprtRslt $TEST/$kernbase PASS $REBOOTCOUNT
	DepmodChk
	SysReport
	exit 0
    fi
}

# Record the test version in the debuglog
testver=$(rpm -qf $0)
DeBug "$testver"

# Current kernel variables
runkernel=$K_RUNNING_VR
kernbase=$(rpm -q --queryformat '%{name}-%{version}-%{release}\n' -qf /boot/config-$(uname -r))
kername=$(rpm -q --queryformat '%{name}\n' -qf /boot/config-$(uname -r))
kernver=$(rpm -q --queryformat '%{version}\n' -qf /boot/config-$(uname -r))
kernrel=$(rpm -q --queryformat '%{release}\n' -qf /boot/config-$(uname -r))
kernarch=$(rpm -q --queryformat '%{arch}\n' -qf /boot/config-$(uname -r))
kernvariant=$(uname -r | sed -e "s/${kernver}-${kernrel}//g" -e "s/\.$(uname -m).//g")
kerndevel=$(rpm -q --queryformat '%{name}-devel-%{version}-%{release}\n' -qf /boot/config-$(uname -r))
DeBug "Running kernel variables"
DeBug "RUNNINGKERNEL=$runkernel"
DeBug "1=$kernbase 2=$kername 3=$kernver 4=$kernrel 5=$kernarch 6=$kernvariant 7=$kerndevel"

# Test ARGS that are defined
DeBug "TEST ARGS (Environment variables)"
DeBug "1=$KERNELARGNAME 2=$KERNELARGVARIANT 3=$KERNELARGVERSION"

# New kernel variables
if [ "$KERNELARGVARIANT" == "up" ]; then
    DeBug "Running up variant, or newer smp"
    testkernbase=$KERNELARGNAME-$KERNELARGVERSION
    testkername=$KERNELARGNAME
    testkernver=$(echo $KERNELARGVERSION | awk -F- '{print $1}')
    testkernrel=$(echo $KERNELARGVERSION | awk -F- '{print $2}')
    testkerndevel=$KERNELARGNAME-devel-$KERNELARGVERSION
    testkerneluname=$KERNELARGVERSION
    DeBug "Test kernel variables"
    DeBug "1=$testkernbase 2=$testkername 3=$testkernver 4=$testkernrel 5=$testkerndevel"
else
    testkernbase=$KERNELARGNAME-$KERNELARGVARIANT-$KERNELARGVERSION
    testkername=$KERNELARGNAME-$KERNELARGVARIANT
    testkernver=$(echo $KERNELARGVERSION | awk -F- '{print $1}')
    testkernrel=$(echo $KERNELARGVERSION | awk -F- '{print $2}')
    testkernvariant=$KERNELARGVARIANT
    testkerndevel=$KERNELARGNAME-$KERNELARGVARIANT-devel-$KERNELARGVERSION
    testkerneluname=$KERNELARGVERSION$KERNELARGVARIANT
    DeBug "Test kernel variables, in the else statement"
    DeBug "1=$testkernbase 2=$testkername 3=$testkernver 4=$testkernrel 5=$testkernvariant 6=$testkerndevel"
fi
testkernfirmware=kernel-firmware-$KERNELARGVERSION

# Determine if we are on RHEL5 distro 
DeBug "Setting the default yum command"
yumcmd="yum"

DeBug "Setting the default CheckKernel Options"
OPTIONSCheckKernel="$KERNELARGVERSION $KERNELARGVARIANT"

RHEL5TREE=$(rpm -qf /etc/redhat-release | grep -q "redhat-release-5")
if [ "$RHE5TREE" == "0" ] ; then
    DeBug "Running on RHEL5 Distribution"
    DeBug "Override yum command"
    yumcmd="yum --noplugins"
fi

RHEL6TREE=$(cat /etc/redhat-release | grep -q "Santiago")
if [ "$RHEL6TREE" == "0" ] ; then
    DeBug "Running on RHEL6 Distribution"
    DeBug "Override yum command"
    yumcmd="yum --noplugins"
fi

# Generic test variables
httpbase=http://download.lab.bos.redhat.com/brewroot/packages/$KERNELARGNAME/$testkernver/$testkernrel
archbase=http://download.lab.bos.redhat.com/brewroot/vol/kernelarchive/packages/$KERNELARGNAME/$testkernver/$testkernrel
if [ "$KERNELARGVARIANT" = "rt-vanilla" -o "$KERNELARGVARIANT" = "rt-debug" -o "$KERNELARGVARIANT" = "rt-trace" -o "$KERNELARGVARIANT" = "rt" ]; then
    DeBug "Setting up HTTPBASE and Firmware for RT kernels"
    httpbase=http://download.lab.bos.redhat.com/brewroot/packages/$KERNELARGNAME-rt/$testkernver/$testkernrel
    testkernfirmware=kernel-rt-firmware-$KERNELARGVERSION
fi

if [ -z "$OUTPUTDIR" ]; then
    OUTPUTDIR=/mnt/testarea
fi

# Record lspci -xxx -vv
if [ -x /sbin/lspci ]; then
    /sbin/lspci -xxx -vv > $OUTPUTDIR/lspci.$kernbase
fi

# Record the boot messages
/bin/dmesg > $OUTPUTDIR/boot.$kernbase
if [ ! -s $OUTPUTDIR/boot.$kernbase ]; then
    # Workaround for /distribution/install zeroing out the dmesg file
    cp $OUTPUTDIR/boot.messages $OUTPUTDIR/boot.$kernbase
fi

if [ -z "$KERNELARGNAME" -o -z "$KERNELARGVARIANT" -o -z "$KERNELARGVERSION" ]; then
    echo "***** Test argument(s) are empty! Can't continue. *****" | tee -a $OUTPUTFILE
    DeBug "name=$KERNELARGNAME variant=$KERNELARGVARIANT version=$KERNELARGVERSION"
    RprtRslt $TEST/$kernbase FAIL 1
    exit 0
else 
    if [ "$REBOOTCOUNT" != "1" ]; then
	Main
    else
	if [ -f $OUTPUTDIR/boot.$kernbase ]; then
	    SubmitLog $OUTPUTDIR/boot.$kernbase
	fi
	DeBug "Running CheckKernel $OPTIONSCheckKernel"
	CheckKernel $OPTIONSCheckKernel
	if [ "$?" = "1" ]; then 
	    DeBug "After reboot we are still not running the correct kernel"
	    RprtRslt $TEST/$kernbase FAIL $REBOOTCOUNT
	    RHTSAbort
	else
	    DeBug "After reboot we are running the correct kernel"
	    YumUpgradeKernelHeaders
	    REBOOT_TIME=$(cat /mnt/testarea/kernelinstall_reboottime.log)
	    DIFF=$(expr ${CUR_TIME} - ${REBOOT_TIME})
	    if [[ ${DIFF} -gt 480 ]]; then
	         DeBug "rhts-reboot took ${DIFF} seconds..."
	         RprtRslt $TEST/${kernbase}_boot WARN $DIFF
	    fi
	    RprtRslt $TEST/$kernbase PASS $DIFF
	    DepmodChk
	    DiffDmesg
	    if [ -x /sbin/lspci ]; then
	        DiffLspci
	    fi
	    SysReport	
	fi
	SubmitLog $DEBUGLOG
    fi
fi