Merge remote branch 'martins/eventscripts'

(This used to be ctdb commit bb008c01989ebb173a3f095ebd2f90ab54f9da91)

4 files changed, 223 insertions, 12 deletions

diff --git a/ctdb/config/events.d/00.ctdb b/ctdb/config/events.d/00.ctdb
index 31f8fdc79b..321ba9c708 100755
--- a/ctdb/config/events.d/00.ctdb
+++ b/ctdb/config/events.d/00.ctdb
@@ -63,6 +63,11 @@ case "$1" in
 	update_config_from_tdb &
 	;;
     monitor)
+	# Inherit the debug level from ctdbd on each monitor run.  If
+	# there's a more urgent need then override CTDB_CURRENT_DEBUGLEVEL
+	# using a file in $CTDB_BASE/rc.local.d/.
+        ctdb_set_current_debuglevel create
+
 	# We should never enter swap, so SwapTotal == SwapFree.
 	[ "$CTDB_CHECK_SWAP_IS_NOT_USED" = "yes" ] && {
 	    if [ -n "`grep '^Swap\(Total\|Free\)' /proc/meminfo | uniq -s 10 -u`" ]; then
diff --git a/ctdb/config/events.d/50.samba b/ctdb/config/events.d/50.samba
index 1c6822714f..bf143bf26e 100755
--- a/ctdb/config/events.d/50.samba
+++ b/ctdb/config/events.d/50.samba
@@ -115,8 +115,6 @@ service_stop ()
 	    check_ctdb_manages_winbind ; then
 		service "$CTDB_SERVICE_WINBIND" stop
 	fi
-
-	return 0
 }
 
 service_reconfigure ()
diff --git a/ctdb/config/events.d/60.nfs b/ctdb/config/events.d/60.nfs
index 19a9ea81a1..87a91df5b5 100755
--- a/ctdb/config/events.d/60.nfs
+++ b/ctdb/config/events.d/60.nfs
@@ -9,7 +9,6 @@ service_start ()
     startstop_nfs stop
     startstop_nfs start
     set_proc "sys/net/ipv4/tcp_tw_recycle" 1
-    touch "$service_state_dir/update-trigger"
 }
 service_stop ()
 {
@@ -32,6 +31,14 @@ loadconfig
 
 ctdb_setup_service_state_dir
 
+statd_update_trigger="$service_state_dir/update-trigger"
+# We want this file to always exist.  The corner case is when
+# auto-start/stop is switched off, NFS is added as a managed service
+# some time after ctdbd is started and someone else starts the NFS
+# service for us.  In this case this file might not otherwise exist
+# when we get to a monitor event.
+touch "$statd_update_trigger"
+
 ctdb_start_stop_service
 
 is_ctdb_managed_service || exit 0
@@ -105,10 +112,10 @@ case "$1" in
 
 	# once every 600 seconds, update the statd state database for which
 	# clients need notifications
-	LAST_UPDATE=`stat --printf="%Y" "$service_state_dir/update-trigger"`
+	LAST_UPDATE=`stat --printf="%Y" "$statd_update_trigger"`
 	CURRENT_TIME=`date +"%s"`
 	[ $CURRENT_TIME -ge $(($LAST_UPDATE + 600)) ] && {
-	    touch "$service_state_dir/update-trigger"
+	    touch "$statd_update_trigger"
 	    $CTDB_BASE/statd-callout updatelocal &
 	    $CTDB_BASE/statd-callout updateremote &
 	}
diff --git a/ctdb/config/functions b/ctdb/config/functions
index b04965281d..f65690359b 100755
--- a/ctdb/config/functions
+++ b/ctdb/config/functions
@@ -38,6 +38,51 @@ loadconfig () {
 }
 
 ##############################################################
+# make sure CTDB_CURRENT_DEBUGLEVEL is set to the desired debug level
+# (integer)
+#
+# If it is already set then do nothing, since it might have been set
+# via a file in rc.local.d/.  If it is not set then set it by sourcing
+# /var/ctdb/eventscript_debuglevel. If this file does not exist then
+# create it using output from "ctdb getdebug".  If the option 1st arg
+# is "create" then don't source an existing file but create a new one
+# instead - this is useful for creating the file just once in each
+# event run in 00.ctdb.  If there's a problem getting the debug level
+# from ctdb then it is silently set to 0 - no use spamming logs if our
+# debug code is broken...
+ctdb_set_current_debuglevel ()
+{
+    [ -z "$CTDB_CURRENT_DEBUGLEVEL" ] || return 0
+
+    _f="$CTDB_VARDIR/eventscript_debuglevel"
+
+    if [ "$1" = "create" -o ! -r "$_f" ] ; then
+	_t=$(ctdb getdebug -Y 2>/dev/null)
+	# get last field of output
+	_t="${_t%:}"
+	_t="${_t##*:}"
+	# Defaults to 0
+	echo "export CTDB_CURRENT_DEBUGLEVEL=\"${_t:-0}\"" >"$_f"
+    fi
+
+    . "$_f"
+}
+
+debug ()
+{
+    if [ $CTDB_CURRENT_DEBUGLEVEL -ge 4 ] ; then
+	# If there are arguments then echo them.  Otherwise expect to
+	# use stdin, which allows us to pass lots of debug using a
+	# here document.
+	if [ -n "$1" ] ; then
+	    echo "DEBUG: $*"
+	elif ! tty -s ; then
+	    sed -e 's@^@DEBUG: @'
+	fi
+    fi
+}
+
+##############################################################
 # determine on what type of system (init style) we are running
 detect_init_style() {
     # only do detection if not already set:
@@ -279,16 +324,166 @@ ctdb_check_directories() {
 # check a set of tcp ports
 # usage: ctdb_check_tcp_ports <ports...>
 ######################################################
-ctdb_check_tcp_ports() {
 
-    for p ; do
-	if ! netstat -a -t -n | grep -q "0\.0\.0\.0:$p .*LISTEN" ; then
-            if ! netstat -a -t -n | grep -q ":::$p .*LISTEN" ; then
-		echo "ERROR: $service_name tcp port $p is not responding"
+# This flag file is created when a service is initially started.  It
+# is deleted the first time TCP port checks for that service succeed.
+# Until then ctdb_check_tcp_ports() prints a more subtle "error"
+# message if a port check fails.
+_ctdb_check_tcp_common ()
+{
+    _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
+}
+
+ctdb_check_tcp_init ()
+{
+    _ctdb_check_tcp_common
+    mkdir -p "${_ctdb_service_started_file%/*}" # dirname
+    touch "$_ctdb_service_started_file"
+}
+
+ctdb_check_tcp_ports()
+{
+    if [ -z "$1" ] ; then
+	echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
+	exit 1
+    fi
+
+    # Set default value for CTDB_TCP_PORT_CHECKS if unset.
+    # If any of these defaults are unsupported then this variable can
+    # be overridden in /etc/sysconfig/ctdb or via a file in
+    # /etc/ctdb/rc.local.d/.
+    : ${CTDB_TCP_PORT_CHECKERS:=ctdb nmap netstat}
+
+    for _c in $CTDB_TCP_PORT_CHECKERS ; do
+	ctdb_check_tcp_ports_$_c "$@"
+	case "$?" in
+	    0)
+		rm -f "$_ctdb_service_started_file"
+		return 0
+		;;
+	    1)
+		_ctdb_check_tcp_common
+		if [ ! -f "$_ctdb_service_started_file" ] ; then
+		    echo "ERROR: $service_name tcp port $_p is not responding"
+		    debug <<EOF
+$ctdb_check_tcp_ports_debug
+EOF
+		else
+		    echo "INFO: $service_name tcp port $_p is not responding"
+		fi
+
 		return 1
-            fi
-	fi
+		;;
+	    127)
+		debug <<EOF
+ctdb_check_ports - checker $_c not implemented
+output from checker was:
+$ctdb_check_tcp_ports_debug
+EOF
+		;;
+	    *)
+		
+	esac
+    done
+
+    echo "INTERNAL ERROR: ctdb_check_ports - no working checkers in CTDB_TCP_PORT_CHECKERS=\"$CTDB_TCP_PORT_CHECKERS\""
+
+    return 127
+}
+
+ctdb_check_tcp_ports_netstat ()
+{
+    _cmd='netstat -l -t -n'
+    _ns=$($_cmd 2>&1)
+    if [ $? -eq 127 ] ; then
+	# netstat probably not installed - unlikely?
+	ctdb_check_tcp_ports_debug="$_ns"
+	return 127
+    fi
+
+    for _p ; do  # process each function argument (port)
+	for _a in '0\.0\.0\.0' '::' ; do
+	    _pat="[[:space:]]${_a}:${_p}[[:space:]]+[^[:space:]]+[[:space:]]+LISTEN"
+	    if echo "$_ns" | grep -E -q "$_pat" ; then
+		# We matched the port, so process next port
+		continue 2
+	    fi
+	done
+
+	# We didn't match the port, so flag an error.
+	ctdb_check_tcp_ports_debug="$_cmd shows this output:
+$_ns"
+	return 1
+    done
+
+    return 0
+}
+
+ctdb_check_tcp_ports_nmap ()
+{
+    # nmap wants a comma-separated list of ports
+    _ports=""
+    for _p ; do
+	_ports="${_ports}${_ports:+,}${_p}"
+    done
+
+    _cmd="nmap -n -oG - -PS 127.0.0.1 -p $_ports"
+
+    _nmap_out=$($_cmd 2>&1)
+    if [ $? -eq 127 ] ; then
+	# nmap probably not installed
+	ctdb_check_tcp_ports_debug="$_nmap_out"
+	return 127
+    fi
+
+    # get the port-related output
+    _port_info=$(echo "$_nmap_out" | sed -n -r -e 's@^.*Ports:[[:space:]]@@p')
+
+    for _p ; do
+	# looking for something like this:
+	#  445/open/tcp//microsoft-ds///
+	# possibly followed by a comma
+	_t="$_p/open/tcp//"
+	case "$_port_info" in
+	    # The info we're after must be either at the beginning of
+	    # the string or it must follow a space.
+            $_t*|*\ $_t*) : ;;
+	    *)
+		# Nope, flag an error...
+		ctdb_check_tcp_ports_debug="$_cmd shows this output:
+$_nmap_out"
+		return 1
+	esac
+    done
+
+    return 0
+}
+
+# Use the new "ctdb checktcpport" command to check the port.
+# This is very cheap.
+ctdb_check_tcp_ports_ctdb ()
+{
+    for _p ; do  # process each function argument (port)
+	_cmd="ctdb checktcpport $_p"
+	_out=$($_cmd 2>&1)
+	case "$?" in
+	    0)
+		ctdb_check_tcp_ports_debug="\"$_cmd\" was able to bind to port"
+		return 1
+		;;
+	    98)
+		# Couldn't bind, something already listening, next port...
+		continue
+		;;
+	    *)
+		ctdb_check_tcp_ports_debug="$_cmd (exited with $?) with output:
+$_out"
+		# assume not implemented
+		return 127
+	esac
     done
+
+    return 0
 }
 
 ######################################################
@@ -906,6 +1101,7 @@ ctdb_service_start ()
     service_start "$@" || return $?
 
     ctdb_counter_init "$@"
+    ctdb_check_tcp_init
 }
 
 ctdb_service_stop ()
@@ -1094,6 +1290,11 @@ update_tickles ()
 	done
 }
 
+# We'll call this here to ensure $CTDB_CURRENT_DEBUGLEVEL is set.
+# This gives us a chance to override the debug level using a file in
+# $CTDB_BASE/rc.local.d/.
+ctdb_set_current_debuglevel
+
 script_name="${0##*/}"       # basename
 service_name="$script_name"  # default is just the script name
 service_fail_limit=1