ctdb-tests: Fix and extend read-only records test

This test currently counts the number of read-only-enabled databases
and expects there to only be 1.  It fails when there are existing
databases with read-only already enabled.  Instead, check just the
test database.

Clean up the test by adding some functions to check for precisely the
read-only flags that should be set on a node after each operation.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

1 files changed, 123 insertions, 93 deletions

diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh
index 80a6adcd60e..6cd2cce6128 100755
--- a/ctdb/tests/simple/75_readonly_records_basic.sh
+++ b/ctdb/tests/simple/75_readonly_records_basic.sh
@@ -3,11 +3,11 @@
 test_info()
 {
     cat <<EOF
-Readonly records can be activated at runtime using a ctdb command.
-If readonly records are not activated, then any attempt to fetch a readonly
+Read-only records can be activated at runtime using a ctdb command.
+If read-only records are not activated, then any attempt to fetch a read-only
 copy should be automatically upgraded to a read-write fetch_lock().
 
-If readonly delegations are present, then any attempt to aquire a read-write
+If read-only delegations are present, then any attempt to aquire a read-write
 fetch_lock will trigger all delegations to be revoked before the fetch lock
 completes.
 
@@ -20,21 +20,16 @@ Steps:
 
 1. Verify that the status on all of the ctdb nodes is 'OK'.
 2. create a test database and some records
-3. try to fetch readonly records, this should not result in any delegations
-4. activate readonly support
-5. try to fetch readonly records, this should result in delegations
+3. try to fetch read-only records, this should not result in any delegations
+4. activate read-only support
+5. try to fetch read-only records, this should result in delegations
 6. do a fetchlock  and the delegations should be revoked
-7. try to fetch readonly records, this should result in delegations
+7. try to fetch read-only records, this should result in delegations
 8. do a recovery  and the delegations should be revoked
 
 Expected results:
 
-3. No delegations created when db is not in readonly mode
-4. It is possible to activate readonly support for a database
-5. Delegations should be created
-6. Delegations should be revoked
-8. Delegations should be revoked
-
+Delegations should be created and revoked as above
 
 EOF
 }
@@ -50,114 +45,149 @@ cluster_is_healthy
 # Reset configuration
 ctdb_restart_when_done
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
-
-
-# create a temporary database to test with
-echo create test database test.tdb
-try_command_on_node 0 $CTDB attach test.tdb
-
-
-# create some records
-try_command_on_node all $CTDB_TEST_WRAPPER ctdb_update_record
-
-#
-# 3
-# try readonly requests
-echo Try some readonly fetches, these should all be upgraded to full fetchlocks
-try_command_on_node 0,1,2 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null"
+######################################################################
 
-# no delegations should have been created
-numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep READONLY | wc -l`
-[ "$numreadonly" != "0" ] && {
-    echo "BAD: readonly delegations were created, but the feature is not activated on the database"
-    exit 1
+# Confirm that no nodes have databases with read-only delegations
+check_no_readonly ()
+{
+    try_command_on_node all $CTDB cattdb $testdb
+    local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS"
+    local numreadonly=$(grep -c -E "$ro_flags" <<<"$out") || true
+    if [ $numreadonly -eq 0 ] ; then
+	echo "GOOD: no read-only delegations"
+    else
+	echo "BAD: there are read-only delegations"
+	echo "$out"
+	exit 1
+    fi
 }
 
+# Check that the test record has the correct read-only flags on the
+# given nodes.  The first node is the dmaster, which should know there
+# are delegations but should not be flagged as having a read-only
+# copy.  Subsequent nodes should have a read-only copy but not know
+# about any (other) delegations.
+check_readonly ()
+{
+    local dmaster="$1" ; shift
+    local others="$*"
+
+    local count
+
+    try_command_on_node $dmaster $CTDB cattdb $testdb
+    count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+    if [ $count -eq 1 ] ; then
+	echo "GOOD: dmaster ${dmaster} has read-only delegations"
+    else
+	echo "BAD: dmaster ${dmaster} has no read-only delegations"
+	echo "$out"
+	exit 1
+    fi
+    count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+    if [ $count -ne 0 ] ; then
+	echo "BAD: dmaster ${dmaster} has a read-only copy"
+	echo "$out"
+	exit 1
+    fi
+
+    local o
+    for o in $others ; do
+	try_command_on_node $o $CTDB cattdb $testdb
+	count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+	if [ $count -eq 1 ] ; then
+	    echo "GOOD: node ${o} has a read-only copy"
+	else
+	    echo "BAD: node ${o} has no read-only copy"
+	    echo "$out"
+	    exit 1
+	fi
+	count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+	if [ $count -ne 0 ] ; then
+	    echo "BAD: other node ${o} has read-only delegations"
+	    echo "$out"
+	    exit 1
+	fi
+    done
+}
 
-#
-# 4
-#
+######################################################################
 
-echo Activating ReadOnly record support for test.tdb ...
-# activate readonly support
-try_command_on_node all $CTDB setdbreadonly test.tdb
-numreadonly=`try_command_on_node -v 0 $CTDB getdbmap | grep READONLY | wc -l`
-[ "$numreadonly" != "1" ] && {
-    echo BAD: could not activate readonly support for the test database
-    exit 1
-}
+echo "Get list of nodes..."
+try_command_on_node any $CTDB -Y listnodes
+all_nodes=$(awk -F: '{print $2}' <<<"$out")
 
+######################################################################
 
+testdb="test.tdb"
+echo "Create test database \"${testdb}\""
+try_command_on_node 0 $CTDB attach $testdb
 
-#
-# 5
-#
+echo "Create some records..."
+try_command_on_node all $CTDB_TEST_WRAPPER ctdb_update_record
 
-echo Create some readonly delegations ...
-# fetch record to node 0 and make it dmaster
-try_command_on_node 1 $CTDB_TEST_WRAPPER ctdb_update_record
+######################################################################
 
-# fetch readonly to node 1
-try_command_on_node -v 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null"
+echo "Try some readonly fetches, these should all be upgraded to full fetchlocks..."
+try_command_on_node all $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null"
 
-numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l`
-[ "$numreadonly" != "2" ] && {
-    echo BAD: could not create readonly delegation
-    exit 1
-}
+check_no_readonly
 
+######################################################################
 
+echo "Activate read-only record support for \"$testdb\"..."
+try_command_on_node all $CTDB setdbreadonly $testdb
 
+# Database should be tagged as READONLY
+try_command_on_node 0 $CTDB getdbmap
+db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' <<<"$out")
+if grep -q "READONLY" <<<"$db_details" ; then
+    echo "GOOD: read-only record support is enabled"
+else
+    echo "BAD: could not activate read-only support"
+    echo "$db_details"
+    exit 1
+fi
 
-#
-# 6
-#
+######################################################################
 
-echo verify that a fetchlock will revoke the delegations ...
-# fetch record to node 0 and make it dmaster
+echo "Create 1 read-only delegation ..."
+# dmaster=1
 try_command_on_node 1 $CTDB_TEST_WRAPPER ctdb_update_record
 
-numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l`
-[ "$numreadonly" != "0" ] && {
-    echo BAD: fetchlock did not revoke delegations
-    exit 1
-}
+# Fetch read-only to node 0
+try_command_on_node 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null"
 
+check_readonly 1 0
 
-#
-# 7
-#
+######################################################################
 
-echo Create some readonly delegations ...
-# fetch record to node 0 and make it dmaster
+echo "Verify that a fetchlock revokes read-only delegations..."
+# Node 1 becomes dmaster
 try_command_on_node 1 $CTDB_TEST_WRAPPER ctdb_update_record
 
-# fetch readonly to node 1
-try_command_on_node -v 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null"
+check_no_readonly
 
-numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l`
-[ "$numreadonly" != "2" ] && {
-    echo BAD: could not create readonly delegation
-    exit 1
-}
+######################################################################
 
+echo "Create more read-only delegations..."
+dmaster=1
+try_command_on_node $dmaster $CTDB_TEST_WRAPPER ctdb_update_record
 
+others=""
+for n in $all_nodes ; do
+    if [ "$n" != "$dmaster" ] ; then
+	# Fetch read-only copy to this node
+	try_command_on_node $n \
+	    $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null"
+	others="${others} ${n}"
+    fi
+done
 
+check_readonly $dmaster $others
 
-#
-# 8
-#
+######################################################################
 
-echo verify that a recovery will revoke the delegations ...
+echo "Verify that a recovery will revoke the delegations..."
 try_command_on_node 0 $CTDB recover
 
-numreadonly=`try_command_on_node -v all $CTDB cattdb test.tdb | grep RO_HAVE | wc -l`
-[ "$numreadonly" != "0" ] && {
-    echo BAD: recovery did not revoke delegations
-    exit 1
-}
-
-echo OK. test completed successfully
-exit 0
+check_no_readonly