ctdb-tests/integration: Neaten up some of the persistent database tests

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Michael Adam <obnox@samba.org>

4 files changed, 171 insertions, 150 deletions

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 7dbccbcbdef..4f0f68b6263 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -967,7 +967,43 @@ nfs_test_cleanup ()
     onnode -q $test_node rmdir "$nfs_test_dir"
 }
 
+#######################################
+
+# $1: pnn, $2: DB name
+db_get_path ()
+{
+    try_command_on_node -v $1 $CTDB getdbstatus "$2" |
+    sed -n -e "s@^path: @@p"
+}
+
+# $1: pnn, $2: DB name
+db_ctdb_cattdb_count_records ()
+{
+    try_command_on_node -v $1 $CTDB cattdb "$2" |
+    grep '^key' | grep -v '__db_sequence_number__' |
+    wc -l
+}
 
+# $1: pnn, $2: DB name, $3: key string, $4: value string, $5: RSN (default 7)
+db_ctdb_tstore ()
+{
+    _tdb=$(db_get_path $1 "$2")
+    _rsn="${5:-7}"
+    try_command_on_node $1 $CTDB tstore "$_tdb" "$3" "$4" "$_rsn"
+}
+
+# $1: pnn, $2: DB name, $3: dbseqnum (must be < 255!!!!!)
+db_ctdb_tstore_dbseqnum ()
+{
+    # "__db_sequence_number__" + trailing 0x00
+    _key='0x5f5f64625f73657175656e63655f6e756d6265725f5f00'
+
+    # Construct 8 byte (unit64_t) database sequence number.  This
+    # probably breaks if $3 > 255
+    _value=$(printf "0x%02x%014x" $3 0)
+
+    db_ctdb_tstore $1 "$2" "$_key" "$_value"
+}
 
 #######################################
 
diff --git a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
index cb6021fd459..a4765b142b0 100755
--- a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
+++ b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
@@ -54,100 +54,97 @@ ctdb_restart_when_done
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
-# create a temporary persistent database to test with
-echo create persistent test database persistent_test.tdb
-try_command_on_node 0 $CTDB attach persistent_test.tdb persistent
-
+add_record_per_node ()
+{
+    _i=0
+    while [ $_i -lt $num_nodes ] ; do
+	_k="KEY${_i}"
+	_d="DATA${_i}"
+	echo "Store key(${_k}) data(${_d}) on node ${_i}"
+	db_ctdb_tstore $_i "$test_db" "$_k" "$_d"
+	_i=$(($_i + 1))
+    done
+}
 
-# set RecoverPDBBySeqNum=0
-echo "setting RecoverPDBBySeqNum to 0"
-try_command_on_node all $CTDB setvar RecoverPDBBySeqNum 0
+test_db="persistent_test.tdb"
+echo "Create persistent test database \"$test_db\""
+try_command_on_node 0 $CTDB attach "$test_db" persistent
 
 
+echo "Setting RecoverPDBBySeqNum=0"
+try_command_on_node all $CTDB setvar "RecoverPDBBySeqNum" 0
 
-# 3,
+# 3.
 # If RecoverPDBBySeqNum==0  and no __db_sequence_number__
 # recover record by record
 #
 # wipe database
 echo
-echo test that RecoverPDBBySeqNum==0 and no __db_sequence_number__ blends the database during recovery
-echo wipe the test database
-try_command_on_node 0 $CTDB wipedb persistent_test.tdb
+echo "Test that RecoverPDBBySeqNum=0 and no __db_sequence_number__ blends the database during recovery"
 
-# add one record to node 0   key==ABC  data==ABC
-TDB=`try_command_on_node -v 0 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(ABC) data(ABC) on node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
-#
-# add one record to node 1   key==DEF  data==DEF
-TDB=`try_command_on_node -v 1 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(DEF) data(DEF) on node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
+echo "Wipe test database"
+try_command_on_node 0 $CTDB wipedb "$test_db"
+
+add_record_per_node
 
 # force a recovery
-echo force a recovery
+echo "Force a recovery"
 try_command_on_node 0 $CTDB recover
 
 # check that we now have both records on node 0
-num_records=$(try_command_on_node -v 0 $CTDB cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
-[ $num_records != "2" ] && {
-    echo "BAD: we did not end up with the expected two records after the recovery"
+num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
+if [ $num_records = "$num_nodes" ] ; then
+    echo "OK: databases were blended"
+else
+    echo "BAD: we did not end up with the expected $num_nodes records after the recovery"
     exit 1
-}
-echo "OK. databases were blended"
-
+fi
 
-
-# 4,
+# 4.
 # If RecoverPDBBySeqNum==0  and __db_sequence_number__
 # recover record by record
 #
 # wipe database
 echo
-echo test that RecoverPDBBySeqNum==0 and __db_sequence_number__ blends the database during recovery
-echo wipe the test database
+echo "Test that RecoverPDBBySeqNum=0 and __db_sequence_number__ blends the database during recovery"
+
+echo "Wipe the test database"
 try_command_on_node 0 $CTDB wipedb persistent_test.tdb
 
-echo "add __db_sequence_number__==5 record to all nodes"
-try_command_on_node -v 0 $CTDB nodestatus all | grep pnn | sed -e"s/^pnn://" -e "s/ .*//" | while read PNN; do
-    TDB=`try_command_on_node -v $PNN $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-    try_command_on_node $PNN $CTDB tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000500000000000000
+add_record_per_node
+
+echo "Add __db_sequence_number__==5 record to all nodes"
+pnn=0
+while [ $pnn -lt $num_nodes ] ; do
+    db_ctdb_tstore_dbseqnum $pnn "$test_db" 5
+    pnn=$(($pnn + 1))
 done
 
-# add one record to node 0   key==ABC  data==ABC
-TDB=`try_command_on_node -v 0 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(ABC) data(ABC) on node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
-echo "add __db_sequence_number__==7 record to node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000700000000000000
+echo "Set __db_sequence_number__ to 7 on node 0"
+db_ctdb_tstore_dbseqnum 0 "$test_db" 7
 
-# add one record to node 1   key==DEF  data==DEF
-TDB=`try_command_on_node -v 1 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(DEF) data(DEF) on node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
-echo "add __db_sequence_number__==8 record to node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000800000000000000
+echo "Set __db_sequence_number__ to 8 on node 1"
+db_ctdb_tstore_dbseqnum 1 "$test_db" 8
 
 # force a recovery
-echo force a recovery
+echo "Force a recovery"
 try_command_on_node 0 $CTDB recover
 
 # check that we now have both records on node 0
-num_records=$(try_command_on_node -v 0 $CTDB cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
-[ $num_records != "2" ] && {
-    echo "BAD: we did not end up with the expected two records after the recovery"
+num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
+if [ $num_records = "$num_nodes" ] ; then
+    echo "OK: databases were blended"
+else
+    echo "BAD: we did not end up with the expected $num_nodes records after the recovery"
+    try_command_on_node -v 0 $CTDB cattdb "$test_db"
     exit 1
-}
-echo "OK. databases were blended"
-
+fi
 
 
 # set RecoverPDBBySeqNum=1
 echo
-echo "setting RecoverPDBBySeqNum to 1"
-try_command_on_node all $CTDB setvar RecoverPDBBySeqNum 1
-
+echo "Setting RecoverPDBBySeqNum to 1"
+try_command_on_node all $CTDB setvar "RecoverPDBBySeqNum" 1
 
 
 # 5,
@@ -156,31 +153,24 @@ try_command_on_node all $CTDB setvar RecoverPDBBySeqNum 1
 #
 # wipe database
 echo
-echo test that RecoverPDBBySeqNum==1 and no __db_sequence_number__ does not blend the database during recovery
-echo wipe the test database
-try_command_on_node 0 $CTDB wipedb persistent_test.tdb
+echo "Test that RecoverPDBBySeqNum=1 and no __db_sequence_number__ does not blend the database during recovery"
+echo "Wipe the test database"
+try_command_on_node 0 $CTDB wipedb "$test_db"
 
-# add one record to node 0   key==ABC  data==ABC
-TDB=`try_command_on_node -v 0 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(ABC) data(ABC) on node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
-
-# add one record to node 1   key==DEF  data==DEF
-TDB=`try_command_on_node -v 1 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(DEF) data(DEF) on node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
+add_record_per_node
 
 # force a recovery
 echo force a recovery
 try_command_on_node 0 $CTDB recover
 
-# check that we now have both records on node 0
-num_records=$(try_command_on_node -v 0 $CTDB cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
-[ $num_records != "1" ] && {
+# Check that we now have 1 record on node 0
+num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
+if [ $num_records = "1" ] ; then
+    echo "OK: databases were not blended"
+else
     echo "BAD: we did not end up with the expected single record after the recovery"
     exit 1
-}
-echo "OK. databases were not blended"
+fi
 
 
 
@@ -194,36 +184,31 @@ echo test that RecoverPDBBySeqNum==1 and __db_sequence_number__ does not blend t
 echo wipe the test database
 try_command_on_node 0 $CTDB wipedb persistent_test.tdb
 
-echo "add __db_sequence_number__==5 record to all nodes"
-try_command_on_node -v 0 $CTDB nodestatus all | grep pnn | sed -e"s/^pnn://" -e "s/ .*//" | while read PNN; do
-    TDB=`try_command_on_node -v $PNN $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-    try_command_on_node $PNN $CTDB tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000500000000000000
+add_record_per_node
+
+echo "Add __db_sequence_number__==5 record to all nodes"
+pnn=0
+while [ $pnn -lt $num_nodes ] ; do
+    db_ctdb_tstore_dbseqnum $pnn "$test_db" 5
+    pnn=$(($pnn + 1))
 done
 
+echo "Set __db_sequence_number__ to 7 on node 0"
+db_ctdb_tstore_dbseqnum 0 "$test_db" 7
 
-# add one record to node 0   key==ABC  data==ABC
-TDB=`try_command_on_node -v 0 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(ABC) data(ABC) on node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
-echo "add __db_sequence_number__==7 record to node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000700000000000000
+echo "Set __db_sequence_number__ to 8 on node 1"
+db_ctdb_tstore_dbseqnum 1 "$test_db" 8
 
-# add one record to node 1   key==DEF  data==DEF
-TDB=`try_command_on_node -v 1 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(DEF) data(DEF) on node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
-echo "add __db_sequence_number__==8 record to node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000800000000000000
 
 # force a recovery
 echo force a recovery
 try_command_on_node 0 $CTDB recover
 
 # check that we now have both records on node 0
-num_records=$(try_command_on_node -v 0 $CTDB cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
-[ $num_records != "1" ] && {
+num_records=$(db_ctdb_cattdb_count_records 0 "$test_db")
+if [ $num_records = "1" ] ; then
+    echo "OK: databases were not blended"
+else
     echo "BAD: we did not end up with the expected single record after the recovery"
     exit 1
-}
-
-echo "OK. databases were not blended"
+fi
diff --git a/ctdb/tests/simple/71_ctdb_wipedb.sh b/ctdb/tests/simple/71_ctdb_wipedb.sh
index 3bf695e0663..886e5e6d6f5 100755
--- a/ctdb/tests/simple/71_ctdb_wipedb.sh
+++ b/ctdb/tests/simple/71_ctdb_wipedb.sh
@@ -13,13 +13,13 @@ Prerequisites:
 Steps:
 
 1. Verify that the status on all of the ctdb nodes is 'OK'.
-2. create a persistent test database
-3, add some records to node #0 and node #1
-4, perform wipedb on node #0 and verify the database is empty on both node 0 and 1
+2. Create a persistent test database
+3. Add some records to node #0 and node #1
+4. Perform wipedb on node #0 and verify the database is empty on both node 0 and 1
 
 Expected results:
 
-* that 4 will result in empty database
+* An empty database will result
 
 EOF
 }
@@ -35,36 +35,31 @@ cluster_is_healthy
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
-# create a temporary persistent database to test with
-echo create persistent test database persistent_test.tdb
-try_command_on_node 0 $CTDB attach persistent_test.tdb persistent
+# 2.
+test_db="persistent_test.tdb"
+echo "Create persistent test database \"$test_db\""
+try_command_on_node 0 $CTDB attach "$test_db" persistent
 
-
-# 3,
+# 3.
 # add one record to node 0   key==ABC  data==ABC
-TDB=`try_command_on_node -v 0 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(ABC) data(ABC) on node 0"
-try_command_on_node 0 $CTDB tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
-#
-# add one record to node 1   key==DEF  data==DEF
-TDB=`try_command_on_node -v 1 $CTDB getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
-echo "store key(DEF) data(DEF) on node 1"
-try_command_on_node 1 $CTDB tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
-
+echo "Store key(ABC) data(ABC) on node 0"
+db_ctdb_tstore 0 "$test_db" "ABC" "ABC"
 
-# 4,
-echo wipe the persistent test database
-try_command_on_node 0 $CTDB wipedb persistent_test.tdb
+# add one record to node 1   key==DEF  data==DEF
+echo "Store key(DEF) data(DEF) on node 1"
+db_ctdb_tstore 1 "$test_db" "DEF" "DEF"
 
-echo force a recovery
+# 4.
+echo "Wipe database"
+try_command_on_node 0 $CTDB wipedb "$test_db"
+echo "Force a recovery"
 try_command_on_node 0 $CTDB recover
 
 # check that the database is wiped
-num_records=$(try_command_on_node -v 1 $CTDB cattdb persistent_test.tdb | \
-    grep key | grep -v '__db_sequence_number__' | wc -l)
-[ $num_records != "0" ] && {
-    echo "BAD: we did not end up with an empty database"
+num_records=$(db_ctdb_cattdb_count_records 1 "$test_db")
+if [ $num_records = "0" ] ; then
+    echo "OK: Database was wiped"
+else
+    echo "BAD: We did not end up with an empty database"
     exit 1
-}
-echo "OK. database was wiped"
-
+fi
diff --git a/ctdb/tests/simple/72_update_record_persistent.sh b/ctdb/tests/simple/72_update_record_persistent.sh
index 3f1d2a137f6..f40933278e6 100755
--- a/ctdb/tests/simple/72_update_record_persistent.sh
+++ b/ctdb/tests/simple/72_update_record_persistent.sh
@@ -37,54 +37,59 @@ cluster_is_healthy
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
-TDB=persistent_test.tdb
+test_db="persistent_test.tdb"
 
 # create a temporary persistent database to test with
-echo create persistent test database $TDB
-try_command_on_node 0 $CTDB attach $TDB persistent
+echo "Create persistent test database \"$test_db\""
+try_command_on_node 0 $CTDB attach "$test_db" persistent
 
 
 # 3,
-echo wipe the persistent test database
-try_command_on_node 0 $CTDB wipedb $TDB
-echo force a recovery
+echo "Wipe the persistent test database"
+try_command_on_node 0 $CTDB wipedb "$test_db"
+echo "Force a recovery"
 try_command_on_node 0 $CTDB recover
 
 # check that the database is wiped
-num_records=$(try_command_on_node -v 1 $CTDB cattdb $TDB | \
-    grep key | grep -v '__db_sequence_number__' | wc -l)
-[ $num_records != "0" ] && {
+num_records=$(db_ctdb_cattdb_count_records 1 "$test_db")
+if [ $num_records = "0" ] ; then
+    echo "OK: database was wiped"
+else
     echo "BAD: we did not end up with an empty database"
     exit 1
-}
-echo "OK. database was wiped"
+fi
 
 # 4,
-echo Create a new record in the persistent database using UPDATE_RECORD
-try_command_on_node 0 $CTDB_TEST_WRAPPER ctdb_update_record_persistent  --database=$TDB --record=Update_Record_Persistent --value=FirstValue
+echo "Create a new record in the persistent database using UPDATE_RECORD"
+try_command_on_node 0 $CTDB_TEST_WRAPPER ctdb_update_record_persistent  --database="$test_db" --record=Update_Record_Persistent --value=FirstValue
 
-try_command_on_node 0 "ctdb cattdb $TDB | grep 'FirstValue' | wc -l"
-[ $out != 1 ] && {
+try_command_on_node 0 "$CTDB cattdb "$test_db" | grep 'FirstValue' | wc -l"
+if [ "$out" = 1 ] ; then
+    echo "GOOD: we did not find the record after the create/update"
+else
     echo "BAD: we did find the record after the create/update"
     exit 1
-}
+fi
 
 # 5,
 echo Modify an existing record in the persistent database using UPDATE_RECORD
-try_command_on_node 0 $CTDB_TEST_WRAPPER ctdb_update_record_persistent  --database=$TDB --record=Update_Record_Persistent --value=SecondValue
+try_command_on_node 0 $CTDB_TEST_WRAPPER ctdb_update_record_persistent  --database="$test_db" --record=Update_Record_Persistent --value=SecondValue
 
-try_command_on_node 0 "ctdb cattdb $TDB | grep 'FirstValue' | wc -l"
-[ $out != 0 ] && {
+try_command_on_node 0 "$CTDB cattdb "$test_db" | grep 'FirstValue' | wc -l"
+if [ "$out" = 0 ] ; then
+    echo "GOOD: did not find old record after the modify/update"
+else
     echo "BAD: we still found the old record after the modify/update"
     exit 1
-}
+fi
 
-try_command_on_node 0 "ctdb cattdb $TDB | grep 'SecondValue' | wc -l"
-[ $out != 1 ] && {
+try_command_on_node 0 "$CTDB cattdb "$test_db" | grep 'SecondValue' | wc -l"
+if [ "$out" = 1 ] ; then
+    echo "GOOD: found the record after the modify/update"
+else
     echo "BAD: could not find the record after the modify/update"
     exit 1
-}
-
+fi
 
-echo wipe the persistent test databases and clean up
-try_command_on_node 0 $CTDB wipedb $TDB
+echo "Wipe the persistent test databases and clean up"
+try_command_on_node 0 $CTDB wipedb "$test_db"