blob: 4b46cbbb3c8254eacfb33c3e11d33261517cdd40 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
#!/bin/bash
test_info()
{
cat <<EOF
The tunable RecoverPDBBySeqNum controls how we perform recovery
on persistent databases.
The default is that persistent databases are recovered exactly the same
way as normal databases. That is that we recover record by record.
If RecoverPDBBySeqNum is set to 1 AND if a record with the key
"__db_sequence_number__" can be found in the database, then instead we will
perform the recovery by picking the copy of the database from the node
that has the highest sequence number and ignore the content on all other
nodes.
Prerequisites:
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. create a persistent test database
3. test that RecoveryPDBBySeqNum==0 and no seqnum record blends the database
during recovery
4. test that RecoveryPDBBySeqNum==0 and seqnum record blends the database
during recovery
5. test that RecoveryPDBBySeqNum==1 and no seqnum record blends the database
during recovery
6. test that RecoveryPDBBySeqNum==1 and seqnum record does not blend the database
during recovery
Expected results:
* that 3,4,5 will blend the databases and that 6 will recovery the highest seqnum
database
EOF
}
. ctdb_test_functions.bash
ctdb_test_init "$@"
set -e
cluster_is_healthy
try_command_on_node 0 "$CTDB listnodes"
num_nodes=$(echo "$out" | wc -l)
# create a temporary persistent database to test with
echo create persistent test database persistent_test.tdb
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach persistent_test.tdb persistent
# set RecoverPDBBySeqNum=0
echo "setting RecoverPDBBySeqNum to 0"
try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 0
# 3,
# If RecoverPDBBySeqNum==0 and no __db_sequence_number__
# recover record by record
#
# wipe database
echo
echo test that RecoverPDBBySeqNum==0 and no __db_sequence_number__ blends the database during recovery
echo wipe the test database
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb
# add one record to node 0 key==ABC data==ABC
TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(ABC) data(ABC) on node 0"
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
#
# add one record to node 1 key==DEF data==DEF
TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(DEF) data(DEF) on node 1"
try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
# force a recovery
echo force a recovery
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover
# check that we now have both records on node 0
num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
[ $num_records != "2" ] && {
echo "BAD: we did not end up with the expected two records after the recovery"
exit 1
}
echo "OK. databases were blended"
# 4,
# If RecoverPDBBySeqNum==0 and __db_sequence_number__
# recover record by record
#
# wipe database
echo
echo test that RecoverPDBBySeqNum==0 and __db_sequence_number__ blends the database during recovery
echo wipe the test database
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb
echo "add __db_sequence_number__==5 record to all nodes"
try_command_on_node -v 0 $CTDB_TEST_WRAPPER ctdb nodestatus all | grep pnn | sed -e"s/^pnn://" -e "s/ .*//" | while read PNN; do
TDB=`try_command_on_node -v -q $PNN $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
try_command_on_node -q $PNN $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000500000000000000
done
# add one record to node 0 key==ABC data==ABC
TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(ABC) data(ABC) on node 0"
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
echo "add __db_sequence_number__==7 record to node 0"
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000700000000000000
# add one record to node 1 key==DEF data==DEF
TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(DEF) data(DEF) on node 1"
try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
echo "add __db_sequence_number__==8 record to node 1"
try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000800000000000000
# force a recovery
echo force a recovery
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover
# check that we now have both records on node 0
num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
[ $num_records != "2" ] && {
echo "BAD: we did not end up with the expected two records after the recovery"
exit 1
}
echo "OK. databases were blended"
# set RecoverPDBBySeqNum=1
echo
echo "setting RecoverPDBBySeqNum to 1"
try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 1
# 5,
# If RecoverPDBBySeqNum==1 and no __db_sequence_number__
# recover record by record
#
# wipe database
echo
echo test that RecoverPDBBySeqNum==1 and no __db_sequence_number__ blends the database during recovery
echo wipe the test database
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb
# add one record to node 0 key==ABC data==ABC
TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(ABC) data(ABC) on node 0"
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
# add one record to node 1 key==DEF data==DEF
TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(DEF) data(DEF) on node 1"
try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
# force a recovery
echo force a recovery
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover
# check that we now have both records on node 0
num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
[ $num_records != "2" ] && {
echo "BAD: we did not end up with the expected two records after the recovery"
exit 1
}
echo "OK. databases were blended"
# 6,
# If RecoverPDBBySeqNum==1 and __db_sequence_number__
# recover whole database
#
# wipe database
echo
echo test that RecoverPDBBySeqNum==1 and __db_sequence_number__ does not blend the database during recovery
echo wipe the test database
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb
echo "add __db_sequence_number__==5 record to all nodes"
try_command_on_node -v 0 $CTDB_TEST_WRAPPER ctdb nodestatus all | grep pnn | sed -e"s/^pnn://" -e "s/ .*//" | while read PNN; do
TDB=`try_command_on_node -v -q $PNN $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
try_command_on_node -q $PNN $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000500000000000000
done
# add one record to node 0 key==ABC data==ABC
TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(ABC) data(ABC) on node 0"
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243
echo "add __db_sequence_number__==7 record to node 0"
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000700000000000000
# add one record to node 1 key==DEF data==DEF
TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"`
echo "store key(DEF) data(DEF) on node 1"
try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546
echo "add __db_sequence_number__==8 record to node 1"
try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000800000000000000
# force a recovery
echo force a recovery
try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover
# check that we now have both records on node 0
num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l)
[ $num_records != "1" ] && {
echo "BAD: we did not end up with the expected single record after the recovery"
exit 1
}
echo "OK. databases were not blended"
# set RecoverPDBBySeqNum=1
echo "setting RecoverPDBBySeqNum back to 0"
try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 0
|
