1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
#!/bin/sh
# this script needs to be installed so that statd points to it with the -H
# command line argument. The easiest way to do that is to put something like this in
# /etc/sysconfig/nfs:
# STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
[ -z "$CTDB_BASE" ] && {
export CTDB_BASE="/etc/ctdb"
}
. $CTDB_BASE/functions
loadconfig nfs
[ -z "$STATD_SHARED_DIRECTORY" ] && {
echo STATD_SHARED_DIRECTORY not configured. statd-callout failed.
exit 0
}
[ -d $STATD_SHARED_DIRECTORY ] || exit 0
[ -z $NFS_HOSTNAME ] && {
echo NFS_HOSTNAME is not configured. statd-callout failed.
exit 0
}
case "$1" in
add-client)
# the callout does not tell us to which ip the client connected
# so we must add it to all the ips that we serve
for f in `/bin/ls $CTDB_BASE/state/statd/ip/*`; do
ip=`basename $f`
[ -d $STATD_SHARED_DIRECTORY/$ip ] || /bin/mkdir $STATD_SHARED_DIRECTORY/$ip
touch $STATD_SHARED_DIRECTORY/$ip/$2
done
;;
del-client)
# the callout does not tell us to which ip the client connected
# so we must add it to all the ips that we serve
for f in `/bin/ls $CTDB_BASE/state/statd/ip/*`; do
ip=`basename $f`
/bin/rm -f $STATD_SHARED_DIRECTORY/$ip/$2
done
;;
notify)
# we must restart the lockmanager (on all nodes) so that we get
# a clusterwide grace period (so other clients dont take out
# conflicting locks through other nodes before all locks have been
# reclaimed)
# we need these settings to make sure that no tcp connections survive
# across a very fast failover/failback
#echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
#echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
#echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
# rebuild the state directory for the local statd to use the correct
# state value and to initally send notifications to all clients
rm -f /var/lib/nfs/statd/sm/*
rm -f /var/lib/nfs/statd/sm.bak/*
cat $STATD_SHARED_DIRECTORY/state >/var/lib/nfs/statd/state
# we must keep a monotonically increasing state variable for the entire
# cluster so state always increases when ip addresses fail from one
# node to another
[ ! -f $STATD_SHARED_DIRECTORY/state ] && {
echo 1 | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
}
# read current state
STATE=`od -t d4 $STATD_SHARED_DIRECTORY/state | head -1 | sed -e "s/^[0-9]*[^0-9]*//"`
# write current state+2 back to the state file
# the /2 *2 are to ensure that state is odd. state must be odd.
STATE=`expr $STATE "/" 2 "*" 2 "+" 3`
echo $STATE | awk '{printf("%c%c%c%c", $0, $0/256, $0/256/256, $0/256/256/256);}' >$STATD_SHARED_DIRECTORY/state
# we must also let some time pass between stopping and restarting the
# lockmanager since othervise there is a window where the lockmanager
# will respond "strangely" immediately after restarting it, which
# causes clients to fail to reclaim the locks.
#
startstop_nfslock stop > /dev/null 2>&1
sleep 2
# copy all monitored clients on this node to the local lockmanager
for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
ip=`basename $f`
[ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
client=`basename $g`
touch /var/lib/nfs/statd/sm/$client
done
}
done
# now start lockmanager again with the new state directory.
startstop_nfslock start > /dev/null 2>&1
# we now need to send out additional statd notifications to ensure
# that clients understand that the lockmanager has restarted.
# we have three cases:
# 1, clients that ignore the ip address the stat notification came from
# and ONLY care about the 'name' in the notify packet.
# these clients ONLY work with lock failover IFF that name
# can be resolved into an ipaddress that matches the one used
# to mount the share. (==linux clients)
# This is handled when starting lockmanager above, but those
# packets are sent from the "wrong" ip address, something linux
# clients are ok with, buth other clients will barf at.
# 2, Some clients only accept statd packets IFF they come from the
# 'correct' ip address.
# 2a,Send out the notification using the 'correct' ip address and also
# specify the 'correct' hostname in the statd packet.
# Some clients require both the correct source address and also the
# correct name. (these clients also ONLY work if the ip addresses
# used to map the share can be resolved into the name returned in
# the notify packet.)
# 2b,Other clients require that the source ip address of the notify
# packet matches the ip address used to take out the lock.
# I.e. that the correct source address is used.
# These clients also require that the statd notify packet contains
# the name as the ip address used when the lock was taken out.
#
# Both 2a and 2b are commonly used in lockmanagers since they maximize
# probability that the client will accept the statd notify packet and
# not just ignore it.
for f in `/bin/ls $CTDB_BASE/state/statd/ip/* 2>/dev/null`; do
ip=`basename $f`
[ -d $STATD_SHARED_DIRECTORY/$ip ] && [ -x /usr/bin/smnotify ] && {
for g in `/bin/ls $STATD_SHARED_DIRECTORY/$ip/* 2>/dev/null`; do
client=`basename $g`
# /bin/rm -f $g
# send out notifications from the "correct" address
# (the same addresse as where the lock was taken out
# on) some clients require that the source address
# matches where the lock was taken out.
# also send it both as a name that the client
# hopefully can resolve into the server ip and
# and also by specifying the raw ip address as name.
/usr/bin/smnotify --client=$client --ip=$ip --server=$ip --stateval=$STATE
/usr/bin/smnotify --client=$client --ip=$ip --server=$NFS_HOSTNAME --stateval=$STATE
done
}
done
;;
esac
|
