diff options
| -rw-r--r-- | ctdb/client/ctdb_client.c | 3 | ||||
| -rw-r--r-- | ctdb/common/rb_tree.c | 11 | ||||
| -rw-r--r-- | ctdb/doc/ctdb.1 | 759 | ||||
| -rw-r--r-- | ctdb/doc/ctdb.1.html | 246 | ||||
| -rw-r--r-- | ctdb/doc/ctdb.1.xml | 90 | ||||
| -rw-r--r-- | ctdb/include/ctdb_private.h | 5 | ||||
| -rw-r--r-- | ctdb/server/ctdb_ltdb_server.c | 20 | ||||
| -rw-r--r-- | ctdb/server/ctdb_tunables.c | 101 | ||||
| -rw-r--r-- | ctdb/server/ctdb_vacuum.c | 859 | ||||
| -rw-r--r-- | ctdb/tools/ctdb.c | 4 | ||||
| -rw-r--r-- | ctdb/tools/ctdb_vacuum.c | 441 |
11 files changed, 1247 insertions, 1292 deletions
diff --git a/ctdb/client/ctdb_client.c b/ctdb/client/ctdb_client.c index 224cdc03ee..a580a3e380 100644 --- a/ctdb/client/ctdb_client.c +++ b/ctdb/client/ctdb_client.c @@ -47,14 +47,13 @@ struct ctdb_req_header *_ctdbd_allocate_pkt(struct ctdb_context *ctdb, length = MAX(length, slength); size = (length+(CTDB_DS_ALIGNMENT-1)) & ~(CTDB_DS_ALIGNMENT-1); - hdr = (struct ctdb_req_header *)talloc_size(mem_ctx, size); + hdr = (struct ctdb_req_header *)talloc_zero_size(mem_ctx, size); if (hdr == NULL) { DEBUG(DEBUG_ERR,("Unable to allocate packet for operation %u of length %u\n", operation, (unsigned)length)); return NULL; } talloc_set_name_const(hdr, type); - memset(hdr, 0, slength); hdr->length = length; hdr->operation = operation; hdr->ctdb_magic = CTDB_MAGIC; diff --git a/ctdb/common/rb_tree.c b/ctdb/common/rb_tree.c index 8458c510be..bff7c5a4c4 100644 --- a/ctdb/common/rb_tree.c +++ b/ctdb/common/rb_tree.c @@ -921,9 +921,12 @@ trbt_traversearray32_node(trbt_node_t *node, uint32_t keylen, int (*callback)(void *param, void *data), void *param) { - if (node->left) { + trbt_node_t *left = node->left; + trbt_node_t *right = node->right; + + if (left) { int ret; - ret = trbt_traversearray32_node(node->left, keylen, callback, param); + ret = trbt_traversearray32_node(left, keylen, callback, param); if (ret != 0) { return ret; } @@ -949,10 +952,10 @@ trbt_traversearray32_node(trbt_node_t *node, uint32_t keylen, } } - if (node->right) { + if (right) { int ret; - ret = trbt_traversearray32_node(node->right, keylen, callback, param); + ret = trbt_traversearray32_node(right, keylen, callback, param); if (ret != 0) { return ret; } diff --git a/ctdb/doc/ctdb.1 b/ctdb/doc/ctdb.1 index ab481fba3e..55454f3cf7 100644 --- a/ctdb/doc/ctdb.1 +++ b/ctdb/doc/ctdb.1 @@ -1,164 +1,218 @@ -.\" ** You probably do not want to edit this file directly ** -.\" It was generated using the DocBook XSL Stylesheets (version 1.69.1). -.\" Instead of manually editing it, you probably should edit the DocBook XML -.\" source for it and then use the DocBook XSL Stylesheets to regenerate it. -.TH "CTDB" "1" "12/14/2011" "ctdb" "CTDB \- clustered TDB database" +'\" t +.\" Title: ctdb +.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] +.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/> +.\" Date: 12/23/2011 +.\" Manual: CTDB - clustered TDB database +.\" Source: ctdb +.\" Language: English +.\" +.TH "CTDB" "1" "12/23/2011" "ctdb" "CTDB \- clustered TDB database" +.\" ----------------------------------------------------------------- +.\" * Define some portability stuff +.\" ----------------------------------------------------------------- +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.\" http://bugs.debian.org/507673 +.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l +.\" ----------------------------------------------------------------- +.\" * MAIN CONTENT STARTS HERE * +.\" ----------------------------------------------------------------- .SH "NAME" ctdb \- clustered tdb database management utility .SH "SYNOPSIS" -.HP 29 -\fBctdb [ OPTIONS ] COMMAND ...\fR -.HP 5 +.HP \w'\fBctdb\ [\ OPTIONS\ ]\ COMMAND\ \&.\&.\&.\fR\ 'u +\fBctdb [ OPTIONS ] COMMAND \&.\&.\&.\fR +.HP \w'\fBctdb\fR\ 'u \fBctdb\fR [\-n\ <node>] [\-Y] [\-t\ <timeout>] [\-T\ <timelimit>] [\-?\ \-\-help] [\-\-usage] [\-d\ \-\-debug=<INTEGER>] [\-\-socket=<filename>] [\-\-print\-emptyrecords] [\-\-print\-datasize] [\-\-print\-lmaster] [\-\-print\-hash] [\-\-print\-recordflags] .SH "DESCRIPTION" .PP -ctdb is a utility to view and manage a ctdb cluster. +ctdb is a utility to view and manage a ctdb cluster\&. .SH "OPTIONS" -.TP +.PP \-n <pnn> -This specifies the physical node number on which to execute the command. Default is to run the command on the daemon running on the local host. +.RS 4 +This specifies the physical node number on which to execute the command\&. Default is to run the command on the daemon running on the local host\&. .sp -The physical node number is an integer that describes the node in the cluster. The first node has physical node number 0. -.TP +The physical node number is an integer that describes the node in the cluster\&. The first node has physical node number 0\&. +.RE +.PP \-Y -Produce output in machine readable form for easier parsing by scripts. Not all commands support this option. -.TP +.RS 4 +Produce output in machine readable form for easier parsing by scripts\&. Not all commands support this option\&. +.RE +.PP \-t <timeout> -How long should ctdb wait for the local ctdb daemon to respond to a command before timing out. Default is 3 seconds. -.TP +.RS 4 +How long should ctdb wait for the local ctdb daemon to respond to a command before timing out\&. Default is 3 seconds\&. +.RE +.PP \-T <timelimit> -A limit on how long the ctdb command will run for before it will be aborted. When this timelimit has been exceeded the ctdb command will terminate. -.TP +.RS 4 +A limit on how long the ctdb command will run for before it will be aborted\&. When this timelimit has been exceeded the ctdb command will terminate\&. +.RE +.PP \-? \-\-help -Print some help text to the screen. -.TP +.RS 4 +Print some help text to the screen\&. +.RE +.PP \-\-usage -Print useage information to the screen. -.TP +.RS 4 +Print useage information to the screen\&. +.RE +.PP \-d \-\-debug=<debuglevel> -Change the debug level for the command. Default is 0. -.TP +.RS 4 +Change the debug level for the command\&. Default is 0\&. +.RE +.PP \-\-socket=<filename> -Specify the socketname to use when connecting to the local ctdb daemon. The default is /tmp/ctdb.socket . +.RS 4 +Specify the socketname to use when connecting to the local ctdb daemon\&. The default is /tmp/ctdb\&.socket \&. .sp -You only need to specify this parameter if you run multiple ctdb daemons on the same physical host and thus can not use the default name for the domain socket. -.TP +You only need to specify this parameter if you run multiple ctdb daemons on the same physical host and thus can not use the default name for the domain socket\&. +.RE +.PP \-\-print\-emptyrecords -This enables printing of empty records when dumping databases with the catdb, cattbd and dumpdbbackup commands. Records with empty data segment are considered deleted by ctdb and cleaned by the vacuuming mechanism, so this switch can come in handy for debugging the vacuuming behaviour. -.TP +.RS 4 +This enables printing of empty records when dumping databases with the catdb, cattbd and dumpdbbackup commands\&. Records with empty data segment are considered deleted by ctdb and cleaned by the vacuuming mechanism, so this switch can come in handy for debugging the vacuuming behaviour\&. +.RE +.PP \-\-print\-datasize -This lets database dumps (catdb, cattdb, dumpdbbackup) print the size of the record data instead of dumping the data contents. -.TP +.RS 4 +This lets database dumps (catdb, cattdb, dumpdbbackup) print the size of the record data instead of dumping the data contents\&. +.RE +.PP \-\-print\-lmaster -This lets catdb print the lmaster for each record. -.TP +.RS 4 +This lets catdb print the lmaster for each record\&. +.RE +.PP \-\-print\-hash -This lets database dumps (catdb, cattdb, dumpdbbackup) print the hash for each record. -.TP +.RS 4 +This lets database dumps (catdb, cattdb, dumpdbbackup) print the hash for each record\&. +.RE +.PP \-\-print\-recordflags -This lets catdb and dumpdbbackup print the record flags for each record. Note that cattdb always prints the flags. +.RS 4 +This lets catdb and dumpdbbackup print the record flags for each record\&. Note that cattdb always prints the flags\&. +.RE .SH "ADMINISTRATIVE COMMANDS" .PP -These are commands used to monitor and administrate a CTDB cluster. +These are commands used to monitor and administrate a CTDB cluster\&. .SS "pnn" .PP -This command displays the pnn of the current node. +This command displays the pnn of the current node\&. .SS "status" .PP -This command shows the current status of the ctdb node. +This command shows the current status of the ctdb node\&. .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br +.ps +1 \fBnode status\fR -.RS 3 +.RS 4 .PP -Node status reflects the current status of the node. There are five possible states: +Node status reflects the current status of the node\&. There are five possible states: .PP -OK \- This node is fully functional. +OK \- This node is fully functional\&. .PP -DISCONNECTED \- This node could not be connected through the network and is currently not participating in the cluster. If there is a public IP address associated with this node it should have been taken over by a different node. No services are running on this node. +DISCONNECTED \- This node could not be connected through the network and is currently not participating in the cluster\&. If there is a public IP address associated with this node it should have been taken over by a different node\&. No services are running on this node\&. .PP -DISABLED \- This node has been administratively disabled. This node is still functional and participates in the CTDB cluster but its IP addresses have been taken over by a different node and no services are currently being hosted. +DISABLED \- This node has been administratively disabled\&. This node is still functional and participates in the CTDB cluster but its IP addresses have been taken over by a different node and no services are currently being hosted\&. .PP -UNHEALTHY \- A service provided by this node is malfunctioning and should be investigated. The CTDB daemon itself is operational and participates in the cluster. Its public IP address has been taken over by a different node and no services are currnetly being hosted. All unhealthy nodes should be investigated and require an administrative action to rectify. +UNHEALTHY \- A service provided by this node is malfunctioning and should be investigated\&. The CTDB daemon itself is operational and participates in the cluster\&. Its public IP address has been taken over by a different node and no services are currnetly being hosted\&. All unhealthy nodes should be investigated and require an administrative action to rectify\&. .PP -BANNED \- This node failed too many recovery attempts and has been banned from participating in the cluster for a period of RecoveryBanPeriod seconds. Any public IP address has been taken over by other nodes. This node does not provide any services. All banned nodes should be investigated and require an administrative action to rectify. This node does not perticipate in the CTDB cluster but can still be communicated with. I.e. ctdb commands can be sent to it. +BANNED \- This node failed too many recovery attempts and has been banned from participating in the cluster for a period of RecoveryBanPeriod seconds\&. Any public IP address has been taken over by other nodes\&. This node does not provide any services\&. All banned nodes should be investigated and require an administrative action to rectify\&. This node does not perticipate in the CTDB cluster but can still be communicated with\&. I\&.e\&. ctdb commands can be sent to it\&. .PP -STOPPED \- A node that is stopped does not host any public ip addresses, nor is it part of the VNNMAP. A stopped node can not become LVSMASTER, RECMASTER or NATGW. This node does not perticipate in the CTDB cluster but can still be communicated with. I.e. ctdb commands can be sent to it. +STOPPED \- A node that is stopped does not host any public ip addresses, nor is it part of the VNNMAP\&. A stopped node can not become LVSMASTER, RECMASTER or NATGW\&. This node does not perticipate in the CTDB cluster but can still be communicated with\&. I\&.e\&. ctdb commands can be sent to it\&. .PP -PARTIALLYONLINE \- A node that is partially online participates in a cluster like a node that is ok. Some interfaces to serve public ip addresses are down, but at least one interface is up. See also "ctdb ifaces". +PARTIALLYONLINE \- A node that is partially online participates in a cluster like a node that is ok\&. Some interfaces to serve public ip addresses are down, but at least one interface is up\&. See also "ctdb ifaces"\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br +.ps +1 \fBgeneration\fR -.RS 3 +.RS 4 .PP -The generation id is a number that indicates the current generation of a cluster instance. Each time a cluster goes through a reconfiguration or a recovery its generation id will be changed. +The generation id is a number that indicates the current generation of a cluster instance\&. Each time a cluster goes through a reconfiguration or a recovery its generation id will be changed\&. .PP -This number does not have any particular meaning other than to keep track of when a cluster has gone through a recovery. It is a random number that represents the current instance of a ctdb cluster and its databases. CTDBD uses this number internally to be able to tell when commands to operate on the cluster and the databases was issued in a different generation of the cluster, to ensure that commands that operate on the databases will not survive across a cluster database recovery. After a recovery, all old outstanding commands will automatically become invalid. +This number does not have any particular meaning other than to keep track of when a cluster has gone through a recovery\&. It is a random number that represents the current instance of a ctdb cluster and its databases\&. CTDBD uses this number internally to be able to tell when commands to operate on the cluster and the databases was issued in a different generation of the cluster, to ensure that commands that operate on the databases will not survive across a cluster database recovery\&. After a recovery, all old outstanding commands will automatically become invalid\&. .PP -Sometimes this number will be shown as "INVALID". This only means that the ctdbd daemon has started but it has not yet merged with the cluster through a recovery. All nodes start with generation "INVALID" and are not assigned a real generation id until they have successfully been merged with a cluster through a recovery. +Sometimes this number will be shown as "INVALID"\&. This only means that the ctdbd daemon has started but it has not yet merged with the cluster through a recovery\&. All nodes start with generation "INVALID" and are not assigned a real generation id until they have successfully been merged with a cluster through a recovery\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br +.ps +1 \fBVNNMAP\fR -.RS 3 +.RS 4 .PP -The list of Virtual Node Numbers. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records. Only nodes that are participating in the vnnmap can become lmaster or dmaster for a database record. +The list of Virtual Node Numbers\&. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records\&. Only nodes that are participating in the vnnmap can become lmaster or dmaster for a database record\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br +.ps +1 \fBRecovery mode\fR -.RS 3 +.RS 4 .PP -This is the current recovery mode of the cluster. There are two possible modes: +This is the current recovery mode of the cluster\&. There are two possible modes: .PP -NORMAL \- The cluster is fully operational. +NORMAL \- The cluster is fully operational\&. .PP -RECOVERY \- The cluster databases have all been frozen, pausing all services while the cluster awaits a recovery process to complete. A recovery process should finish within seconds. If a cluster is stuck in the RECOVERY state this would indicate a cluster malfunction which needs to be investigated. +RECOVERY \- The cluster databases have all been frozen, pausing all services while the cluster awaits a recovery process to complete\&. A recovery process should finish within seconds\&. If a cluster is stuck in the RECOVERY state this would indicate a cluster malfunction which needs to be investigated\&. .PP -Once the recovery master detects an inconsistency, for example a node becomes disconnected/connected, the recovery daemon will trigger a cluster recovery process, where all databases are remerged across the cluster. When this process starts, the recovery master will first "freeze" all databases to prevent applications such as samba from accessing the databases and it will also mark the recovery mode as RECOVERY. +Once the recovery master detects an inconsistency, for example a node becomes disconnected/connected, the recovery daemon will trigger a cluster recovery process, where all databases are remerged across the cluster\&. When this process starts, the recovery master will first "freeze" all databases to prevent applications such as samba from accessing the databases and it will also mark the recovery mode as RECOVERY\&. .PP -When CTDBD starts up, it will start in RECOVERY mode. Once the node has been merged into a cluster and all databases have been recovered, the node mode will change into NORMAL mode and the databases will be "thawed", allowing samba to access the databases again. +When CTDBD starts up, it will start in RECOVERY mode\&. Once the node has been merged into a cluster and all databases have been recovered, the node mode will change into NORMAL mode and the databases will be "thawed", allowing samba to access the databases again\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br +.ps +1 \fBRecovery master\fR -.RS 3 +.RS 4 .PP -This is the cluster node that is currently designated as the recovery master. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired. +This is the cluster node that is currently designated as the recovery master\&. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired\&. .PP -Only one node at a time can be the designated recovery master. Which node is designated the recovery master is decided by an election process in the recovery daemons running on each node. +Only one node at a time can be the designated recovery master\&. Which node is designated the recovery master is decided by an election process in the recovery daemons running on each node\&. .RE .PP Example: ctdb status .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf Number of nodes:4 -pnn:0 11.1.2.200 OK (THIS NODE) -pnn:1 11.1.2.201 OK -pnn:2 11.1.2.202 OK -pnn:3 11.1.2.203 OK +pnn:0 11\&.1\&.2\&.200 OK (THIS NODE) +pnn:1 11\&.1\&.2\&.201 OK +pnn:2 11\&.1\&.2\&.202 OK +pnn:3 11\&.1\&.2\&.203 OK Generation:1362079228 Size:4 hash:0 lmaster:0 @@ -169,62 +223,86 @@ Recovery mode:NORMAL (0) Recovery master:0 .fi +.if n \{\ +.RE +.\} .SS "recmaster" .PP -This command shows the pnn of the node which is currently the recmaster. +This command shows the pnn of the node which is currently the recmaster\&. .SS "uptime" .PP -This command shows the uptime for the ctdb daemon. When the last recovery or ip\-failover completed and how long it took. If the "duration" is shown as a negative number, this indicates that there is a recovery/failover in progress and it started that many seconds ago. +This command shows the uptime for the ctdb daemon\&. When the last recovery or ip\-failover completed and how long it took\&. If the "duration" is shown as a negative number, this indicates that there is a recovery/failover in progress and it started that many seconds ago\&. .PP Example: ctdb uptime .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf Current time of node : Thu Oct 29 10:38:54 2009 Ctdbd start time : (000 16:54:28) Wed Oct 28 17:44:26 2009 Time of last recovery/failover: (000 16:53:31) Wed Oct 28 17:45:23 2009 -Duration of last recovery/failover: 2.248552 seconds +Duration of last recovery/failover: 2\&.248552 seconds .fi +.if n \{\ +.RE +.\} .SS "listnodes" .PP -This command shows lists the ip addresses of all the nodes in the cluster. +This command shows lists the ip addresses of all the nodes in the cluster\&. .PP Example: ctdb listnodes .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf -10.0.0.71 -10.0.0.72 -10.0.0.73 -10.0.0.74 +10\&.0\&.0\&.71 +10\&.0\&.0\&.72 +10\&.0\&.0\&.73 +10\&.0\&.0\&.74 .fi +.if n \{\ +.RE +.\} .SS "ping" .PP -This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly. +This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly\&. .PP Example: ctdb ping .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf -response from 0 time=0.000054 sec (3 clients) -response from 1 time=0.000144 sec (2 clients) -response from 2 time=0.000105 sec (2 clients) -response from 3 time=0.000114 sec (2 clients) +response from 0 time=0\&.000054 sec (3 clients) +response from 1 time=0\&.000144 sec (2 clients) +response from 2 time=0\&.000105 sec (2 clients) +response from 3 time=0\&.000114 sec (2 clients) .fi +.if n \{\ +.RE +.\} .SS "ifaces" .PP -This command will display the list of network interfaces, which could host public addresses, along with their status. +This command will display the list of network interfaces, which could host public addresses, along with their status\&. .PP Example: ctdb ifaces .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf Interfaces on node 0 name:eth5 link:up references:2 @@ -233,11 +311,17 @@ name:eth3 link:up references:1 name:eth2 link:up references:1 .fi +.if n \{\ +.RE +.\} .PP Example: ctdb ifaces \-Y .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf :Name:LinkStatus:References: :eth5:1:2 @@ -246,210 +330,264 @@ Example output: :eth2:1:1 .fi +.if n \{\ +.RE +.\} .SS "setifacelink <iface> <status>" .PP -This command will set the status of a network interface. The status needs to be "up" or "down". This is typically used in the 10.interfaces script in the "monitor" event. +This command will set the status of a network interface\&. The status needs to be "up" or "down"\&. This is typically used in the 10\&.interfaces script in the "monitor" event\&. .PP Example: ctdb setifacelink eth0 up .SS "ip" .PP -This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip. By default this command will ONLY show those public addresses that are known to the node itself. To see the full list of all public ips across the cluster you must use "ctdb ip \-n all". +This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip\&. By default this command will ONLY show those public addresses that are known to the node itself\&. To see the full list of all public ips across the cluster you must use "ctdb ip \-n all"\&. .PP Example: ctdb ip .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf Public IPs on node 0 -172.31.91.82 node[1] active[] available[eth2,eth3] configured[eth2,eth3] -172.31.91.83 node[0] active[eth3] available[eth2,eth3] configured[eth2,eth3] -172.31.91.84 node[1] active[] available[eth2,eth3] configured[eth2,eth3] -172.31.91.85 node[0] active[eth2] available[eth2,eth3] configured[eth2,eth3] -172.31.92.82 node[1] active[] available[eth5] configured[eth4,eth5] -172.31.92.83 node[0] active[eth5] available[eth5] configured[eth4,eth5] -172.31.92.84 node[1] active[] available[eth5] configured[eth4,eth5] -172.31.92.85 node[0] active[eth5] available[eth5] configured[eth4,eth5] +172\&.31\&.91\&.82 node[1] active[] available[eth2,eth3] configured[eth2,eth3] +172\&.31\&.91\&.83 node[0] active[eth3] available[eth2,eth3] configured[eth2,eth3] +172\&.31\&.91\&.84 node[1] active[] available[eth2,eth3] configured[eth2,eth3] +172\&.31\&.91\&.85 node[0] active[eth2] available[eth2,eth3] configured[eth2,eth3] +172\&.31\&.92\&.82 node[1] active[] available[eth5] configured[eth4,eth5] +172\&.31\&.92\&.83 node[0] active[eth5] available[eth5] configured[eth4,eth5] +172\&.31\&.92\&.84 node[1] active[] available[eth5] configured[eth4,eth5] +172\&.31\&.92\&.85 node[0] active[eth5] available[eth5] configured[eth4,eth5] .fi +.if n \{\ +.RE +.\} .PP Example: ctdb ip \-Y .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf :Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces: -:172.31.91.82:1::eth2,eth3:eth2,eth3: -:172.31.91.83:0:eth3:eth2,eth3:eth2,eth3: -:172.31.91.84:1::eth2,eth3:eth2,eth3: -:172.31.91.85:0:eth2:eth2,eth3:eth2,eth3: -:172.31.92.82:1::eth5:eth4,eth5: -:172.31.92.83:0:eth5:eth5:eth4,eth5: -:172.31.92.84:1::eth5:eth4,eth5: -:172.31.92.85:0:eth5:eth5:eth4,eth5: +:172\&.31\&.91\&.82:1::eth2,eth3:eth2,eth3: +:172\&.31\&.91\&.83:0:eth3:eth2,eth3:eth2,eth3: +:172\&.31\&.91\&.84:1::eth2,eth3:eth2,eth3: +:172\&.31\&.91\&.85:0:eth2:eth2,eth3:eth2,eth3: +:172\&.31\&.92\&.82:1::eth5:eth4,eth5: +:172\&.31\&.92\&.83:0:eth5:eth5:eth4,eth5: +:172\&.31\&.92\&.84:1::eth5:eth4,eth5: +:172\&.31\&.92\&.85:0:eth5:eth5:eth4,eth5: .fi +.if n \{\ +.RE +.\} .SS "ipinfo <ip>" .PP -This command will display details about the specified public addresses. +This command will display details about the specified public addresses\&. .PP -Example: ctdb ipinfo 172.31.92.85 +Example: ctdb ipinfo 172\&.31\&.92\&.85 .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf -Public IP[172.31.92.85] info on node 0 -IP:172.31.92.85 +Public IP[172\&.31\&.92\&.85] info on node 0 +IP:172\&.31\&.92\&.85 CurrentNode:0 NumInterfaces:2 Interface[1]: Name:eth4 Link:down References:0 Interface[2]: Name:eth5 Link:up References:2 (active) .fi +.if n \{\ +.RE +.\} .SS "scriptstatus" .PP -This command displays which scripts where run in the previous monitoring cycle and the result of each script. If a script failed with an error, causing the node to become unhealthy, the output from that script is also shown. +This command displays which scripts where run in the previous monitoring cycle and the result of each script\&. If a script failed with an error, causing the node to become unhealthy, the output from that script is also shown\&. .PP Example: ctdb scriptstatus .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf 7 scripts were executed last monitoring cycle -00.ctdb Status:OK Duration:0.056 Tue Mar 24 18:56:57 2009 -10.interface Status:OK Duration:0.077 Tue Mar 24 18:56:57 2009 -11.natgw Status:OK Duration:0.039 Tue Mar 24 18:56:57 2009 -20.multipathd Status:OK Duration:0.038 Tue Mar 24 18:56:57 2009 -31.clamd Status:DISABLED -40.vsftpd Status:OK Duration:0.045 Tue Mar 24 18:56:57 2009 -41.httpd Status:OK Duration:0.039 Tue Mar 24 18:56:57 2009 -50.samba Status:ERROR Duration:0.082 Tue Mar 24 18:56:57 2009 +00\&.ctdb Status:OK Duration:0\&.056 Tue Mar 24 18:56:57 2009 +10\&.interface Status:OK Duration:0\&.077 Tue Mar 24 18:56:57 2009 +11\&.natgw Status:OK Duration:0\&.039 Tue Mar 24 18:56:57 2009 +20\&.multipathd Status:OK Duration:0\&.038 Tue Mar 24 18:56:57 2009 +31\&.clamd Status:DISABLED +40\&.vsftpd Status:OK Duration:0\&.045 Tue Mar 24 18:56:57 2009 +41\&.httpd Status:OK Duration:0\&.039 Tue Mar 24 18:56:57 2009 +50\&.samba Status:ERROR Duration:0\&.082 Tue Mar 24 18:56:57 2009 OUTPUT:ERROR: Samba tcp port 445 is not responding .fi +.if n \{\ +.RE +.\} .SS "disablescript <script>" .PP -This command is used to disable an eventscript. +This command is used to disable an eventscript\&. .PP -This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in 'scriptstatus'. +This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in \*(Aqscriptstatus\*(Aq\&. .SS "enablescript <script>" .PP -This command is used to enable an eventscript. +This command is used to enable an eventscript\&. .PP -This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in 'scriptstatus'. +This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in \*(Aqscriptstatus\*(Aq\&. .SS "getvar <name>" .PP -Get the runtime value of a tuneable variable. +Get the runtime value of a tuneable variable\&. .PP Example: ctdb getvar MaxRedirectCount .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf MaxRedirectCount = 3 .fi +.if n \{\ +.RE +.\} .SS "setvar <name> <value>" .PP -Set the runtime value of a tuneable variable. +Set the runtime value of a tuneable variable\&. .PP Example: ctdb setvar MaxRedirectCount 5 .SS "listvars" .PP -List all tuneable variables. +List all tuneable variables, except the values of the obsolete tunables like VacuumMinInterval\&. The obsolete tunables can be retrieved only explicitly with the "ctdb getvar" command\&. .PP Example: ctdb listvars .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf -MaxRedirectCount = 3 -SeqnumInterval = 1000 -ControlTimeout = 60 -TraverseTimeout = 20 -KeepaliveInterval = 5 -KeepaliveLimit = 5 -RecoverTimeout = 20 -RecoverInterval = 1 -ElectionTimeout = 3 -TakeoverTimeout = 9 -MonitorInterval = 15 -TickleUpdateInterval = 20 -EventScriptTimeout = 30 +MaxRedirectCount = 3 +SeqnumInterval = 1000 +ControlTimeout = 60 +TraverseTimeout = 20 +KeepaliveInterval = 5 +KeepaliveLimit = 5 +RecoverTimeout = 20 +RecoverInterval = 1 +ElectionTimeout = 3 +TakeoverTimeout = 9 +MonitorInterval = 15 +TickleUpdateInterval = 20 +EventScriptTimeout = 30 EventScriptTimeoutCount = 1 -EventScriptUnhealthyOnTimeout = 0 -RecoveryGracePeriod = 120 -RecoveryBanPeriod = 300 -DatabaseHashSize = 100001 -DatabaseMaxDead = 5 -RerecoveryTimeout = 10 -EnableBans = 1 -DeterministicIPs = 1 -ReclockPingPeriod = 60 -NoIPFailback = 0 -DisableIPFailover = 0 -VerboseMemoryNames = 0 -RecdPingTimeout = 60 -RecdFailCount = 10 -LogLatencyMs = 0 -RecLockLatencyMs = 1000 -RecoveryDropAllIPs = 120 -VerifyRecoveryLock = 1 -VacuumDefaultInterval = 10 -VacuumMaxRunTime = 30 -RepackLimit = 10000 -VacuumLimit = 5000 -VacuumMinInterval = 10 -VacuumMaxInterval = 10 -VacuumFastPathCount = 60 -MaxQueueDropMsg = 1000000 -UseStatusEvents = 0 -AllowUnhealthyDBRead = 0 -StatHistoryInterval = 1 -DeferredAttachTO = 120 +RecoveryGracePeriod = 120 +RecoveryBanPeriod = 300 +DatabaseHashSize = 100001 +DatabaseMaxDead = 5 +RerecoveryTimeout = 10 +EnableBans = 1 +DeterministicIPs = 0 +LCP2PublicIPs = 1 +ReclockPingPeriod = 60 +NoIPFailback = 0 +DisableIPFailover = 0 +VerboseMemoryNames = 0 +RecdPingTimeout = 60 +RecdFailCount = 10 +LogLatencyMs = 0 +RecLockLatencyMs = 1000 +RecoveryDropAllIPs = 120 +VerifyRecoveryLock = 1 +VacuumInterval = 10 +VacuumMaxRunTime = 30 +RepackLimit = 10000 +VacuumLimit = 5000 +VacuumFastPathCount = 60 +MaxQueueDropMsg = 1000000 +UseStatusEvents = 0 +AllowUnhealthyDBRead = 0 +StatHistoryInterval = 1 +DeferredAttachTO = 120 +AllowClientDBAttach = 1 +RecoverPDBBySeqNum = 0 .fi +.if n \{\ +.RE +.\} .SS "lvsmaster" .PP -This command shows which node is currently the LVSMASTER. The LVSMASTER is the node in the cluster which drives the LVS system and which receives all incoming traffic from clients. +This command shows which node is currently the LVSMASTER\&. The LVSMASTER is the node in the cluster which drives the LVS system and which receives all incoming traffic from clients\&. .PP -LVS is the mode where the entire CTDB/Samba cluster uses a single ip address for the entire cluster. In this mode all clients connect to one specific node which will then multiplex/loadbalance the clients evenly onto the other nodes in the cluster. This is an alternative to using public ip addresses. See the manpage for ctdbd for more information about LVS. +LVS is the mode where the entire CTDB/Samba cluster uses a single ip address for the entire cluster\&. In this mode all clients connect to one specific node which will then multiplex/loadbalance the clients evenly onto the other nodes in the cluster\&. This is an alternative to using public ip addresses\&. See the manpage for ctdbd for more information about LVS\&. .SS "lvs" .PP -This command shows which nodes in the cluster are currently active in the LVS configuration. I.e. which nodes we are currently loadbalancing the single ip address across. +This command shows which nodes in the cluster are currently active in the LVS configuration\&. I\&.e\&. which nodes we are currently loadbalancing the single ip address across\&. .PP -LVS will by default only loadbalance across those nodes that are both LVS capable and also HEALTHY. Except if all nodes are UNHEALTHY in which case LVS will loadbalance across all UNHEALTHY nodes as well. LVS will never use nodes that are DISCONNECTED, STOPPED, BANNED or DISABLED. +LVS will by default only loadbalance across those nodes that are both LVS capable and also HEALTHY\&. Except if all nodes are UNHEALTHY in which case LVS will loadbalance across all UNHEALTHY nodes as well\&. LVS will never use nodes that are DISCONNECTED, STOPPED, BANNED or DISABLED\&. .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf -2:10.0.0.13 -3:10.0.0.14 +2:10\&.0\&.0\&.13 +3:10\&.0\&.0\&.14 .fi +.if n \{\ +.RE +.\} .SS "getcapabilities" .PP -This command shows the capabilities of the current node. Please see manpage for ctdbd for a full list of all capabilities and more detailed description. +This command shows the capabilities of the current node\&. Please see manpage for ctdbd for a full list of all capabilities and more detailed description\&. .PP -RECMASTER and LMASTER capabilities are primarily used when CTDBD is used to create a cluster spanning across WAN links. In which case ctdbd acts as a WAN accelerator. +RECMASTER and LMASTER capabilities are primarily used when CTDBD is used to create a cluster spanning across WAN links\&. In which case ctdbd acts as a WAN accelerator\&. .PP -LVS capabile means that the node is participating in LVS, a mode where the entire CTDB cluster uses one single ip address for the entire cluster instead of using public ip address failover. This is an alternative to using a loadbalancing layer\-4 switch. +LVS capabile means that the node is participating in LVS, a mode where the entire CTDB cluster uses one single ip address for the entire cluster instead of using public ip address failover\&. This is an alternative to using a loadbalancing layer\-4 switch\&. .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf RECMASTER: YES LMASTER: YES LVS: NO .fi +.if n \{\ +.RE +.\} .SS "statistics" .PP -Collect statistics from the CTDB daemon about how many calls it has served. +Collect statistics from the CTDB daemon about how many calls it has served\&. .PP Example: ctdb statistics .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf CTDB version 1 num_clients 3 @@ -484,74 +622,83 @@ CTDB version 1 pending_lockwait_calls 0 memory_used 5040 max_hop_count 0 - max_call_latency 4.948321 sec - max_lockwait_latency 0.000000 sec + max_call_latency 4\&.948321 sec + max_lockwait_latency 0\&.000000 sec .fi +.if n \{\ +.RE +.\} .SS "statisticsreset" .PP -This command is used to clear all statistics counters in a node. +This command is used to clear all statistics counters in a node\&. .PP Example: ctdb statisticsreset .SS "getreclock" .PP -This command is used to show the filename of the reclock file that is used. +This command is used to show the filename of the reclock file that is used\&. .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf -Reclock file:/gpfs/.ctdb/shared +Reclock file:/gpfs/\&.ctdb/shared .fi +.if n \{\ +.RE +.\} .SS "setreclock [filename]" .PP -This command is used to modify, or clear, the file that is used as the reclock file at runtime. When this command is used, the reclock file checks are disabled. To re\-enable the checks the administrator needs to activate the "VerifyRecoveryLock" tunable using "ctdb setvar". +This command is used to modify, or clear, the file that is used as the reclock file at runtime\&. When this command is used, the reclock file checks are disabled\&. To re\-enable the checks the administrator needs to activate the "VerifyRecoveryLock" tunable using "ctdb setvar"\&. .PP -If run with no parameter this will remove the reclock file completely. If run with a parameter the parameter specifies the new filename to use for the recovery lock. +If run with no parameter this will remove the reclock file completely\&. If run with a parameter the parameter specifies the new filename to use for the recovery lock\&. .PP -This command only affects the runtime settings of a ctdb node and will be lost when ctdb is restarted. For persistent changes to the reclock file setting you must edit /etc/sysconfig/ctdb. +This command only affects the runtime settings of a ctdb node and will be lost when ctdb is restarted\&. For persistent changes to the reclock file setting you must edit /etc/sysconfig/ctdb\&. .SS "getdebug" .PP -Get the current debug level for the node. the debug level controls what information is written to the log file. +Get the current debug level for the node\&. the debug level controls what information is written to the log file\&. .PP -The debug levels are mapped to the corresponding syslog levels. When a debug level is set, only those messages at that level and higher levels will be printed. +The debug levels are mapped to the corresponding syslog levels\&. When a debug level is set, only those messages at that level and higher levels will be printed\&. .PP The list of debug levels from highest to lowest are : .PP EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG .SS "setdebug <debuglevel>" .PP -Set the debug level of a node. This controls what information will be logged. +Set the debug level of a node\&. This controls what information will be logged\&. .PP The debuglevel is one of EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG .SS "getpid" .PP -This command will return the process id of the ctdb daemon. +This command will return the process id of the ctdb daemon\&. .SS "disable" .PP -This command is used to administratively disable a node in the cluster. A disabled node will still participate in the cluster and host clustered TDB records but its public ip address has been taken over by a different node and it no longer hosts any services. +This command is used to administratively disable a node in the cluster\&. A disabled node will still participate in the cluster and host clustered TDB records but its public ip address has been taken over by a different node and it no longer hosts any services\&. .SS "enable" .PP -Re\-enable a node that has been administratively disabled. +Re\-enable a node that has been administratively disabled\&. .SS "stop" .PP -This command is used to administratively STOP a node in the cluster. A STOPPED node is connected to the cluster but will not host any public ip addresse, nor does it participate in the VNNMAP. The difference between a DISABLED node and a STOPPED node is that a STOPPED node does not host any parts of the database which means that a recovery is required to stop/continue nodes. +This command is used to administratively STOP a node in the cluster\&. A STOPPED node is connected to the cluster but will not host any public ip addresse, nor does it participate in the VNNMAP\&. The difference between a DISABLED node and a STOPPED node is that a STOPPED node does not host any parts of the database which means that a recovery is required to stop/continue nodes\&. .SS "continue" .PP -Re\-start a node that has been administratively stopped. +Re\-start a node that has been administratively stopped\&. .SS "addip <public_ip/mask> <iface>" .PP -This command is used to add a new public ip to a node during runtime. This allows public addresses to be added to a cluster without having to restart the ctdb daemons. +This command is used to add a new public ip to a node during runtime\&. This allows public addresses to be added to a cluster without having to restart the ctdb daemons\&. .PP -Note that this only updates the runtime instance of ctdb. Any changes will be lost next time ctdb is restarted and the public addresses file is re\-read. If you want this change to be permanent you must also update the public addresses file manually. +Note that this only updates the runtime instance of ctdb\&. Any changes will be lost next time ctdb is restarted and the public addresses file is re\-read\&. If you want this change to be permanent you must also update the public addresses file manually\&. .SS "delip <public_ip>" .PP -This command is used to remove a public ip from a node during runtime. If this public ip is currently hosted by the node it being removed from, the ip will first be failed over to another node, if possible, before it is removed. +This command is used to remove a public ip from a node during runtime\&. If this public ip is currently hosted by the node it being removed from, the ip will first be failed over to another node, if possible, before it is removed\&. .PP -Note that this only updates the runtime instance of ctdb. Any changes will be lost next time ctdb is restarted and the public addresses file is re\-read. If you want this change to be permanent you must also update the public addresses file manually. +Note that this only updates the runtime instance of ctdb\&. Any changes will be lost next time ctdb is restarted and the public addresses file is re\-read\&. If you want this change to be permanent you must also update the public addresses file manually\&. .SS "moveip <public_ip> <node>" .PP -This command can be used to manually fail a public ip address to a specific node. +This command can be used to manually fail a public ip address to a specific node\&. .PP In order to manually override the "automatic" distribution of public ip addresses that ctdb normally provides, this command only works when you have changed the tunables for the daemon to: .PP @@ -560,251 +707,284 @@ DeterministicIPs = 0 NoIPFailback = 1 .SS "shutdown" .PP -This command will shutdown a specific CTDB daemon. +This command will shutdown a specific CTDB daemon\&. .SS "recover" .PP -This command will trigger the recovery daemon to do a cluster recovery. +This command will trigger the recovery daemon to do a cluster recovery\&. .SS "ipreallocate" .PP -This command will force the recovery master to perform a full ip reallocation process and redistribute all ip addresses. This is useful to "reset" the allocations back to its default state if they have been changed using the "moveip" command. While a "recover" will also perform this reallocation, a recovery is much more hevyweight since it will also rebuild all the databases. +This command will force the recovery master to perform a full ip reallocation process and redistribute all ip addresses\&. This is useful to "reset" the allocations back to its default state if they have been changed using the "moveip" command\&. While a "recover" will also perform this reallocation, a recovery is much more hevyweight since it will also rebuild all the databases\&. .SS "setlmasterrole <on|off>" .PP -This command is used ot enable/disable the LMASTER capability for a node at runtime. This capability determines whether or not a node can be used as an LMASTER for records in the database. A node that does not have the LMASTER capability will not show up in the vnnmap. +This command is used ot enable/disable the LMASTER capability for a node at runtime\&. This capability determines whether or not a node can be used as an LMASTER for records in the database\&. A node that does not have the LMASTER capability will not show up in the vnnmap\&. .PP -Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command. +Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command\&. .PP -Once this setting has been enabled/disabled, you need to perform a recovery for it to take effect. +Once this setting has been enabled/disabled, you need to perform a recovery for it to take effect\&. .PP See also "ctdb getcapabilities" .SS "setrecmasterrole <on|off>" .PP -This command is used ot enable/disable the RECMASTER capability for a node at runtime. This capability determines whether or not a node can be used as an RECMASTER for the cluster. A node that does not have the RECMASTER capability can not win a recmaster election. A node that already is the recmaster for the cluster when the capability is stripped off the node will remain the recmaster until the next cluster election. +This command is used ot enable/disable the RECMASTER capability for a node at runtime\&. This capability determines whether or not a node can be used as an RECMASTER for the cluster\&. A node that does not have the RECMASTER capability can not win a recmaster election\&. A node that already is the recmaster for the cluster when the capability is stripped off the node will remain the recmaster until the next cluster election\&. .PP -Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command. +Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command\&. .PP See also "ctdb getcapabilities" .SS "killtcp <srcip:port> <dstip:port>" .PP -This command will kill the specified TCP connection by issuing a TCP RST to the srcip:port endpoint. This is a command used by the ctdb eventscripts. +This command will kill the specified TCP connection by issuing a TCP RST to the srcip:port endpoint\&. This is a command used by the ctdb eventscripts\&. .SS "gratiousarp <ip> <interface>" .PP -This command will send out a gratious arp for the specified interface through the specified interface. This command is mainly used by the ctdb eventscripts. +This command will send out a gratious arp for the specified interface through the specified interface\&. This command is mainly used by the ctdb eventscripts\&. .SS "reloadnodes" .PP -This command is used when adding new nodes, or removing existing nodes from an existing cluster. +This command is used when adding new nodes, or removing existing nodes from an existing cluster\&. .PP Procedure to add a node: .PP -1, To expand an existing cluster, first ensure with 'ctdb status' that all nodes are up and running and that they are all healthy. Do not try to expand a cluster unless it is completely healthy! +1, To expand an existing cluster, first ensure with \*(Aqctdb status\*(Aq that all nodes are up and running and that they are all healthy\&. Do not try to expand a cluster unless it is completely healthy! .PP -2, On all nodes, edit /etc/ctdb/nodes and add the new node as the last entry to the file. The new node MUST be added to the end of this file! +2, On all nodes, edit /etc/ctdb/nodes and add the new node as the last entry to the file\&. The new node MUST be added to the end of this file! .PP 3, Verify that all the nodes have identical /etc/ctdb/nodes files after you edited them and added the new node! .PP -4, Run 'ctdb reloadnodes' to force all nodes to reload the nodesfile. +4, Run \*(Aqctdb reloadnodes\*(Aq to force all nodes to reload the nodesfile\&. .PP -5, Use 'ctdb status' on all nodes and verify that they now show the additional node. +5, Use \*(Aqctdb status\*(Aq on all nodes and verify that they now show the additional node\&. .PP -6, Install and configure the new node and bring it online. +6, Install and configure the new node and bring it online\&. .PP Procedure to remove a node: .PP -1, To remove a node from an existing cluster, first ensure with 'ctdb status' that all nodes, except the node to be deleted, are up and running and that they are all healthy. Do not try to remove a node from a cluster unless the cluster is completely healthy! +1, To remove a node from an existing cluster, first ensure with \*(Aqctdb status\*(Aq that all nodes, except the node to be deleted, are up and running and that they are all healthy\&. Do not try to remove a node from a cluster unless the cluster is completely healthy! .PP -2, Shutdown and poweroff the node to be removed. +2, Shutdown and poweroff the node to be removed\&. .PP -3, On all other nodes, edit the /etc/ctdb/nodes file and comment out the node to be removed. Do not delete the line for that node, just comment it out by adding a '#' at the beginning of the line. +3, On all other nodes, edit the /etc/ctdb/nodes file and comment out the node to be removed\&. Do not delete the line for that node, just comment it out by adding a \*(Aq#\*(Aq at the beginning of the line\&. .PP -4, Run 'ctdb reloadnodes' to force all nodes to reload the nodesfile. +4, Run \*(Aqctdb reloadnodes\*(Aq to force all nodes to reload the nodesfile\&. .PP -5, Use 'ctdb status' on all nodes and verify that the deleted node no longer shows up in the list.. +5, Use \*(Aqctdb status\*(Aq on all nodes and verify that the deleted node no longer shows up in the list\&.\&. .PP .SS "tickle <srcip:port> <dstip:port>" .PP -This command will will send a TCP tickle to the source host for the specified TCP connection. A TCP tickle is a TCP ACK packet with an invalid sequence and acknowledge number and will when received by the source host result in it sending an immediate correct ACK back to the other end. +This command will will send a TCP tickle to the source host for the specified TCP connection\&. A TCP tickle is a TCP ACK packet with an invalid sequence and acknowledge number and will when received by the source host result in it sending an immediate correct ACK back to the other end\&. .PP -TCP tickles are useful to "tickle" clients after a IP failover has occured since this will make the client immediately recognize the TCP connection has been disrupted and that the client will need to reestablish. This greatly speeds up the time it takes for a client to detect and reestablish after an IP failover in the ctdb cluster. +TCP tickles are useful to "tickle" clients after a IP failover has occured since this will make the client immediately recognize the TCP connection has been disrupted and that the client will need to reestablish\&. This greatly speeds up the time it takes for a client to detect and reestablish after an IP failover in the ctdb cluster\&. .SS "gettickles <ip>" .PP -This command is used to show which TCP connections are registered with CTDB to be "tickled" if there is a failover. +This command is used to show which TCP connections are registered with CTDB to be "tickled" if there is a failover\&. .SS "repack [max_freelist]" .PP -Over time, when records are created and deleted in a TDB, the TDB list of free space will become fragmented. This can lead to a slowdown in accessing TDB records. This command is used to defragment a TDB database and pruning the freelist. +Over time, when records are created and deleted in a TDB, the TDB list of free space will become fragmented\&. This can lead to a slowdown in accessing TDB records\&. This command is used to defragment a TDB database and pruning the freelist\&. .PP -If [max_freelist] is specified, then a database will only be repacked if it has more than this number of entries in the freelist. +If [max_freelist] is specified, then a database will only be repacked if it has more than this number of entries in the freelist\&. .PP -During repacking of the database, the entire TDB database will be locked to prevent writes. If samba tries to write to a record in the database during a repack operation, samba will block until the repacking has completed. +During repacking of the database, the entire TDB database will be locked to prevent writes\&. If samba tries to write to a record in the database during a repack operation, samba will block until the repacking has completed\&. .PP -This command can be disruptive and can cause samba to block for the duration of the repack operation. In general, a repack operation will take less than one second to complete. +This command can be disruptive and can cause samba to block for the duration of the repack operation\&. In general, a repack operation will take less than one second to complete\&. .PP -A repack operation will only defragment the local TDB copy of the CTDB database. You need to run this command on all of the nodes to repack a CTDB database completely. +A repack operation will only defragment the local TDB copy of the CTDB database\&. You need to run this command on all of the nodes to repack a CTDB database completely\&. .PP Example: ctdb repack 1000 .PP -By default, this operation is issued from the 00.ctdb event script every 5 minutes. +By default, this operation is issued from the 00\&.ctdb event script every 5 minutes\&. .SS "vacuum [max_records]" .PP -Over time CTDB databases will fill up with empty deleted records which will lead to a progressive slow down of CTDB database access. This command is used to prune all databases and delete all empty records from the cluster. +Over time CTDB databases will fill up with empty deleted records which will lead to a progressive slow down of CTDB database access\&. This command is used to prune all databases and delete all empty records from the cluster\&. .PP -By default, vacuum will delete all empty records from all databases. If [max_records] is specified, the command will only delete the first [max_records] empty records for each database. +By default, vacuum will delete all empty records from all databases\&. If [max_records] is specified, the command will only delete the first [max_records] empty records for each database\&. .PP -Vacuum only deletes records where the local node is the lmaster. To delete all records from the entire cluster you need to run a vacuum from each node. This command is not disruptive. Samba is unaffected and will still be able to read/write records normally while the database is being vacuumed. +Vacuum only deletes records where the local node is the lmaster\&. To delete all records from the entire cluster you need to run a vacuum from each node\&. This command is not disruptive\&. Samba is unaffected and will still be able to read/write records normally while the database is being vacuumed\&. .PP Example: ctdb vacuum .PP -By default, this operation is issued from the 00.ctdb event script every 5 minutes. +By default, this operation is issued from the 00\&.ctdb event script every 5 minutes\&. .SS "backupdb <dbname> <file>" .PP -This command can be used to copy the entire content of a database out to a file. This file can later be read back into ctdb using the restoredb command. This is mainly useful for backing up persistent databases such as secrets.tdb and similar. +This command can be used to copy the entire content of a database out to a file\&. This file can later be read back into ctdb using the restoredb command\&. This is mainly useful for backing up persistent databases such as secrets\&.tdb and similar\&. .SS "restoredb <file> [<dbname>]" .PP -This command restores a persistent database that was previously backed up using backupdb. By default the data will be restored back into the same database as it was created from. By specifying dbname you can restore the data into a different database. +This command restores a persistent database that was previously backed up using backupdb\&. By default the data will be restored back into the same database as it was created from\&. By specifying dbname you can restore the data into a different database\&. .SS "wipedb <dbname>" .PP -This command can be used to remove all content of a database. +This command can be used to remove all content of a database\&. .SS "getlog <level>" .PP -In addition to the normal loggign to a log file, CTDBD also keeps a in\-memory ringbuffer containing the most recent log entries for all log levels (except DEBUG). +In addition to the normal loggign to a log file, CTDBD also keeps a in\-memory ringbuffer containing the most recent log entries for all log levels (except DEBUG)\&. .PP -This is useful since it allows for keeping continuous logs to a file at a reasonable non\-verbose level, but shortly after an incident has occured, a much more detailed log can be pulled from memory. This can allow you to avoid having to reproduce an issue due to the on\-disk logs being of insufficient detail. +This is useful since it allows for keeping continuous logs to a file at a reasonable non\-verbose level, but shortly after an incident has occured, a much more detailed log can be pulled from memory\&. This can allow you to avoid having to reproduce an issue due to the on\-disk logs being of insufficient detail\&. .PP -This command extracts all messages of level or lower log level from memory and prints it to the screen. +This command extracts all messages of level or lower log level from memory and prints it to the screen\&. .SS "clearlog" .PP -This command clears the in\-memory logging ringbuffer. +This command clears the in\-memory logging ringbuffer\&. .SH "DEBUGGING COMMANDS" .PP -These commands are primarily used for CTDB development and testing and should not be used for normal administration. +These commands are primarily used for CTDB development and testing and should not be used for normal administration\&. .SS "process\-exists <pid>" .PP -This command checks if a specific process exists on the CTDB host. This is mainly used by Samba to check if remote instances of samba are still running or not. +This command checks if a specific process exists on the CTDB host\&. This is mainly used by Samba to check if remote instances of samba are still running or not\&. .SS "getdbmap" .PP -This command lists all clustered TDB databases that the CTDB daemon has attached to. Some databases are flagged as PERSISTENT, this means that the database stores data persistently and the data will remain across reboots. One example of such a database is secrets.tdb where information about how the cluster was joined to the domain is stored. +This command lists all clustered TDB databases that the CTDB daemon has attached to\&. Some databases are flagged as PERSISTENT, this means that the database stores data persistently and the data will remain across reboots\&. One example of such a database is secrets\&.tdb where information about how the cluster was joined to the domain is stored\&. .PP -If a PERSISTENT database is not in a healthy state the database is flagged as UNHEALTHY. If there's at least one completely healthy node running in the cluster, it's possible that the content is restored by a recovery run automaticly. Otherwise an administrator needs to analyze the problem. +If a PERSISTENT database is not in a healthy state the database is flagged as UNHEALTHY\&. If there\*(Aqs at least one completely healthy node running in the cluster, it\*(Aqs possible that the content is restored by a recovery run automaticly\&. Otherwise an administrator needs to analyze the problem\&. .PP -See also "ctdb getdbstatus", "ctdb backupdb", "ctdb restoredb", "ctdb dumpbackup", "ctdb wipedb", "ctdb setvar AllowUnhealthyDBRead 1" and (if samba or tdb\-utils are installed) "tdbtool check". +See also "ctdb getdbstatus", "ctdb backupdb", "ctdb restoredb", "ctdb dumpbackup", "ctdb wipedb", "ctdb setvar AllowUnhealthyDBRead 1" and (if samba or tdb\-utils are installed) "tdbtool check"\&. .PP -Most databases are not persistent and only store the state information that the currently running samba daemons need. These databases are always wiped when ctdb/samba starts and when a node is rebooted. +Most databases are not persistent and only store the state information that the currently running samba daemons need\&. These databases are always wiped when ctdb/samba starts and when a node is rebooted\&. .PP Example: ctdb getdbmap .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf Number of databases:10 -dbid:0x435d3410 name:notify.tdb path:/var/ctdb/notify.tdb.0 -dbid:0x42fe72c5 name:locking.tdb path:/var/ctdb/locking.tdb.0 -dbid:0x1421fb78 name:brlock.tdb path:/var/ctdb/brlock.tdb.0 -dbid:0x17055d90 name:connections.tdb path:/var/ctdb/connections.tdb.0 -dbid:0xc0bdde6a name:sessionid.tdb path:/var/ctdb/sessionid.tdb.0 -dbid:0x122224da name:test.tdb path:/var/ctdb/test.tdb.0 -dbid:0x2672a57f name:idmap2.tdb path:/var/ctdb/persistent/idmap2.tdb.0 PERSISTENT -dbid:0xb775fff6 name:secrets.tdb path:/var/ctdb/persistent/secrets.tdb.0 PERSISTENT -dbid:0xe98e08b6 name:group_mapping.tdb path:/var/ctdb/persistent/group_mapping.tdb.0 PERSISTENT -dbid:0x7bbbd26c name:passdb.tdb path:/var/ctdb/persistent/passdb.tdb.0 PERSISTENT +dbid:0x435d3410 name:notify\&.tdb path:/var/ctdb/notify\&.tdb\&.0 +dbid:0x42fe72c5 name:locking\&.tdb path:/var/ctdb/locking\&.tdb\&.0 +dbid:0x1421fb78 name:brlock\&.tdb path:/var/ctdb/brlock\&.tdb\&.0 +dbid:0x17055d90 name:connections\&.tdb path:/var/ctdb/connections\&.tdb\&.0 +dbid:0xc0bdde6a name:sessionid\&.tdb path:/var/ctdb/sessionid\&.tdb\&.0 +dbid:0x122224da name:test\&.tdb path:/var/ctdb/test\&.tdb\&.0 +dbid:0x2672a57f name:idmap2\&.tdb path:/var/ctdb/persistent/idmap2\&.tdb\&.0 PERSISTENT +dbid:0xb775fff6 name:secrets\&.tdb path:/var/ctdb/persistent/secrets\&.tdb\&.0 PERSISTENT +dbid:0xe98e08b6 name:group_mapping\&.tdb path:/var/ctdb/persistent/group_mapping\&.tdb\&.0 PERSISTENT +dbid:0x7bbbd26c name:passdb\&.tdb path:/var/ctdb/persistent/passdb\&.tdb\&.0 PERSISTENT .fi +.if n \{\ +.RE +.\} .PP Example output for an unhealthy database: .sp +.if n \{\ +.RS 4 +.\} .nf Number of databases:1 -dbid:0xb775fff6 name:secrets.tdb path:/var/ctdb/persistent/secrets.tdb.0 PERSISTENT UNHEALTHY +dbid:0xb775fff6 name:secrets\&.tdb path:/var/ctdb/persistent/secrets\&.tdb\&.0 PERSISTENT UNHEALTHY .fi +.if n \{\ +.RE +.\} .PP Example output for a healthy database as machinereadable output \-Y: .sp +.if n \{\ +.RS 4 +.\} .nf :ID:Name:Path:Persistent:Unhealthy: -:0x7bbbd26c:passdb.tdb:/var/ctdb/persistent/passdb.tdb.0:1:0: +:0x7bbbd26c:passdb\&.tdb:/var/ctdb/persistent/passdb\&.tdb\&.0:1:0: .fi +.if n \{\ +.RE +.\} .SS "getdbstatus <dbname>" .PP -This command displays more details about a database. +This command displays more details about a database\&. .PP -Example: ctdb getdbstatus test.tdb.0 +Example: ctdb getdbstatus test\&.tdb\&.0 .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf dbid: 0x122224da -name: test.tdb -path: /var/ctdb/test.tdb.0 +name: test\&.tdb +path: /var/ctdb/test\&.tdb\&.0 PERSISTENT: no HEALTH: OK .fi +.if n \{\ +.RE +.\} .PP -Example: ctdb getdbstatus registry.tdb (with a corrupted TDB) +Example: ctdb getdbstatus registry\&.tdb (with a corrupted TDB) .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf dbid: 0xf2a58948 -name: registry.tdb -path: /var/ctdb/persistent/registry.tdb.0 +name: registry\&.tdb +path: /var/ctdb/persistent/registry\&.tdb\&.0 PERSISTENT: yes -HEALTH: NO\-HEALTHY\-NODES \- ERROR \- Backup of corrupted TDB in '/var/ctdb/persistent/registry.tdb.0.corrupted.20091208091949.0Z' +HEALTH: NO\-HEALTHY\-NODES \- ERROR \- Backup of corrupted TDB in \*(Aq/var/ctdb/persistent/registry\&.tdb\&.0\&.corrupted\&.20091208091949\&.0Z\*(Aq .fi +.if n \{\ +.RE +.\} .SS "catdb <dbname>" .PP -This command will dump a clustered TDB database to the screen. This is a debugging command. +This command will dump a clustered TDB database to the screen\&. This is a debugging command\&. .SS "cattdb <dbname>" .PP -This command will dump the content of the local TDB database to the screen. This is a debugging command. +This command will dump the content of the local TDB database to the screen\&. This is a debugging command\&. .SS "dumpdbbackup <backup\-file>" .PP -This command will dump the content of database backup to the screen (similar to ctdb catdb). This is a debugging command. +This command will dump the content of database backup to the screen (similar to ctdb catdb)\&. This is a debugging command\&. .SS "getmonmode" .PP -This command returns the monutoring mode of a node. The monitoring mode is either ACTIVE or DISABLED. Normally a node will continuously monitor that all other nodes that are expected are in fact connected and that they respond to commands. +This command returns the monutoring mode of a node\&. The monitoring mode is either ACTIVE or DISABLED\&. Normally a node will continuously monitor that all other nodes that are expected are in fact connected and that they respond to commands\&. .PP -ACTIVE \- This is the normal mode. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster. +ACTIVE \- This is the normal mode\&. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands\&. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster\&. .PP -DISABLED \- This node is not monitoring that other nodes are available. In this mode a node failure will not be detected and no recovery will be performed. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery. +DISABLED \- This node is not monitoring that other nodes are available\&. In this mode a node failure will not be detected and no recovery will be performed\&. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery\&. .SS "setmonmode <0|1>" .PP -This command can be used to explicitly disable/enable monitoring mode on a node. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery. To do this, set monitoring mode to 0 on all nodes before attaching with GDB. Remember to set monitoring mode back to 1 afterwards. +This command can be used to explicitly disable/enable monitoring mode on a node\&. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery\&. To do this, set monitoring mode to 0 on all nodes before attaching with GDB\&. Remember to set monitoring mode back to 1 afterwards\&. .SS "attach <dbname> [persistent]" .PP -This is a debugging command. This command will make the CTDB daemon create a new CTDB database and attach to it. +This is a debugging command\&. This command will make the CTDB daemon create a new CTDB database and attach to it\&. .SS "dumpmemory" .PP -This is a debugging command. This command will make the ctdb daemon to write a fill memory allocation map to standard output. +This is a debugging command\&. This command will make the ctdb daemon to write a fill memory allocation map to standard output\&. .SS "rddumpmemory" .PP -This is a debugging command. This command will dump the talloc memory allocation tree for the recovery daemon to standard output. +This is a debugging command\&. This command will dump the talloc memory allocation tree for the recovery daemon to standard output\&. .SS "thaw" .PP -Thaw a previously frozen node. +Thaw a previously frozen node\&. .SS "eventscript <arguments>" .PP -This is a debugging command. This command can be used to manually invoke and run the eventscritps with arbitrary arguments. +This is a debugging command\&. This command can be used to manually invoke and run the eventscritps with arbitrary arguments\&. .SS "ban <bantime|0>" .PP -Administratively ban a node for bantime seconds. A bantime of 0 means that the node should be permanently banned. +Administratively ban a node for bantime seconds\&. A bantime of 0 means that the node should be permanently banned\&. .PP -A banned node does not participate in the cluster and does not host any records for the clustered TDB. Its ip address has been taken over by another node and no services are hosted. +A banned node does not participate in the cluster and does not host any records for the clustered TDB\&. Its ip address has been taken over by another node and no services are hosted\&. .PP -Nodes are automatically banned if they are the cause of too many cluster recoveries. +Nodes are automatically banned if they are the cause of too many cluster recoveries\&. .PP -This is primarily a testing command. Note that the recovery daemon controls the overall ban state and it may automatically unban nodes at will. Meaning that a node that has been banned by the administrator can and ofter are unbanned before the admin specifid timeout triggers. If wanting to "drop" a node out from the cluster for mainentance or other reasons, use the "stop" / "continue" commands instad of "ban" / "unban". +This is primarily a testing command\&. Note that the recovery daemon controls the overall ban state and it may automatically unban nodes at will\&. Meaning that a node that has been banned by the administrator can and ofter are unbanned before the admin specifid timeout triggers\&. If wanting to "drop" a node out from the cluster for mainentance or other reasons, use the "stop" / "continue" commands instad of "ban" / "unban"\&. .SS "unban" .PP -This command is used to unban a node that has either been administratively banned using the ban command or has been automatically banned by the recovery daemon. -.SS "check_srvids <srvid> ..." +This command is used to unban a node that has either been administratively banned using the ban command or has been automatically banned by the recovery daemon\&. +.SS "check_srvids <srvid> \&.\&.\&." .PP -This command checks whether a set of srvid message ports are registered on the node or not. The command takes a list of values to check. +This command checks whether a set of srvid message ports are registered on the node or not\&. The command takes a list of values to check\&. .PP Example: ctdb check_srvids 1 2 3 14765 .PP Example output: .sp +.if n \{\ +.RS 4 +.\} .nf Server id 0:1 does not exist Server id 0:2 does not exist @@ -812,12 +992,18 @@ Server id 0:3 does not exist Server id 0:14765 exists .fi +.if n \{\ +.RE +.\} .SH "SEE ALSO" .PP ctdbd(1), onnode(1) -\fI\%http://ctdb.samba.org/\fR +\m[blue]\fB\%http://ctdb.samba.org/\fR\m[] .SH "COPYRIGHT/LICENSE" .sp +.if n \{\ +.RS 4 +.\} .nf Copyright (C) Andrew Tridgell 2007 Copyright (C) Ronnie sahlberg 2007 @@ -825,13 +1011,16 @@ Copyright (C) Ronnie sahlberg 2007 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at -your option) any later version. +your option) any later version\&. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -General Public License for more details. +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE\&. See the GNU +General Public License for more details\&. You should have received a copy of the GNU General Public License -along with this program; if not, see http://www.gnu.org/licenses/. +along with this program; if not, see http://www\&.gnu\&.org/licenses/\&. .fi +.if n \{\ +.RE +.\} diff --git a/ctdb/doc/ctdb.1.html b/ctdb/doc/ctdb.1.html index 0366491dc4..e18131ffc0 100644 --- a/ctdb/doc/ctdb.1.html +++ b/ctdb/doc/ctdb.1.html @@ -1,6 +1,6 @@ -<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>ctdb</title><meta name="generator" content="DocBook XSL Stylesheets V1.69.1"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" lang="en"><a name="ctdb.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>ctdb — clustered tdb database management utility</p></div><div class="refsynopsisdiv"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">ctdb [ OPTIONS ] COMMAND ...</code> </p></div><div class="cmdsynopsis"><p><code class="command">ctdb</code> [-n <node>] [-Y] [-t <timeout>] [-T <timelimit>] [-? --help] [--usage] [-d --debug=<INTEGER>] [--socket=<filename>] [--print-emptyrecords] [--print-datasize] [--print-lmaster] [--print-hash] [--print-recordflags]</p></div></div><div class="refsect1" lang="en"><a name="id3243409"></a><h2>DESCRIPTION</h2><p> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>ctdb</title><meta name="generator" content="DocBook XSL Stylesheets V1.75.2"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" title="ctdb"><a name="ctdb.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>ctdb — clustered tdb database management utility</p></div><div class="refsynopsisdiv" title="Synopsis"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">ctdb [ OPTIONS ] COMMAND ...</code> </p></div><div class="cmdsynopsis"><p><code class="command">ctdb</code> [-n <node>] [-Y] [-t <timeout>] [-T <timelimit>] [-? --help] [--usage] [-d --debug=<INTEGER>] [--socket=<filename>] [--print-emptyrecords] [--print-datasize] [--print-lmaster] [--print-hash] [--print-recordflags]</p></div></div><div class="refsect1" title="DESCRIPTION"><a name="id428070"></a><h2>DESCRIPTION</h2><p> ctdb is a utility to view and manage a ctdb cluster. - </p></div><div class="refsect1" lang="en"><a name="id3243418"></a><h2>OPTIONS</h2><div class="variablelist"><dl><dt><span class="term">-n <pnn></span></dt><dd><p> + </p></div><div class="refsect1" title="OPTIONS"><a name="id428079"></a><h2>OPTIONS</h2><div class="variablelist"><dl><dt><span class="term">-n <pnn></span></dt><dd><p> This specifies the physical node number on which to execute the command. Default is to run the command on the daemon running on the local host. @@ -46,13 +46,13 @@ This lets catdb and dumpdbbackup print the record flags for each record. Note that cattdb always prints the flags. - </p></dd></dl></div></div><div class="refsect1" lang="en"><a name="id3243607"></a><h2>Administrative Commands</h2><p> + </p></dd></dl></div></div><div class="refsect1" title="Administrative Commands"><a name="id428271"></a><h2>Administrative Commands</h2><p> These are commands used to monitor and administrate a CTDB cluster. - </p><div class="refsect2" lang="en"><a name="id3243615"></a><h3>pnn</h3><p> + </p><div class="refsect2" title="pnn"><a name="id428278"></a><h3>pnn</h3><p> This command displays the pnn of the current node. - </p></div><div class="refsect2" lang="en"><a name="id3243625"></a><h3>status</h3><p> + </p></div><div class="refsect2" title="status"><a name="id428287"></a><h3>status</h3><p> This command shows the current status of the ctdb node. - </p><div class="refsect3" lang="en"><a name="id3243633"></a><h4>node status</h4><p> + </p><div class="refsect3" title="node status"><a name="id428295"></a><h4>node status</h4><p> Node status reflects the current status of the node. There are five possible states: </p><p> OK - This node is fully functional. @@ -75,7 +75,7 @@ in a cluster like a node that is ok. Some interfaces to serve public ip addresses are down, but at least one interface is up. See also "ctdb ifaces". - </p></div><div class="refsect3" lang="en"><a name="id3243682"></a><h4>generation</h4><p> + </p></div><div class="refsect3" title="generation"><a name="id428344"></a><h4>generation</h4><p> The generation id is a number that indicates the current generation of a cluster instance. Each time a cluster goes through a reconfiguration or a recovery its generation id will be changed. @@ -96,10 +96,10 @@ All nodes start with generation "INVALID" and are not assigned a real generation id until they have successfully been merged with a cluster through a recovery. - </p></div><div class="refsect3" lang="en"><a name="id3243707"></a><h4>VNNMAP</h4><p> + </p></div><div class="refsect3" title="VNNMAP"><a name="id428370"></a><h4>VNNMAP</h4><p> The list of Virtual Node Numbers. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records. Only nodes that are participating in the vnnmap can become lmaster or dmaster for a database record. - </p></div><div class="refsect3" lang="en"><a name="id3243718"></a><h4>Recovery mode</h4><p> + </p></div><div class="refsect3" title="Recovery mode"><a name="id428381"></a><h4>Recovery mode</h4><p> This is the current recovery mode of the cluster. There are two possible modes: </p><p> NORMAL - The cluster is fully operational. @@ -119,7 +119,7 @@ have been recovered, the node mode will change into NORMAL mode and the databases will be "thawed", allowing samba to access the databases again. - </p></div><div class="refsect3" lang="en"><a name="id3243750"></a><h4>Recovery master</h4><p> + </p></div><div class="refsect3" title="Recovery master"><a name="id428413"></a><h4>Recovery master</h4><p> This is the cluster node that is currently designated as the recovery master. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired. </p><p> Only one node at a time can be the designated recovery master. Which @@ -141,9 +141,9 @@ hash:2 lmaster:2 hash:3 lmaster:3 Recovery mode:NORMAL (0) Recovery master:0 - </pre></div><div class="refsect2" lang="en"><a name="id3243781"></a><h3>recmaster</h3><p> + </pre></div><div class="refsect2" title="recmaster"><a name="id428445"></a><h3>recmaster</h3><p> This command shows the pnn of the node which is currently the recmaster. - </p></div><div class="refsect2" lang="en"><a name="id3243790"></a><h3>uptime</h3><p> + </p></div><div class="refsect2" title="uptime"><a name="id428454"></a><h3>uptime</h3><p> This command shows the uptime for the ctdb daemon. When the last recovery or ip-failover completed and how long it took. If the "duration" is shown as a negative number, this indicates that there is a recovery/failover in progress and it started that many seconds ago. </p><p> Example: ctdb uptime @@ -152,7 +152,7 @@ Current time of node : Thu Oct 29 10:38:54 2009 Ctdbd start time : (000 16:54:28) Wed Oct 28 17:44:26 2009 Time of last recovery/failover: (000 16:53:31) Wed Oct 28 17:45:23 2009 Duration of last recovery/failover: 2.248552 seconds - </pre></div><div class="refsect2" lang="en"><a name="id3243815"></a><h3>listnodes</h3><p> + </pre></div><div class="refsect2" title="listnodes"><a name="id428481"></a><h3>listnodes</h3><p> This command shows lists the ip addresses of all the nodes in the cluster. </p><p> Example: ctdb listnodes @@ -161,7 +161,7 @@ Duration of last recovery/failover: 2.248552 seconds 10.0.0.72 10.0.0.73 10.0.0.74 - </pre></div><div class="refsect2" lang="en"><a name="id3243836"></a><h3>ping</h3><p> + </pre></div><div class="refsect2" title="ping"><a name="id385145"></a><h3>ping</h3><p> This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly. </p><p> Example: ctdb ping @@ -172,7 +172,7 @@ response from 0 time=0.000054 sec (3 clients) response from 1 time=0.000144 sec (2 clients) response from 2 time=0.000105 sec (2 clients) response from 3 time=0.000114 sec (2 clients) - </pre></div><div class="refsect2" lang="en"><a name="id3243860"></a><h3>ifaces</h3><p> + </pre></div><div class="refsect2" title="ifaces"><a name="id385169"></a><h3>ifaces</h3><p> This command will display the list of network interfaces, which could host public addresses, along with their status. </p><p> @@ -195,13 +195,13 @@ name:eth2 link:up references:1 :eth4:0:0 :eth3:1:1 :eth2:1:1 - </pre></div><div class="refsect2" lang="en"><a name="id3243895"></a><h3>setifacelink <iface> <status></h3><p> + </pre></div><div class="refsect2" title="setifacelink <iface> <status>"><a name="id385206"></a><h3>setifacelink <iface> <status></h3><p> This command will set the status of a network interface. The status needs to be "up" or "down". This is typically used in the 10.interfaces script in the "monitor" event. </p><p> Example: ctdb setifacelink eth0 up - </p></div><div class="refsect2" lang="en"><a name="id3243909"></a><h3>ip</h3><p> + </p></div><div class="refsect2" title="ip"><a name="id385220"></a><h3>ip</h3><p> This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip. By default this command will ONLY show those public addresses that are known to the node itself. To see the full list of all public ips across the cluster you must use "ctdb ip -n all". </p><p> Example: ctdb ip @@ -231,7 +231,7 @@ Public IPs on node 0 :172.31.92.83:0:eth5:eth5:eth4,eth5: :172.31.92.84:1::eth5:eth4,eth5: :172.31.92.85:0:eth5:eth5:eth4,eth5: - </pre></div><div class="refsect2" lang="en"><a name="id3243957"></a><h3>ipinfo <ip></h3><p> + </pre></div><div class="refsect2" title="ipinfo <ip>"><a name="id385270"></a><h3>ipinfo <ip></h3><p> This command will display details about the specified public addresses. </p><p> Example: ctdb ipinfo 172.31.92.85 @@ -244,7 +244,7 @@ CurrentNode:0 NumInterfaces:2 Interface[1]: Name:eth4 Link:down References:0 Interface[2]: Name:eth5 Link:up References:2 (active) - </pre></div><div class="refsect2" lang="en"><a name="id3243980"></a><h3>scriptstatus</h3><p> + </pre></div><div class="refsect2" title="scriptstatus"><a name="id385293"></a><h3>scriptstatus</h3><p> This command displays which scripts where run in the previous monitoring cycle and the result of each script. If a script failed with an error, causing the node to become unhealthy, the output from that script is also shown. </p><p> Example: ctdb scriptstatus @@ -261,15 +261,15 @@ Interface[2]: Name:eth5 Link:up References:2 (active) 41.httpd Status:OK Duration:0.039 Tue Mar 24 18:56:57 2009 50.samba Status:ERROR Duration:0.082 Tue Mar 24 18:56:57 2009 OUTPUT:ERROR: Samba tcp port 445 is not responding - </pre></div><div class="refsect2" lang="en"><a name="id3244008"></a><h3>disablescript <script></h3><p> + </pre></div><div class="refsect2" title="disablescript <script>"><a name="id385322"></a><h3>disablescript <script></h3><p> This command is used to disable an eventscript. </p><p> This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in 'scriptstatus'. - </p></div><div class="refsect2" lang="en"><a name="id3244022"></a><h3>enablescript <script></h3><p> + </p></div><div class="refsect2" title="enablescript <script>"><a name="id385336"></a><h3>enablescript <script></h3><p> This command is used to enable an eventscript. </p><p> This will take effect the next time the eventscripts are being executed so it can take a short while until this is reflected in 'scriptstatus'. - </p></div><div class="refsect2" lang="en"><a name="id3244036"></a><h3>getvar <name></h3><p> + </p></div><div class="refsect2" title="getvar <name>"><a name="id385350"></a><h3>getvar <name></h3><p> Get the runtime value of a tuneable variable. </p><p> Example: ctdb getvar MaxRedirectCount @@ -277,62 +277,64 @@ Interface[2]: Name:eth5 Link:up References:2 (active) Example output: </p><pre class="screen"> MaxRedirectCount = 3 - </pre></div><div class="refsect2" lang="en"><a name="id3244057"></a><h3>setvar <name> <value></h3><p> + </pre></div><div class="refsect2" title="setvar <name> <value>"><a name="id385371"></a><h3>setvar <name> <value></h3><p> Set the runtime value of a tuneable variable. </p><p> Example: ctdb setvar MaxRedirectCount 5 - </p></div><div class="refsect2" lang="en"><a name="id3244070"></a><h3>listvars</h3><p> - List all tuneable variables. + </p></div><div class="refsect2" title="listvars"><a name="id385383"></a><h3>listvars</h3><p> + List all tuneable variables, except the values of the obsolete tunables + like VacuumMinInterval. The obsolete tunables can be retrieved only + explicitly with the "ctdb getvar" command. </p><p> Example: ctdb listvars </p><p> Example output: </p><pre class="screen"> -MaxRedirectCount = 3 -SeqnumInterval = 1000 -ControlTimeout = 60 -TraverseTimeout = 20 -KeepaliveInterval = 5 -KeepaliveLimit = 5 -RecoverTimeout = 20 -RecoverInterval = 1 -ElectionTimeout = 3 -TakeoverTimeout = 9 -MonitorInterval = 15 -TickleUpdateInterval = 20 -EventScriptTimeout = 30 +MaxRedirectCount = 3 +SeqnumInterval = 1000 +ControlTimeout = 60 +TraverseTimeout = 20 +KeepaliveInterval = 5 +KeepaliveLimit = 5 +RecoverTimeout = 20 +RecoverInterval = 1 +ElectionTimeout = 3 +TakeoverTimeout = 9 +MonitorInterval = 15 +TickleUpdateInterval = 20 +EventScriptTimeout = 30 EventScriptTimeoutCount = 1 -EventScriptUnhealthyOnTimeout = 0 -RecoveryGracePeriod = 120 -RecoveryBanPeriod = 300 -DatabaseHashSize = 100001 -DatabaseMaxDead = 5 -RerecoveryTimeout = 10 -EnableBans = 1 -DeterministicIPs = 1 -ReclockPingPeriod = 60 -NoIPFailback = 0 -DisableIPFailover = 0 -VerboseMemoryNames = 0 -RecdPingTimeout = 60 -RecdFailCount = 10 -LogLatencyMs = 0 -RecLockLatencyMs = 1000 -RecoveryDropAllIPs = 120 -VerifyRecoveryLock = 1 -VacuumDefaultInterval = 10 -VacuumMaxRunTime = 30 -RepackLimit = 10000 -VacuumLimit = 5000 -VacuumMinInterval = 10 -VacuumMaxInterval = 10 -VacuumFastPathCount = 60 -MaxQueueDropMsg = 1000000 -UseStatusEvents = 0 -AllowUnhealthyDBRead = 0 -StatHistoryInterval = 1 -DeferredAttachTO = 120 - </pre></div><div class="refsect2" lang="en"><a name="id3244101"></a><h3>lvsmaster</h3><p> +RecoveryGracePeriod = 120 +RecoveryBanPeriod = 300 +DatabaseHashSize = 100001 +DatabaseMaxDead = 5 +RerecoveryTimeout = 10 +EnableBans = 1 +DeterministicIPs = 0 +LCP2PublicIPs = 1 +ReclockPingPeriod = 60 +NoIPFailback = 0 +DisableIPFailover = 0 +VerboseMemoryNames = 0 +RecdPingTimeout = 60 +RecdFailCount = 10 +LogLatencyMs = 0 +RecLockLatencyMs = 1000 +RecoveryDropAllIPs = 120 +VerifyRecoveryLock = 1 +VacuumInterval = 10 +VacuumMaxRunTime = 30 +RepackLimit = 10000 +VacuumLimit = 5000 +VacuumFastPathCount = 60 +MaxQueueDropMsg = 1000000 +UseStatusEvents = 0 +AllowUnhealthyDBRead = 0 +StatHistoryInterval = 1 +DeferredAttachTO = 120 +AllowClientDBAttach = 1 +RecoverPDBBySeqNum = 0 + </pre></div><div class="refsect2" title="lvsmaster"><a name="id385417"></a><h3>lvsmaster</h3><p> This command shows which node is currently the LVSMASTER. The LVSMASTER is the node in the cluster which drives the LVS system and which receives all incoming traffic from clients. @@ -343,7 +345,7 @@ DeferredAttachTO = 120 evenly onto the other nodes in the cluster. This is an alternative to using public ip addresses. See the manpage for ctdbd for more information about LVS. - </p></div><div class="refsect2" lang="en"><a name="id3244118"></a><h3>lvs</h3><p> + </p></div><div class="refsect2" title="lvs"><a name="id385435"></a><h3>lvs</h3><p> This command shows which nodes in the cluster are currently active in the LVS configuration. I.e. which nodes we are currently loadbalancing the single ip address across. @@ -358,7 +360,7 @@ DeferredAttachTO = 120 </p><pre class="screen"> 2:10.0.0.13 3:10.0.0.14 - </pre></div><div class="refsect2" lang="en"><a name="id3244142"></a><h3>getcapabilities</h3><p> + </pre></div><div class="refsect2" title="getcapabilities"><a name="id385461"></a><h3>getcapabilities</h3><p> This command shows the capabilities of the current node. Please see manpage for ctdbd for a full list of all capabilities and more detailed description. @@ -377,7 +379,7 @@ DeferredAttachTO = 120 RECMASTER: YES LMASTER: YES LVS: NO - </pre></div><div class="refsect2" lang="en"><a name="id3244172"></a><h3>statistics</h3><p> + </pre></div><div class="refsect2" title="statistics"><a name="id385491"></a><h3>statistics</h3><p> Collect statistics from the CTDB daemon about how many calls it has served. </p><p> Example: ctdb statistics @@ -419,23 +421,23 @@ CTDB version 1 max_hop_count 0 max_call_latency 4.948321 sec max_lockwait_latency 0.000000 sec - </pre></div><div class="refsect2" lang="en"><a name="id3244214"></a><h3>statisticsreset</h3><p> + </pre></div><div class="refsect2" title="statisticsreset"><a name="id385524"></a><h3>statisticsreset</h3><p> This command is used to clear all statistics counters in a node. </p><p> Example: ctdb statisticsreset - </p></div><div class="refsect2" lang="en"><a name="id3244227"></a><h3>getreclock</h3><p> + </p></div><div class="refsect2" title="getreclock"><a name="id385536"></a><h3>getreclock</h3><p> This command is used to show the filename of the reclock file that is used. </p><p> Example output: </p><pre class="screen"> Reclock file:/gpfs/.ctdb/shared - </pre></div><div class="refsect2" lang="en"><a name="id3244244"></a><h3>setreclock [filename]</h3><p> + </pre></div><div class="refsect2" title="setreclock [filename]"><a name="id385555"></a><h3>setreclock [filename]</h3><p> This command is used to modify, or clear, the file that is used as the reclock file at runtime. When this command is used, the reclock file checks are disabled. To re-enable the checks the administrator needs to activate the "VerifyRecoveryLock" tunable using "ctdb setvar". </p><p> If run with no parameter this will remove the reclock file completely. If run with a parameter the parameter specifies the new filename to use for the recovery lock. </p><p> This command only affects the runtime settings of a ctdb node and will be lost when ctdb is restarted. For persistent changes to the reclock file setting you must edit /etc/sysconfig/ctdb. - </p></div><div class="refsect2" lang="en"><a name="id3244265"></a><h3>getdebug</h3><p> + </p></div><div class="refsect2" title="getdebug"><a name="id385576"></a><h3>getdebug</h3><p> Get the current debug level for the node. the debug level controls what information is written to the log file. </p><p> The debug levels are mapped to the corresponding syslog levels. @@ -445,42 +447,42 @@ Reclock file:/gpfs/.ctdb/shared The list of debug levels from highest to lowest are : </p><p> EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG - </p></div><div class="refsect2" lang="en"><a name="id3244287"></a><h3>setdebug <debuglevel></h3><p> + </p></div><div class="refsect2" title="setdebug <debuglevel>"><a name="id385597"></a><h3>setdebug <debuglevel></h3><p> Set the debug level of a node. This controls what information will be logged. </p><p> The debuglevel is one of EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG - </p></div><div class="refsect2" lang="en"><a name="id3244301"></a><h3>getpid</h3><p> + </p></div><div class="refsect2" title="getpid"><a name="id385610"></a><h3>getpid</h3><p> This command will return the process id of the ctdb daemon. - </p></div><div class="refsect2" lang="en"><a name="id3244310"></a><h3>disable</h3><p> + </p></div><div class="refsect2" title="disable"><a name="id385619"></a><h3>disable</h3><p> This command is used to administratively disable a node in the cluster. A disabled node will still participate in the cluster and host clustered TDB records but its public ip address has been taken over by a different node and it no longer hosts any services. - </p></div><div class="refsect2" lang="en"><a name="id3244321"></a><h3>enable</h3><p> + </p></div><div class="refsect2" title="enable"><a name="id385630"></a><h3>enable</h3><p> Re-enable a node that has been administratively disabled. - </p></div><div class="refsect2" lang="en"><a name="id3244331"></a><h3>stop</h3><p> + </p></div><div class="refsect2" title="stop"><a name="id385639"></a><h3>stop</h3><p> This command is used to administratively STOP a node in the cluster. A STOPPED node is connected to the cluster but will not host any public ip addresse, nor does it participate in the VNNMAP. The difference between a DISABLED node and a STOPPED node is that a STOPPED node does not host any parts of the database which means that a recovery is required to stop/continue nodes. - </p></div><div class="refsect2" lang="en"><a name="id3244343"></a><h3>continue</h3><p> + </p></div><div class="refsect2" title="continue"><a name="id385651"></a><h3>continue</h3><p> Re-start a node that has been administratively stopped. - </p></div><div class="refsect2" lang="en"><a name="id3244352"></a><h3>addip <public_ip/mask> <iface></h3><p> + </p></div><div class="refsect2" title="addip <public_ip/mask> <iface>"><a name="id385660"></a><h3>addip <public_ip/mask> <iface></h3><p> This command is used to add a new public ip to a node during runtime. This allows public addresses to be added to a cluster without having to restart the ctdb daemons. </p><p> Note that this only updates the runtime instance of ctdb. Any changes will be lost next time ctdb is restarted and the public addresses file is re-read. If you want this change to be permanent you must also update the public addresses file manually. - </p></div><div class="refsect2" lang="en"><a name="id3244368"></a><h3>delip <public_ip></h3><p> + </p></div><div class="refsect2" title="delip <public_ip>"><a name="id385676"></a><h3>delip <public_ip></h3><p> This command is used to remove a public ip from a node during runtime. If this public ip is currently hosted by the node it being removed from, the ip will first be failed over to another node, if possible, before it is removed. </p><p> Note that this only updates the runtime instance of ctdb. Any changes will be lost next time ctdb is restarted and the public addresses file is re-read. If you want this change to be permanent you must also update the public addresses file manually. - </p></div><div class="refsect2" lang="en"><a name="id3244385"></a><h3>moveip <public_ip> <node></h3><p> + </p></div><div class="refsect2" title="moveip <public_ip> <node>"><a name="id385693"></a><h3>moveip <public_ip> <node></h3><p> This command can be used to manually fail a public ip address to a specific node. </p><p> @@ -491,14 +493,14 @@ Reclock file:/gpfs/.ctdb/shared DeterministicIPs = 0 </p><p> NoIPFailback = 1 - </p></div><div class="refsect2" lang="en"><a name="id3244406"></a><h3>shutdown</h3><p> + </p></div><div class="refsect2" title="shutdown"><a name="id385715"></a><h3>shutdown</h3><p> This command will shutdown a specific CTDB daemon. - </p></div><div class="refsect2" lang="en"><a name="id3244415"></a><h3>recover</h3><p> + </p></div><div class="refsect2" title="recover"><a name="id385724"></a><h3>recover</h3><p> This command will trigger the recovery daemon to do a cluster recovery. - </p></div><div class="refsect2" lang="en"><a name="id3244424"></a><h3>ipreallocate</h3><p> + </p></div><div class="refsect2" title="ipreallocate"><a name="id385734"></a><h3>ipreallocate</h3><p> This command will force the recovery master to perform a full ip reallocation process and redistribute all ip addresses. This is useful to "reset" the allocations back to its default state if they have been changed using the "moveip" command. While a "recover" will also perform this reallocation, a recovery is much more hevyweight since it will also rebuild all the databases. - </p></div><div class="refsect2" lang="en"><a name="id3244436"></a><h3>setlmasterrole <on|off></h3><p> + </p></div><div class="refsect2" title="setlmasterrole <on|off>"><a name="id429314"></a><h3>setlmasterrole <on|off></h3><p> This command is used ot enable/disable the LMASTER capability for a node at runtime. This capability determines whether or not a node can be used as an LMASTER for records in the database. A node that does not have the LMASTER capability will not show up in the vnnmap. </p><p> Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command. @@ -506,21 +508,21 @@ Reclock file:/gpfs/.ctdb/shared Once this setting has been enabled/disabled, you need to perform a recovery for it to take effect. </p><p> See also "ctdb getcapabilities" - </p></div><div class="refsect2" lang="en"><a name="id3244460"></a><h3>setrecmasterrole <on|off></h3><p> + </p></div><div class="refsect2" title="setrecmasterrole <on|off>"><a name="id429337"></a><h3>setrecmasterrole <on|off></h3><p> This command is used ot enable/disable the RECMASTER capability for a node at runtime. This capability determines whether or not a node can be used as an RECMASTER for the cluster. A node that does not have the RECMASTER capability can not win a recmaster election. A node that already is the recmaster for the cluster when the capability is stripped off the node will remain the recmaster until the next cluster election. </p><p> Nodes will by default have this capability, but it can be stripped off nodes by the setting in the sysconfig file or by using this command. </p><p> See also "ctdb getcapabilities" - </p></div><div class="refsect2" lang="en"><a name="id3244484"></a><h3>killtcp <srcip:port> <dstip:port></h3><p> + </p></div><div class="refsect2" title="killtcp <srcip:port> <dstip:port>"><a name="id429357"></a><h3>killtcp <srcip:port> <dstip:port></h3><p> This command will kill the specified TCP connection by issuing a TCP RST to the srcip:port endpoint. This is a command used by the ctdb eventscripts. - </p></div><div class="refsect2" lang="en"><a name="id3244495"></a><h3>gratiousarp <ip> <interface></h3><p> + </p></div><div class="refsect2" title="gratiousarp <ip> <interface>"><a name="id429367"></a><h3>gratiousarp <ip> <interface></h3><p> This command will send out a gratious arp for the specified interface through the specified interface. This command is mainly used by the ctdb eventscripts. - </p></div><div class="refsect2" lang="en"><a name="id3244505"></a><h3>reloadnodes</h3><p> + </p></div><div class="refsect2" title="reloadnodes"><a name="id429377"></a><h3>reloadnodes</h3><p> This command is used when adding new nodes, or removing existing nodes from an existing cluster. </p><p> Procedure to add a node: @@ -554,7 +556,7 @@ Reclock file:/gpfs/.ctdb/shared </p><p> 5, Use 'ctdb status' on all nodes and verify that the deleted node no longer shows up in the list.. </p><p> - </p></div><div class="refsect2" lang="en"><a name="id3244574"></a><h3>tickle <srcip:port> <dstip:port></h3><p> + </p></div><div class="refsect2" title="tickle <srcip:port> <dstip:port>"><a name="id429442"></a><h3>tickle <srcip:port> <dstip:port></h3><p> This command will will send a TCP tickle to the source host for the specified TCP connection. A TCP tickle is a TCP ACK packet with an invalid sequence and @@ -566,10 +568,10 @@ Reclock file:/gpfs/.ctdb/shared TCP connection has been disrupted and that the client will need to reestablish. This greatly speeds up the time it takes for a client to detect and reestablish after an IP failover in the ctdb cluster. - </p></div><div class="refsect2" lang="en"><a name="id3244593"></a><h3>gettickles <ip></h3><p> + </p></div><div class="refsect2" title="gettickles <ip>"><a name="id429464"></a><h3>gettickles <ip></h3><p> This command is used to show which TCP connections are registered with CTDB to be "tickled" if there is a failover. - </p></div><div class="refsect2" lang="en"><a name="id3244603"></a><h3>repack [max_freelist]</h3><p> + </p></div><div class="refsect2" title="repack [max_freelist]"><a name="id429473"></a><h3>repack [max_freelist]</h3><p> Over time, when records are created and deleted in a TDB, the TDB list of free space will become fragmented. This can lead to a slowdown in accessing TDB records. This command is used to defragment a TDB database and pruning the freelist. </p><p> @@ -584,7 +586,7 @@ Reclock file:/gpfs/.ctdb/shared Example: ctdb repack 1000 </p><p> By default, this operation is issued from the 00.ctdb event script every 5 minutes. - </p></div><div class="refsect2" lang="en"><a name="id3244640"></a><h3>vacuum [max_records]</h3><p> + </p></div><div class="refsect2" title="vacuum [max_records]"><a name="id429511"></a><h3>vacuum [max_records]</h3><p> Over time CTDB databases will fill up with empty deleted records which will lead to a progressive slow down of CTDB database access. This command is used to prune all databases and delete all empty records from the cluster. </p><p> @@ -600,17 +602,17 @@ Reclock file:/gpfs/.ctdb/shared Example: ctdb vacuum </p><p> By default, this operation is issued from the 00.ctdb event script every 5 minutes. - </p></div><div class="refsect2" lang="en"><a name="id3244669"></a><h3>backupdb <dbname> <file></h3><p> + </p></div><div class="refsect2" title="backupdb <dbname> <file>"><a name="id429540"></a><h3>backupdb <dbname> <file></h3><p> This command can be used to copy the entire content of a database out to a file. This file can later be read back into ctdb using the restoredb command. This is mainly useful for backing up persistent databases such as secrets.tdb and similar. - </p></div><div class="refsect2" lang="en"><a name="id3244680"></a><h3>restoredb <file> [<dbname>]</h3><p> + </p></div><div class="refsect2" title="restoredb <file> [<dbname>]"><a name="id429550"></a><h3>restoredb <file> [<dbname>]</h3><p> This command restores a persistent database that was previously backed up using backupdb. By default the data will be restored back into the same database as it was created from. By specifying dbname you can restore the data into a different database. - </p></div><div class="refsect2" lang="en"><a name="id3244692"></a><h3>wipedb <dbname></h3><p> + </p></div><div class="refsect2" title="wipedb <dbname>"><a name="id429561"></a><h3>wipedb <dbname></h3><p> This command can be used to remove all content of a database. - </p></div></div><div class="refsect2" lang="en"><a name="id3244702"></a><h3>getlog <level></h3><p> + </p></div></div><div class="refsect2" title="getlog <level>"><a name="id429572"></a><h3>getlog <level></h3><p> In addition to the normal loggign to a log file, CTDBD also keeps a in-memory ringbuffer containing the most recent log entries for all log levels (except DEBUG). @@ -623,14 +625,14 @@ This is mainly useful for backing up persistent databases such as secrets.tdb an </p><p> This command extracts all messages of level or lower log level from memory and prints it to the screen. - </p></div><div class="refsect2" lang="en"><a name="id3244720"></a><h3>clearlog</h3><p> + </p></div><div class="refsect2" title="clearlog"><a name="id429590"></a><h3>clearlog</h3><p> This command clears the in-memory logging ringbuffer. - </p></div><div class="refsect1" lang="en"><a name="id3244730"></a><h2>Debugging Commands</h2><p> + </p></div><div class="refsect1" title="Debugging Commands"><a name="id429598"></a><h2>Debugging Commands</h2><p> These commands are primarily used for CTDB development and testing and should not be used for normal administration. - </p><div class="refsect2" lang="en"><a name="id3244738"></a><h3>process-exists <pid></h3><p> + </p><div class="refsect2" title="process-exists <pid>"><a name="id429607"></a><h3>process-exists <pid></h3><p> This command checks if a specific process exists on the CTDB host. This is mainly used by Samba to check if remote instances of samba are still running or not. - </p></div><div class="refsect2" lang="en"><a name="id3244748"></a><h3>getdbmap</h3><p> + </p></div><div class="refsect2" title="getdbmap"><a name="id429617"></a><h3>getdbmap</h3><p> This command lists all clustered TDB databases that the CTDB daemon has attached to. Some databases are flagged as PERSISTENT, this means that the database stores data persistently and the data will remain across reboots. One example of such a database is secrets.tdb where information about how the cluster was joined to the domain is stored. </p><p> If a PERSISTENT database is not in a healthy state the database is @@ -670,7 +672,7 @@ dbid:0xb775fff6 name:secrets.tdb path:/var/ctdb/persistent/secrets.tdb.0 PERSIST </p><pre class="screen"> :ID:Name:Path:Persistent:Unhealthy: :0x7bbbd26c:passdb.tdb:/var/ctdb/persistent/passdb.tdb.0:1:0: - </pre></div><div class="refsect2" lang="en"><a name="id3244815"></a><h3>getdbstatus <dbname></h3><p> + </pre></div><div class="refsect2" title="getdbstatus <dbname>"><a name="id429683"></a><h3>getdbstatus <dbname></h3><p> This command displays more details about a database. </p><p> Example: ctdb getdbstatus test.tdb.0 @@ -692,35 +694,35 @@ name: registry.tdb path: /var/ctdb/persistent/registry.tdb.0 PERSISTENT: yes HEALTH: NO-HEALTHY-NODES - ERROR - Backup of corrupted TDB in '/var/ctdb/persistent/registry.tdb.0.corrupted.20091208091949.0Z' - </pre></div><div class="refsect2" lang="en"><a name="id3244853"></a><h3>catdb <dbname></h3><p> + </pre></div><div class="refsect2" title="catdb <dbname>"><a name="id429717"></a><h3>catdb <dbname></h3><p> This command will dump a clustered TDB database to the screen. This is a debugging command. - </p></div><div class="refsect2" lang="en"><a name="id3244863"></a><h3>cattdb <dbname></h3><p> + </p></div><div class="refsect2" title="cattdb <dbname>"><a name="id429727"></a><h3>cattdb <dbname></h3><p> This command will dump the content of the local TDB database to the screen. This is a debugging command. - </p></div><div class="refsect2" lang="en"><a name="id3244873"></a><h3>dumpdbbackup <backup-file></h3><p> + </p></div><div class="refsect2" title="dumpdbbackup <backup-file>"><a name="id429737"></a><h3>dumpdbbackup <backup-file></h3><p> This command will dump the content of database backup to the screen (similar to ctdb catdb). This is a debugging command. - </p></div><div class="refsect2" lang="en"><a name="id3244883"></a><h3>getmonmode</h3><p> + </p></div><div class="refsect2" title="getmonmode"><a name="id429746"></a><h3>getmonmode</h3><p> This command returns the monutoring mode of a node. The monitoring mode is either ACTIVE or DISABLED. Normally a node will continuously monitor that all other nodes that are expected are in fact connected and that they respond to commands. </p><p> ACTIVE - This is the normal mode. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster. </p><p> DISABLED - This node is not monitoring that other nodes are available. In this mode a node failure will not be detected and no recovery will be performed. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery. - </p></div><div class="refsect2" lang="en"><a name="id3244906"></a><h3>setmonmode <0|1></h3><p> + </p></div><div class="refsect2" title="setmonmode <0|1>"><a name="id429769"></a><h3>setmonmode <0|1></h3><p> This command can be used to explicitly disable/enable monitoring mode on a node. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery. To do this, set monitoring mode to 0 on all nodes before attaching with GDB. Remember to set monitoring mode back to 1 afterwards. - </p></div><div class="refsect2" lang="en"><a name="id3244919"></a><h3>attach <dbname> [persistent]</h3><p> + </p></div><div class="refsect2" title="attach <dbname> [persistent]"><a name="id429781"></a><h3>attach <dbname> [persistent]</h3><p> This is a debugging command. This command will make the CTDB daemon create a new CTDB database and attach to it. - </p></div><div class="refsect2" lang="en"><a name="id3244929"></a><h3>dumpmemory</h3><p> + </p></div><div class="refsect2" title="dumpmemory"><a name="id429791"></a><h3>dumpmemory</h3><p> This is a debugging command. This command will make the ctdb daemon to write a fill memory allocation map to standard output. - </p></div><div class="refsect2" lang="en"><a name="id3244939"></a><h3>rddumpmemory</h3><p> + </p></div><div class="refsect2" title="rddumpmemory"><a name="id429801"></a><h3>rddumpmemory</h3><p> This is a debugging command. This command will dump the talloc memory allocation tree for the recovery daemon to standard output. - </p></div><div class="refsect2" lang="en"><a name="id3244950"></a><h3>thaw</h3><p> + </p></div><div class="refsect2" title="thaw"><a name="id429811"></a><h3>thaw</h3><p> Thaw a previously frozen node. - </p></div><div class="refsect2" lang="en"><a name="id3244959"></a><h3>eventscript <arguments></h3><p> + </p></div><div class="refsect2" title="eventscript <arguments>"><a name="id429819"></a><h3>eventscript <arguments></h3><p> This is a debugging command. This command can be used to manually invoke and run the eventscritps with arbitrary arguments. - </p></div><div class="refsect2" lang="en"><a name="id3244969"></a><h3>ban <bantime|0></h3><p> + </p></div><div class="refsect2" title="ban <bantime|0>"><a name="id429829"></a><h3>ban <bantime|0></h3><p> Administratively ban a node for bantime seconds. A bantime of 0 means that the node should be permanently banned. </p><p> A banned node does not participate in the cluster and does not host any records for the clustered TDB. Its ip address has been taken over by another node and no services are hosted. @@ -729,11 +731,11 @@ HEALTH: NO-HEALTHY-NODES - ERROR - Backup of corrupted TDB in '/var/ctdb/persist cluster recoveries. </p><p> This is primarily a testing command. Note that the recovery daemon controls the overall ban state and it may automatically unban nodes at will. Meaning that a node that has been banned by the administrator can and ofter are unbanned before the admin specifid timeout triggers. If wanting to "drop" a node out from the cluster for mainentance or other reasons, use the "stop" / "continue" commands instad of "ban" / "unban". - </p></div><div class="refsect2" lang="en"><a name="id3244996"></a><h3>unban</h3><p> + </p></div><div class="refsect2" title="unban"><a name="id429855"></a><h3>unban</h3><p> This command is used to unban a node that has either been administratively banned using the ban command or has been automatically banned by the recovery daemon. - </p></div><div class="refsect2" lang="en"><a name="id3245006"></a><h3>check_srvids <srvid> ...</h3><p> + </p></div><div class="refsect2" title="check_srvids <srvid> ..."><a name="id429865"></a><h3>check_srvids <srvid> ...</h3><p> This command checks whether a set of srvid message ports are registered on the node or not. The command takes a list of values to check. </p><p> @@ -745,10 +747,10 @@ Server id 0:1 does not exist Server id 0:2 does not exist Server id 0:3 does not exist Server id 0:14765 exists - </pre></div></div><div class="refsect1" lang="en"><a name="id3245031"></a><h2>SEE ALSO</h2><p> + </pre></div></div><div class="refsect1" title="SEE ALSO"><a name="id429889"></a><h2>SEE ALSO</h2><p> ctdbd(1), onnode(1) - <a href="http://ctdb.samba.org/" target="_top">http://ctdb.samba.org/</a> - </p></div><div class="refsect1" lang="en"><a name="id3245045"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br> + <a class="ulink" href="http://ctdb.samba.org/" target="_top">http://ctdb.samba.org/</a> + </p></div><div class="refsect1" title="COPYRIGHT/LICENSE"><a name="id429902"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br> Copyright (C) Andrew Tridgell 2007<br> Copyright (C) Ronnie sahlberg 2007<br> <br> diff --git a/ctdb/doc/ctdb.1.xml b/ctdb/doc/ctdb.1.xml index d0ac0e8d3d..d9c287636c 100644 --- a/ctdb/doc/ctdb.1.xml +++ b/ctdb/doc/ctdb.1.xml @@ -554,7 +554,9 @@ MaxRedirectCount = 3 <refsect2><title>listvars</title> <para> - List all tuneable variables. + List all tuneable variables, except the values of the obsolete tunables + like VacuumMinInterval. The obsolete tunables can be retrieved only + explicitly with the "ctdb getvar" command. </para> <para> Example: ctdb listvars @@ -563,50 +565,50 @@ MaxRedirectCount = 3 Example output: </para> <screen format="linespecific"> -MaxRedirectCount = 3 -SeqnumInterval = 1000 -ControlTimeout = 60 -TraverseTimeout = 20 -KeepaliveInterval = 5 -KeepaliveLimit = 5 -RecoverTimeout = 20 -RecoverInterval = 1 -ElectionTimeout = 3 -TakeoverTimeout = 9 -MonitorInterval = 15 -TickleUpdateInterval = 20 -EventScriptTimeout = 30 +MaxRedirectCount = 3 +SeqnumInterval = 1000 +ControlTimeout = 60 +TraverseTimeout = 20 +KeepaliveInterval = 5 +KeepaliveLimit = 5 +RecoverTimeout = 20 +RecoverInterval = 1 +ElectionTimeout = 3 +TakeoverTimeout = 9 +MonitorInterval = 15 +TickleUpdateInterval = 20 +EventScriptTimeout = 30 EventScriptTimeoutCount = 1 -EventScriptUnhealthyOnTimeout = 0 -RecoveryGracePeriod = 120 -RecoveryBanPeriod = 300 -DatabaseHashSize = 100001 -DatabaseMaxDead = 5 -RerecoveryTimeout = 10 -EnableBans = 1 -DeterministicIPs = 1 -ReclockPingPeriod = 60 -NoIPFailback = 0 -DisableIPFailover = 0 -VerboseMemoryNames = 0 -RecdPingTimeout = 60 -RecdFailCount = 10 -LogLatencyMs = 0 -RecLockLatencyMs = 1000 -RecoveryDropAllIPs = 120 -VerifyRecoveryLock = 1 -VacuumDefaultInterval = 10 -VacuumMaxRunTime = 30 -RepackLimit = 10000 -VacuumLimit = 5000 -VacuumMinInterval = 10 -VacuumMaxInterval = 10 -VacuumFastPathCount = 60 -MaxQueueDropMsg = 1000000 -UseStatusEvents = 0 -AllowUnhealthyDBRead = 0 -StatHistoryInterval = 1 -DeferredAttachTO = 120 +RecoveryGracePeriod = 120 +RecoveryBanPeriod = 300 +DatabaseHashSize = 100001 +DatabaseMaxDead = 5 +RerecoveryTimeout = 10 +EnableBans = 1 +DeterministicIPs = 0 +LCP2PublicIPs = 1 +ReclockPingPeriod = 60 +NoIPFailback = 0 +DisableIPFailover = 0 +VerboseMemoryNames = 0 +RecdPingTimeout = 60 +RecdFailCount = 10 +LogLatencyMs = 0 +RecLockLatencyMs = 1000 +RecoveryDropAllIPs = 120 +VerifyRecoveryLock = 1 +VacuumInterval = 10 +VacuumMaxRunTime = 30 +RepackLimit = 10000 +VacuumLimit = 5000 +VacuumFastPathCount = 60 +MaxQueueDropMsg = 1000000 +UseStatusEvents = 0 +AllowUnhealthyDBRead = 0 +StatHistoryInterval = 1 +DeferredAttachTO = 120 +AllowClientDBAttach = 1 +RecoverPDBBySeqNum = 0 </screen> </refsect2> diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h index 9e5e188c49..92d498aaf7 100644 --- a/ctdb/include/ctdb_private.h +++ b/ctdb/include/ctdb_private.h @@ -108,6 +108,7 @@ struct ctdb_tunable { uint32_t reclock_latency_ms; uint32_t recovery_drop_all_ips; uint32_t verify_recovery_lock; + uint32_t vacuum_interval; uint32_t vacuum_default_interval; uint32_t vacuum_max_run_time; uint32_t repack_limit; @@ -1427,6 +1428,10 @@ int32_t ctdb_local_schedule_for_deletion(struct ctdb_db_context *ctdb_db, const struct ctdb_ltdb_header *hdr, TDB_DATA key); +void ctdb_local_remove_from_delete_queue(struct ctdb_db_context *ctdb_db, + const struct ctdb_ltdb_header *hdr, + const TDB_DATA key); + struct ctdb_ltdb_header *ctdb_header_from_record_handle(struct ctdb_record_handle *h); /* For unit testing ctdb_transaction.c. */ diff --git a/ctdb/server/ctdb_ltdb_server.c b/ctdb/server/ctdb_ltdb_server.c index 8423388433..9a0132a362 100644 --- a/ctdb/server/ctdb_ltdb_server.c +++ b/ctdb/server/ctdb_ltdb_server.c @@ -51,6 +51,7 @@ static int ctdb_ltdb_store_server(struct ctdb_db_context *ctdb_db, bool seqnum_suppressed = false; bool keep = false; bool schedule_for_deletion = false; + bool remove_from_delete_queue = false; uint32_t lmaster; if (ctdb->flags & CTDB_FLAG_TORTURE) { @@ -122,12 +123,14 @@ static int ctdb_ltdb_store_server(struct ctdb_db_context *ctdb_db, keep = true; } - if (keep && - (data.dsize == 0) && - !ctdb_db->persistent && - (ctdb_db->ctdb->pnn == header->dmaster)) - { - schedule_for_deletion = true; + if (keep) { + if ((data.dsize == 0) && + !ctdb_db->persistent && + (ctdb_db->ctdb->pnn == header->dmaster)) + { + schedule_for_deletion = true; + } + remove_from_delete_queue = !schedule_for_deletion; } store: @@ -208,6 +211,7 @@ store: tdb_errorstr(ctdb_db->ltdb->tdb))); schedule_for_deletion = false; + remove_from_delete_queue = false; } if (seqnum_suppressed) { tdb_add_flags(ctdb_db->ltdb->tdb, TDB_SEQNUM); @@ -223,6 +227,10 @@ store: } } + if (remove_from_delete_queue) { + ctdb_local_remove_from_delete_queue(ctdb_db, header, key); + } + return ret; } diff --git a/ctdb/server/ctdb_tunables.c b/ctdb/server/ctdb_tunables.c index cfc80e5ca9..d7323ec77c 100644 --- a/ctdb/server/ctdb_tunables.c +++ b/ctdb/server/ctdb_tunables.c @@ -22,55 +22,57 @@ static const struct { const char *name; uint32_t default_v; - size_t offset; + size_t offset; + bool obsolete; } tunable_map[] = { - { "MaxRedirectCount", 3, offsetof(struct ctdb_tunable, max_redirect_count) }, - { "SeqnumInterval", 1000, offsetof(struct ctdb_tunable, seqnum_interval) }, - { "ControlTimeout", 60, offsetof(struct ctdb_tunable, control_timeout) }, - { "TraverseTimeout", 20, offsetof(struct ctdb_tunable, traverse_timeout) }, - { "KeepaliveInterval", 5, offsetof(struct ctdb_tunable, keepalive_interval) }, - { "KeepaliveLimit", 5, offsetof(struct ctdb_tunable, keepalive_limit) }, - { "RecoverTimeout", 20, offsetof(struct ctdb_tunable, recover_timeout) }, - { "RecoverInterval", 1, offsetof(struct ctdb_tunable, recover_interval) }, - { "ElectionTimeout", 3, offsetof(struct ctdb_tunable, election_timeout) }, - { "TakeoverTimeout", 9, offsetof(struct ctdb_tunable, takeover_timeout) }, - { "MonitorInterval", 15, offsetof(struct ctdb_tunable, monitor_interval) }, - { "TickleUpdateInterval",20, offsetof(struct ctdb_tunable, tickle_update_interval) }, - { "EventScriptTimeout", 30, offsetof(struct ctdb_tunable, script_timeout) }, - { "EventScriptTimeoutCount", 1, offsetof(struct ctdb_tunable, script_timeout_count) }, - { "EventScriptUnhealthyOnTimeout", 0, offsetof(struct ctdb_tunable, script_unhealthy_on_timeout) },/* OBSOLETE */ - { "RecoveryGracePeriod", 120, offsetof(struct ctdb_tunable, recovery_grace_period) }, - { "RecoveryBanPeriod", 300, offsetof(struct ctdb_tunable, recovery_ban_period) }, - { "DatabaseHashSize", 100001, offsetof(struct ctdb_tunable, database_hash_size) }, - { "DatabaseMaxDead", 5, offsetof(struct ctdb_tunable, database_max_dead) }, - { "RerecoveryTimeout", 10, offsetof(struct ctdb_tunable, rerecovery_timeout) }, - { "EnableBans", 1, offsetof(struct ctdb_tunable, enable_bans) }, - { "DeterministicIPs", 0, offsetof(struct ctdb_tunable, deterministic_public_ips) }, - { "LCP2PublicIPs", 1, offsetof(struct ctdb_tunable, lcp2_public_ip_assignment) }, - { "ReclockPingPeriod", 60, offsetof(struct ctdb_tunable, reclock_ping_period) }, - { "NoIPFailback", 0, offsetof(struct ctdb_tunable, no_ip_failback) }, - { "DisableIPFailover", 0, offsetof(struct ctdb_tunable, disable_ip_failover) }, - { "VerboseMemoryNames", 0, offsetof(struct ctdb_tunable, verbose_memory_names) }, - { "RecdPingTimeout", 60, offsetof(struct ctdb_tunable, recd_ping_timeout) }, - { "RecdFailCount", 10, offsetof(struct ctdb_tunable, recd_ping_failcount) }, - { "LogLatencyMs", 0, offsetof(struct ctdb_tunable, log_latency_ms) }, - { "RecLockLatencyMs", 1000, offsetof(struct ctdb_tunable, reclock_latency_ms) }, - { "RecoveryDropAllIPs", 120, offsetof(struct ctdb_tunable, recovery_drop_all_ips) }, - { "VerifyRecoveryLock", 1, offsetof(struct ctdb_tunable, verify_recovery_lock) }, - { "VacuumDefaultInterval", 10, offsetof(struct ctdb_tunable, vacuum_default_interval) }, - { "VacuumMaxRunTime", 30, offsetof(struct ctdb_tunable, vacuum_max_run_time) }, - { "RepackLimit", 10000, offsetof(struct ctdb_tunable, repack_limit) }, - { "VacuumLimit", 5000, offsetof(struct ctdb_tunable, vacuum_limit) }, - { "VacuumMinInterval", 10, offsetof(struct ctdb_tunable, vacuum_min_interval) }, - { "VacuumMaxInterval", 10, offsetof(struct ctdb_tunable, vacuum_max_interval) }, - { "VacuumFastPathCount", 60, offsetof(struct ctdb_tunable, vacuum_fast_path_count) }, - { "MaxQueueDropMsg", 1000000, offsetof(struct ctdb_tunable, max_queue_depth_drop_msg) }, - { "UseStatusEvents", 0, offsetof(struct ctdb_tunable, use_status_events_for_monitoring) }, - { "AllowUnhealthyDBRead", 0, offsetof(struct ctdb_tunable, allow_unhealthy_db_read) }, - { "StatHistoryInterval", 1, offsetof(struct ctdb_tunable, stat_history_interval) }, - { "DeferredAttachTO", 120, offsetof(struct ctdb_tunable, deferred_attach_timeout) }, - { "AllowClientDBAttach", 1, offsetof(struct ctdb_tunable, allow_client_db_attach) }, - { "RecoverPDBBySeqNum", 0, offsetof(struct ctdb_tunable, recover_pdb_by_seqnum) } + { "MaxRedirectCount", 3, offsetof(struct ctdb_tunable, max_redirect_count), false }, + { "SeqnumInterval", 1000, offsetof(struct ctdb_tunable, seqnum_interval), false }, + { "ControlTimeout", 60, offsetof(struct ctdb_tunable, control_timeout), false }, + { "TraverseTimeout", 20, offsetof(struct ctdb_tunable, traverse_timeout), false }, + { "KeepaliveInterval", 5, offsetof(struct ctdb_tunable, keepalive_interval), false }, + { "KeepaliveLimit", 5, offsetof(struct ctdb_tunable, keepalive_limit), false }, + { "RecoverTimeout", 20, offsetof(struct ctdb_tunable, recover_timeout), false }, + { "RecoverInterval", 1, offsetof(struct ctdb_tunable, recover_interval), false }, + { "ElectionTimeout", 3, offsetof(struct ctdb_tunable, election_timeout), false }, + { "TakeoverTimeout", 9, offsetof(struct ctdb_tunable, takeover_timeout), false }, + { "MonitorInterval", 15, offsetof(struct ctdb_tunable, monitor_interval), false }, + { "TickleUpdateInterval",20, offsetof(struct ctdb_tunable, tickle_update_interval), false }, + { "EventScriptTimeout", 30, offsetof(struct ctdb_tunable, script_timeout), false }, + { "EventScriptTimeoutCount", 1, offsetof(struct ctdb_tunable, script_timeout_count), false }, + { "EventScriptUnhealthyOnTimeout", 0, offsetof(struct ctdb_tunable, script_unhealthy_on_timeout), true }, + { "RecoveryGracePeriod", 120, offsetof(struct ctdb_tunable, recovery_grace_period), false }, + { "RecoveryBanPeriod", 300, offsetof(struct ctdb_tunable, recovery_ban_period), false }, + { "DatabaseHashSize", 100001, offsetof(struct ctdb_tunable, database_hash_size), false }, + { "DatabaseMaxDead", 5, offsetof(struct ctdb_tunable, database_max_dead), false }, + { "RerecoveryTimeout", 10, offsetof(struct ctdb_tunable, rerecovery_timeout), false }, + { "EnableBans", 1, offsetof(struct ctdb_tunable, enable_bans), false }, + { "DeterministicIPs", 0, offsetof(struct ctdb_tunable, deterministic_public_ips), false }, + { "LCP2PublicIPs", 1, offsetof(struct ctdb_tunable, lcp2_public_ip_assignment), false }, + { "ReclockPingPeriod", 60, offsetof(struct ctdb_tunable, reclock_ping_period), false }, + { "NoIPFailback", 0, offsetof(struct ctdb_tunable, no_ip_failback), false }, + { "DisableIPFailover", 0, offsetof(struct ctdb_tunable, disable_ip_failover), false }, + { "VerboseMemoryNames", 0, offsetof(struct ctdb_tunable, verbose_memory_names), false }, + { "RecdPingTimeout", 60, offsetof(struct ctdb_tunable, recd_ping_timeout), false }, + { "RecdFailCount", 10, offsetof(struct ctdb_tunable, recd_ping_failcount), false }, + { "LogLatencyMs", 0, offsetof(struct ctdb_tunable, log_latency_ms), false }, + { "RecLockLatencyMs", 1000, offsetof(struct ctdb_tunable, reclock_latency_ms), false }, + { "RecoveryDropAllIPs", 120, offsetof(struct ctdb_tunable, recovery_drop_all_ips), false }, + { "VerifyRecoveryLock", 1, offsetof(struct ctdb_tunable, verify_recovery_lock), false }, + { "VacuumInterval", 10, offsetof(struct ctdb_tunable, vacuum_interval), false }, + { "VacuumDefaultInterval", 10, offsetof(struct ctdb_tunable, vacuum_default_interval), true }, + { "VacuumMaxRunTime", 30, offsetof(struct ctdb_tunable, vacuum_max_run_time), false }, + { "RepackLimit", 10000, offsetof(struct ctdb_tunable, repack_limit), false }, + { "VacuumLimit", 5000, offsetof(struct ctdb_tunable, vacuum_limit), false }, + { "VacuumMinInterval", 10, offsetof(struct ctdb_tunable, vacuum_min_interval), true }, + { "VacuumMaxInterval", 10, offsetof(struct ctdb_tunable, vacuum_max_interval), true }, + { "VacuumFastPathCount", 60, offsetof(struct ctdb_tunable, vacuum_fast_path_count), false }, + { "MaxQueueDropMsg", 1000000, offsetof(struct ctdb_tunable, max_queue_depth_drop_msg), false }, + { "UseStatusEvents", 0, offsetof(struct ctdb_tunable, use_status_events_for_monitoring), false }, + { "AllowUnhealthyDBRead", 0, offsetof(struct ctdb_tunable, allow_unhealthy_db_read), false }, + { "StatHistoryInterval", 1, offsetof(struct ctdb_tunable, stat_history_interval), false }, + { "DeferredAttachTO", 120, offsetof(struct ctdb_tunable, deferred_attach_timeout), false }, + { "AllowClientDBAttach", 1, offsetof(struct ctdb_tunable, allow_client_db_attach), false }, + { "RecoverPDBBySeqNum", 0, offsetof(struct ctdb_tunable, recover_pdb_by_seqnum), false } }; /* @@ -181,6 +183,9 @@ int32_t ctdb_control_list_tunables(struct ctdb_context *ctdb, TDB_DATA *outdata) CTDB_NO_MEMORY(ctdb, list); for (i=1;i<ARRAY_SIZE(tunable_map);i++) { + if (tunable_map[i].obsolete) { + continue; + } list = talloc_asprintf_append(list, ":%s", tunable_map[i].name); CTDB_NO_MEMORY(ctdb, list); } diff --git a/ctdb/server/ctdb_vacuum.c b/ctdb/server/ctdb_vacuum.c index 5be8abe420..ff373564f8 100644 --- a/ctdb/server/ctdb_vacuum.c +++ b/ctdb/server/ctdb_vacuum.c @@ -33,7 +33,6 @@ #include "../common/rb_tree.h" #define TIMELIMIT() timeval_current_ofs(10, 0) -#define TUNINGDBNAME "vactune.tdb" enum vacuum_child_status { VACUUM_RUNNING, VACUUM_OK, VACUUM_ERROR, VACUUM_TIMEOUT}; @@ -61,9 +60,9 @@ struct vacuum_data { struct ctdb_context *ctdb; struct ctdb_db_context *ctdb_db; struct tdb_context *dest_db; - trbt_tree_t *delete_tree; + trbt_tree_t *delete_list; uint32_t delete_count; - struct ctdb_marshall_buffer **list; + struct ctdb_marshall_buffer **vacuum_fetch_list; struct timeval start; bool traverse_error; bool vacuum; @@ -71,26 +70,21 @@ struct vacuum_data { uint32_t vacuumed; uint32_t copied; uint32_t fast_added_to_vacuum_fetch_list; - uint32_t fast_added_to_delete_tree; + uint32_t fast_added_to_delete_list; uint32_t fast_deleted; uint32_t fast_skipped; uint32_t fast_error; uint32_t fast_total; uint32_t full_added_to_vacuum_fetch_list; - uint32_t full_added_to_delete_tree; + uint32_t full_added_to_delete_list; uint32_t full_skipped; uint32_t full_error; uint32_t full_total; -}; - -/* tuning information stored for every db */ -struct vacuum_tuning_data { - uint32_t last_num_repack; - uint32_t last_num_empty; - uint32_t last_interval; - uint32_t new_interval; - struct timeval last_start; - double last_duration; + uint32_t delete_left; + uint32_t delete_remote_error; + uint32_t delete_local_error; + uint32_t delete_deleted; + uint32_t delete_skipped; }; /* this structure contains the information for one record to be deleted */ @@ -141,7 +135,7 @@ static int insert_delete_record_data_into_tree(struct ctdb_context *ctdb, return 0; } -static int add_record_to_delete_tree(struct vacuum_data *vdata, TDB_DATA key, +static int add_record_to_delete_list(struct vacuum_data *vdata, TDB_DATA key, struct ctdb_ltdb_header *hdr) { struct ctdb_context *ctdb = vdata->ctdb; @@ -151,13 +145,13 @@ static int add_record_to_delete_tree(struct vacuum_data *vdata, TDB_DATA key, hash = ctdb_hash(&key); - if (trbt_lookup32(vdata->delete_tree, hash)) { + if (trbt_lookup32(vdata->delete_list, hash)) { DEBUG(DEBUG_INFO, (__location__ " Hash collission when vacuuming, skipping this record.\n")); return 0; } ret = insert_delete_record_data_into_tree(ctdb, ctdb_db, - vdata->delete_tree, + vdata->delete_list, hdr, key); if (ret != 0) { return -1; @@ -179,27 +173,30 @@ static int add_record_to_vacuum_fetch_list(struct vacuum_data *vdata, struct ctdb_rec_data *rec; uint32_t lmaster; size_t old_size; + struct ctdb_marshall_buffer *vfl; lmaster = ctdb_lmaster(ctdb, &key); - rec = ctdb_marshall_record(vdata->list[lmaster], ctdb->pnn, key, NULL, tdb_null); + vfl = vdata->vacuum_fetch_list[lmaster]; + + rec = ctdb_marshall_record(vfl, ctdb->pnn, key, NULL, tdb_null); if (rec == NULL) { DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); vdata->traverse_error = true; return -1; } - old_size = talloc_get_size(vdata->list[lmaster]); - vdata->list[lmaster] = talloc_realloc_size(NULL, vdata->list[lmaster], - old_size + rec->length); - if (vdata->list[lmaster] == NULL) { + old_size = talloc_get_size(vfl); + vfl = talloc_realloc_size(NULL, vfl, old_size + rec->length); + if (vfl == NULL) { DEBUG(DEBUG_ERR,(__location__ " Failed to expand\n")); vdata->traverse_error = true; return -1; } + vdata->vacuum_fetch_list[lmaster] = vfl; - vdata->list[lmaster]->count++; - memcpy(old_size+(uint8_t *)vdata->list[lmaster], rec, rec->length); + vfl->count++; + memcpy(old_size+(uint8_t *)vfl, rec, rec->length); talloc_free(rec); vdata->total++; @@ -255,11 +252,11 @@ static int vacuum_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, * We are both lmaster and dmaster, and the record is empty. * So we should be able to delete it. */ - res = add_record_to_delete_tree(vdata, key, hdr); + res = add_record_to_delete_list(vdata, key, hdr); if (res != 0) { vdata->full_error++; } else { - vdata->full_added_to_delete_tree++; + vdata->full_added_to_delete_list++; } } else { /* @@ -281,7 +278,7 @@ static int vacuum_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, * traverse the tree of records to delete and marshall them into * a blob */ -static int delete_traverse(void *param, void *data) +static int delete_marshall_traverse(void *param, void *data) { struct delete_record_data *dd = talloc_get_type(data, struct delete_record_data); struct delete_records_list *recs = talloc_get_type(param, struct delete_records_list); @@ -336,12 +333,16 @@ static int delete_queue_traverse(void *param, void *data) struct ctdb_ltdb_header *header; TDB_DATA tdb_data; uint32_t lmaster; + uint32_t hash = ctdb_hash(&(dd->key)); vdata->fast_total++; res = tdb_chainlock(ctdb_db->ltdb->tdb, dd->key); if (res != 0) { - DEBUG(DEBUG_ERR, (__location__ " Error getting chainlock.\n")); + DEBUG(DEBUG_ERR, + (__location__ " Error getting chainlock on record with " + "key hash [0x%08x] on database db[%s].\n", + hash, ctdb_db->db_name)); vdata->fast_error++; return 0; } @@ -364,7 +365,6 @@ static int delete_queue_traverse(void *param, void *data) goto skipped; } - if (header->rsn != dd->hdr.rsn) { /* * The record has been migrated off the node and back again. @@ -400,7 +400,7 @@ static int delete_queue_traverse(void *param, void *data) /* use header->flags or dd->hdr.flags ?? */ if (dd->hdr.flags & CTDB_REC_FLAG_MIGRATED_WITH_DATA) { - res = add_record_to_delete_tree(vdata, dd->key, &dd->hdr); + res = add_record_to_delete_list(vdata, dd->key, &dd->hdr); if (res != 0) { DEBUG(DEBUG_ERR, @@ -408,17 +408,22 @@ static int delete_queue_traverse(void *param, void *data) "of records for deletion on lmaster.\n")); vdata->fast_error++; } else { - vdata->fast_added_to_delete_tree++; + vdata->fast_added_to_delete_list++; } } else { res = tdb_delete(ctdb_db->ltdb->tdb, dd->key); if (res != 0) { DEBUG(DEBUG_ERR, - (__location__ " Error deleting record from local " - "data base.\n")); + (__location__ " Error deleting record with key " + "hash [0x%08x] from local data base db[%s].\n", + hash, ctdb_db->db_name)); vdata->fast_error++; } else { + DEBUG(DEBUG_DEBUG, + (__location__ " Deleted record with key hash " + "[0x%08x] from local data base db[%s].\n", + hash, ctdb_db->db_name)); vdata->fast_deleted++; } } @@ -438,97 +443,123 @@ done: } /** - * Vacuum a DB: - * - Always do the fast vacuuming run, which traverses - * the in-memory delete queue: these records have been - * scheduled for deletion. - * - Only if explicitly requested, the database is traversed - * in order to use the traditional heuristics on empty records - * to trigger deletion. - * This is done only every VacuumFastPathCount'th vacuuming run. - * - * The traverse runs fill two lists: - * - * - The delete_list: - * This is the list of empty records the current - * node is lmaster and dmaster for. These records are later - * deleted first on other nodes and then locally. - * - * The fast vacuuming run has a short cut for those records - * that have never been migrated with data: these records - * are immediately deleted locally, since they have left - * no trace on other nodes. - * - * - The vacuum_fetch lists - * (one for each other lmaster node): - * The records in this list are sent for deletion to - * their lmaster in a bulk VACUUM_FETCH message. - * - * The lmaster then migrates all these records to itelf - * so that they can be vacuumed there. - * - * This executes in the child context. + * Delete the records that we are lmaster and dmaster for and + * that could be deleted on all other nodes via the TRY_DELETE_RECORDS + * control. */ -static int ctdb_vacuum_db(struct ctdb_db_context *ctdb_db, - struct vacuum_data *vdata, - bool full_vacuum_run) +static int delete_record_traverse(void *param, void *data) { + struct delete_record_data *dd = + talloc_get_type(data, struct delete_record_data); + struct vacuum_data *vdata = talloc_get_type(param, struct vacuum_data); + struct ctdb_db_context *ctdb_db = dd->ctdb_db; struct ctdb_context *ctdb = ctdb_db->ctdb; - const char *name = ctdb_db->db_name; - int ret, i, pnn; + int res; + struct ctdb_ltdb_header *header; + TDB_DATA tdb_data; + uint32_t lmaster; + bool deleted = false; + uint32_t hash = ctdb_hash(&(dd->key)); - DEBUG(DEBUG_INFO, (__location__ " Entering %s vacuum run for db " - "%s db_id[0x%08x]\n", - full_vacuum_run ? "full" : "fast", - ctdb_db->db_name, ctdb_db->db_id)); + res = tdb_chainlock(ctdb_db->ltdb->tdb, dd->key); + if (res != 0) { + DEBUG(DEBUG_ERR, + (__location__ " Error getting chainlock on record with " + "key hash [0x%08x] on database db[%s].\n", + hash, ctdb_db->db_name)); + vdata->delete_local_error++; + return 0; + } - ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &ctdb->vnn_map); - if (ret != 0) { - DEBUG(DEBUG_ERR, ("Unable to get vnnmap from local node\n")); - return ret; + /* + * Verify that the record is still empty, its RSN has not + * changed and that we are still its lmaster and dmaster. + */ + + tdb_data = tdb_fetch(ctdb_db->ltdb->tdb, dd->key); + if (tdb_data.dsize < sizeof(struct ctdb_ltdb_header)) { + /* Does not exist or not a ctdb record. Skip. */ + vdata->delete_skipped++; + goto done; } - pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE); - if (pnn == -1) { - DEBUG(DEBUG_ERR, ("Unable to get pnn from local node\n")); - return -1; + if (tdb_data.dsize > sizeof(struct ctdb_ltdb_header)) { + /* The record has been recycled (filled with data). Skip. */ + vdata->delete_skipped++; + goto done; } - ctdb->pnn = pnn; + header = (struct ctdb_ltdb_header *)tdb_data.dptr; - vdata->fast_added_to_delete_tree = 0; - vdata->fast_added_to_vacuum_fetch_list = 0; - vdata->fast_deleted = 0; - vdata->fast_skipped = 0; - vdata->fast_error = 0; - vdata->fast_total = 0; - vdata->full_added_to_delete_tree = 0; - vdata->full_added_to_vacuum_fetch_list = 0; - vdata->full_skipped = 0; - vdata->full_error = 0; - vdata->full_total = 0; + if (header->dmaster != ctdb->pnn) { + /* The record has been migrated off the node. Skip. */ + vdata->delete_skipped++; + goto done; + } - /* the list needs to be of length num_nodes */ - vdata->list = talloc_array(vdata, struct ctdb_marshall_buffer *, ctdb->num_nodes); - if (vdata->list == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; + + if (header->rsn != dd->hdr.rsn) { + /* + * The record has been migrated off the node and back again. + * But not requeued for deletion. Skip it. + */ + vdata->delete_skipped++; + goto done; } - for (i = 0; i < ctdb->num_nodes; i++) { - vdata->list[i] = (struct ctdb_marshall_buffer *) - talloc_zero_size(vdata->list, - offsetof(struct ctdb_marshall_buffer, data)); - if (vdata->list[i] == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - vdata->list[i]->db_id = ctdb_db->db_id; + + lmaster = ctdb_lmaster(ctdb_db->ctdb, &dd->key); + + if (lmaster != ctdb->pnn) { + /* we are not lmaster - strange */ + vdata->delete_skipped++; + goto done; } - /* - * Traverse the delete_queue. - * This builds the same lists as the db traverse. - */ + res = tdb_delete(ctdb_db->ltdb->tdb, dd->key); + + if (res != 0) { + DEBUG(DEBUG_ERR, + (__location__ " Error deleting record with key hash " + "[0x%08x] from local data base db[%s].\n", + hash, ctdb_db->db_name)); + vdata->delete_local_error++; + goto done; + } + + deleted = true; + + DEBUG(DEBUG_DEBUG, + (__location__ " Deleted record with key hash [0x%08x] from " + "local data base db[%s].\n", hash, ctdb_db->db_name)); + +done: + if (tdb_data.dptr != NULL) { + free(tdb_data.dptr); + } + + tdb_chainunlock(ctdb_db->ltdb->tdb, dd->key); + + if (deleted) { + /* + * successfully deleted the record locally. + * remove it from the list and update statistics. + */ + talloc_free(dd); + vdata->delete_deleted++; + vdata->delete_left--; + } + + return 0; +} + +/** + * Fast vacuuming run: + * Traverse the delete_queue. + * This fills the same lists as the database traverse. + */ +static void ctdb_vacuum_db_fast(struct ctdb_db_context *ctdb_db, + struct vacuum_data *vdata) +{ trbt_traversearray32(ctdb_db->delete_queue, 1, delete_queue_traverse, vdata); if (vdata->fast_total > 0) { @@ -540,79 +571,122 @@ static int ctdb_vacuum_db(struct ctdb_db_context *ctdb_db, "del[%u] " "skp[%u] " "err[%u] " - "adt[%u] " + "adl[%u] " "avf[%u]\n", ctdb_db->db_name, (unsigned)vdata->fast_total, (unsigned)vdata->fast_deleted, (unsigned)vdata->fast_skipped, (unsigned)vdata->fast_error, - (unsigned)vdata->fast_added_to_delete_tree, + (unsigned)vdata->fast_added_to_delete_list, (unsigned)vdata->fast_added_to_vacuum_fetch_list)); } - /* - * read-only traverse of the database, looking for records that - * might be able to be vacuumed. - * - * This is not done each time but only every tunable - * VacuumFastPathCount times. - */ - if (full_vacuum_run) { - ret = tdb_traverse_read(ctdb_db->ltdb->tdb, vacuum_traverse, vdata); - if (ret == -1 || vdata->traverse_error) { - DEBUG(DEBUG_ERR,(__location__ " Traverse error in vacuuming '%s'\n", name)); - return -1; - } - if (vdata->full_total > 0) { - DEBUG(DEBUG_INFO, - (__location__ - " full vacuuming db traverse statistics: " - "db[%s] " - "total[%u] " - "skp[%u] " - "err[%u] " - "adt[%u] " - "avf[%u]\n", - ctdb_db->db_name, - (unsigned)vdata->full_total, - (unsigned)vdata->full_skipped, - (unsigned)vdata->full_error, - (unsigned)vdata->full_added_to_delete_tree, - (unsigned)vdata->full_added_to_vacuum_fetch_list)); - } + return; +} + +/** + * Full vacuum run: + * read-only traverse of the database, looking for records that + * might be able to be vacuumed. + * + * This is not done each time but only every tunable + * VacuumFastPathCount times. + */ +static int ctdb_vacuum_db_full(struct ctdb_db_context *ctdb_db, + struct vacuum_data *vdata, + bool full_vacuum_run) +{ + int ret; + + if (!full_vacuum_run) { + return 0; } - /* - * For records where we are not the lmaster, - * tell the lmaster to fetch the record. - */ + ret = tdb_traverse_read(ctdb_db->ltdb->tdb, vacuum_traverse, vdata); + if (ret == -1 || vdata->traverse_error) { + DEBUG(DEBUG_ERR, (__location__ " Traverse error in vacuuming " + "'%s'\n", ctdb_db->db_name)); + return -1; + } + + if (vdata->full_total > 0) { + DEBUG(DEBUG_INFO, + (__location__ + " full vacuuming db traverse statistics: " + "db[%s] " + "total[%u] " + "skp[%u] " + "err[%u] " + "adl[%u] " + "avf[%u]\n", + ctdb_db->db_name, + (unsigned)vdata->full_total, + (unsigned)vdata->full_skipped, + (unsigned)vdata->full_error, + (unsigned)vdata->full_added_to_delete_list, + (unsigned)vdata->full_added_to_vacuum_fetch_list)); + } + + return 0; +} + +/** + * Process the vacuum fetch lists: + * For records for which we are not the lmaster, tell the lmaster to + * fetch the record. + */ +static int ctdb_process_vacuum_fetch_lists(struct ctdb_db_context *ctdb_db, + struct vacuum_data *vdata) +{ + int i; + struct ctdb_context *ctdb = ctdb_db->ctdb; + for (i = 0; i < ctdb->num_nodes; i++) { TDB_DATA data; + struct ctdb_marshall_buffer *vfl = vdata->vacuum_fetch_list[i]; if (ctdb->nodes[i]->pnn == ctdb->pnn) { continue; } - if (vdata->list[i]->count == 0) { + if (vfl->count == 0) { continue; } DEBUG(DEBUG_INFO, ("Found %u records for lmaster %u in '%s'\n", - vdata->list[i]->count, ctdb->nodes[i]->pnn, - name)); - - data.dsize = talloc_get_size(vdata->list[i]); - data.dptr = (void *)vdata->list[i]; - if (ctdb_client_send_message(ctdb, ctdb->nodes[i]->pnn, CTDB_SRVID_VACUUM_FETCH, data) != 0) { + vfl->count, ctdb->nodes[i]->pnn, + ctdb_db->db_name)); + + data.dsize = talloc_get_size(vfl); + data.dptr = (void *)vfl; + if (ctdb_client_send_message(ctdb, ctdb->nodes[i]->pnn, + CTDB_SRVID_VACUUM_FETCH, + data) != 0) + { DEBUG(DEBUG_ERR, (__location__ " Failed to send vacuum " "fetch message to %u\n", ctdb->nodes[i]->pnn)); return -1; } - } + } + + return 0; +} + +/** + * Proces the delete list: + * Send the records to delete to all other nodes with the + * try_delete_records control. + */ +static int ctdb_process_delete_list(struct ctdb_db_context *ctdb_db, + struct vacuum_data *vdata) +{ + int ret, i; + struct ctdb_context *ctdb = ctdb_db->ctdb; + + vdata->delete_left = vdata->delete_count; - /* Process all records we can delete (if any) */ if (vdata->delete_count > 0) { struct delete_records_list *recs; TDB_DATA indata, outdata; @@ -639,7 +713,8 @@ static int ctdb_vacuum_db(struct ctdb_db_context *ctdb_db, * traverse the tree of all records we want to delete and * create a blob we can send to the other nodes. */ - trbt_traversearray32(vdata->delete_tree, 1, delete_traverse, recs); + trbt_traversearray32(vdata->delete_list, 1, + delete_marshall_traverse, recs); indata.dsize = talloc_get_size(recs->records); indata.dptr = (void *)recs->records; @@ -679,15 +754,28 @@ static int ctdb_vacuum_db(struct ctdb_db_context *ctdb_db, return -1; } - /* - * outdata countains the list of records coming back - * from the node which the node could not delete + /* + * outdata contains the list of records coming back + * from the node: These are the records that the + * remote node could not delete. + * + * NOTE: There is a problem here: + * + * When a node failed to delete the record, but + * others succeeded, we may have created gaps in the + * history of the record. Hence when a node dies, an + * closed file handle might be resurrected or an open + * file handle might be lost, leading to blocked access + * or data corruption. + * + * TODO: This needs to be fixed! */ records = (struct ctdb_marshall_buffer *)outdata.dptr; rec = (struct ctdb_rec_data *)&records->data[0]; while (records->count-- > 1) { TDB_DATA reckey, recdata; struct ctdb_ltdb_header *rechdr; + struct delete_record_data *dd; reckey.dptr = &rec->data[0]; reckey.dsize = rec->keylen; @@ -702,28 +790,190 @@ static int ctdb_vacuum_db(struct ctdb_db_context *ctdb_db, recdata.dptr += sizeof(*rechdr); recdata.dsize -= sizeof(*rechdr); - /* - * that other node couldnt delete the record - * so we should delete it and thereby remove it from the tree - */ - talloc_free(trbt_lookup32(vdata->delete_tree, ctdb_hash(&reckey))); + dd = (struct delete_record_data *)trbt_lookup32( + vdata->delete_list, + ctdb_hash(&reckey)); + if (dd != NULL) { + /* + * The other node could not delete the + * record and it is the first node that + * failed. So we should remove it from + * the tree and update statistics. + */ + talloc_free(dd); + vdata->delete_remote_error++; + vdata->delete_left--; + } rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec); - } + } } /* free nodemap and active_nodes */ talloc_free(nodemap); + } - /* - * The only records remaining in the tree would be those - * records where all other nodes could successfully - * delete them, so we can safely delete them on the - * lmaster as well. Deletion implictely happens while - * we repack the database. The repack algorithm revisits - * the tree in order to find the records that don't need - * to be copied / repacked. + if (vdata->delete_left > 0) { + /* + * The only records remaining in the tree are those + * records which all other nodes could successfully + * delete, so we can safely delete them on the + * lmaster as well. */ + trbt_traversearray32(vdata->delete_list, 1, + delete_record_traverse, vdata); + } + + if (vdata->delete_count > 0) { + DEBUG(DEBUG_INFO, + (__location__ + " vacuum delete list statistics: " + "db[%s] " + "coll[%u] " + "rem.err[%u] " + "loc.err[%u] " + "skip[%u] " + "del[%u] " + "left[%u]\n", + ctdb_db->db_name, + (unsigned)vdata->delete_count, + (unsigned)vdata->delete_remote_error, + (unsigned)vdata->delete_local_error, + (unsigned)vdata->delete_skipped, + (unsigned)vdata->delete_deleted, + (unsigned)vdata->delete_left)); + } + + return 0; +} + +/** + * initialize the vacuum_data + */ +static int ctdb_vacuum_init_vacuum_data(struct ctdb_db_context *ctdb_db, + struct vacuum_data *vdata) +{ + int i; + struct ctdb_context *ctdb = ctdb_db->ctdb; + + vdata->fast_added_to_delete_list = 0; + vdata->fast_added_to_vacuum_fetch_list = 0; + vdata->fast_deleted = 0; + vdata->fast_skipped = 0; + vdata->fast_error = 0; + vdata->fast_total = 0; + vdata->full_added_to_delete_list = 0; + vdata->full_added_to_vacuum_fetch_list = 0; + vdata->full_skipped = 0; + vdata->full_error = 0; + vdata->full_total = 0; + vdata->delete_count = 0; + vdata->delete_left = 0; + vdata->delete_remote_error = 0; + vdata->delete_local_error = 0; + vdata->delete_skipped = 0; + vdata->delete_deleted = 0; + + /* the list needs to be of length num_nodes */ + vdata->vacuum_fetch_list = talloc_zero_array(vdata, + struct ctdb_marshall_buffer *, + ctdb->num_nodes); + if (vdata->vacuum_fetch_list == NULL) { + DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); + return -1; + } + for (i = 0; i < ctdb->num_nodes; i++) { + vdata->vacuum_fetch_list[i] = (struct ctdb_marshall_buffer *) + talloc_zero_size(vdata->vacuum_fetch_list, + offsetof(struct ctdb_marshall_buffer, data)); + if (vdata->vacuum_fetch_list[i] == NULL) { + DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); + return -1; + } + vdata->vacuum_fetch_list[i]->db_id = ctdb_db->db_id; + } + + return 0; +} + +/** + * Vacuum a DB: + * - Always do the fast vacuuming run, which traverses + * the in-memory delete queue: these records have been + * scheduled for deletion. + * - Only if explicitly requested, the database is traversed + * in order to use the traditional heuristics on empty records + * to trigger deletion. + * This is done only every VacuumFastPathCount'th vacuuming run. + * + * The traverse runs fill two lists: + * + * - The delete_list: + * This is the list of empty records the current + * node is lmaster and dmaster for. These records are later + * deleted first on other nodes and then locally. + * + * The fast vacuuming run has a short cut for those records + * that have never been migrated with data: these records + * are immediately deleted locally, since they have left + * no trace on other nodes. + * + * - The vacuum_fetch lists + * (one for each other lmaster node): + * The records in this list are sent for deletion to + * their lmaster in a bulk VACUUM_FETCH message. + * + * The lmaster then migrates all these records to itelf + * so that they can be vacuumed there. + * + * This executes in the child context. + */ +static int ctdb_vacuum_db(struct ctdb_db_context *ctdb_db, + struct vacuum_data *vdata, + bool full_vacuum_run) +{ + struct ctdb_context *ctdb = ctdb_db->ctdb; + int ret, pnn; + + DEBUG(DEBUG_INFO, (__location__ " Entering %s vacuum run for db " + "%s db_id[0x%08x]\n", + full_vacuum_run ? "full" : "fast", + ctdb_db->db_name, ctdb_db->db_id)); + + ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &ctdb->vnn_map); + if (ret != 0) { + DEBUG(DEBUG_ERR, ("Unable to get vnnmap from local node\n")); + return ret; + } + + pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE); + if (pnn == -1) { + DEBUG(DEBUG_ERR, ("Unable to get pnn from local node\n")); + return -1; + } + + ctdb->pnn = pnn; + + ret = ctdb_vacuum_init_vacuum_data(ctdb_db, vdata); + if (ret != 0) { + return ret; + } + + ctdb_vacuum_db_fast(ctdb_db, vdata); + + ret = ctdb_vacuum_db_full(ctdb_db, vdata, full_vacuum_run); + if (ret != 0) { + return ret; + } + + ret = ctdb_process_vacuum_fetch_lists(ctdb_db, vdata); + if (ret != 0) { + return ret; + } + + ret = ctdb_process_delete_list(ctdb_db, vdata); + if (ret != 0) { + return ret; } /* this ensures we run our event queue */ @@ -744,9 +994,9 @@ static int repack_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, uint32_t hash = ctdb_hash(&key); struct delete_record_data *kd; /* - * check if we can ignore this record because it's in the delete_tree + * check if we can ignore this record because it's in the delete_list */ - kd = (struct delete_record_data *)trbt_lookup32(vdata->delete_tree, hash); + kd = (struct delete_record_data *)trbt_lookup32(vdata->delete_list, hash); /* * there might be hash collisions so we have to compare the keys here to be sure */ @@ -858,104 +1108,6 @@ static int ctdb_repack_tdb(struct tdb_context *tdb, TALLOC_CTX *mem_ctx, struct return 0; } -static int update_tuning_db(struct ctdb_db_context *ctdb_db, struct vacuum_data *vdata, uint32_t freelist) -{ - TALLOC_CTX *tmp_ctx = talloc_new(NULL); - TDB_CONTEXT *tune_tdb; - TDB_DATA key, value; - struct vacuum_tuning_data tdata; - struct vacuum_tuning_data *tptr; - char *vac_dbname; - int flags; - - vac_dbname = talloc_asprintf(tmp_ctx, "%s/%s.%u", - ctdb_db->ctdb->db_directory_state, - TUNINGDBNAME, ctdb_db->ctdb->pnn); - if (vac_dbname == NULL) { - DEBUG(DEBUG_CRIT,(__location__ " Out of memory error while allocating '%s'\n", vac_dbname)); - talloc_free(tmp_ctx); - return -1; - } - - flags = ctdb_db->ctdb->valgrinding ? TDB_NOMMAP : 0; - flags |= TDB_DISALLOW_NESTING; - tune_tdb = tdb_open(vac_dbname, 0, - flags, - O_RDWR|O_CREAT, 0600); - if (tune_tdb == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Failed to create/open %s\n", TUNINGDBNAME)); - talloc_free(tmp_ctx); - return -1; - } - - if (tdb_transaction_start(tune_tdb) != 0) { - DEBUG(DEBUG_ERR,(__location__ " Failed to start transaction\n")); - tdb_close(tune_tdb); - return -1; - } - key.dptr = discard_const(ctdb_db->db_name); - key.dsize = strlen(ctdb_db->db_name); - value = tdb_fetch(tune_tdb, key); - - if (value.dptr != NULL && value.dsize == sizeof(struct vacuum_tuning_data)) { - tptr = (struct vacuum_tuning_data *)value.dptr; - tdata = *tptr; - - /* - * re-calc new vacuum interval: - * in case no limit was reached we continuously increase the interval - * until vacuum_max_interval is reached - * in case a limit was reached we divide the current interval by 2 - * unless vacuum_min_interval is reached - */ - if (freelist < vdata->repack_limit && - vdata->delete_count < vdata->vacuum_limit) { - if (tdata.last_interval < ctdb_db->ctdb->tunable.vacuum_max_interval) { - tdata.new_interval = tdata.last_interval * 110 / 100; - DEBUG(DEBUG_INFO,("Increasing vacuum interval %u -> %u for %s\n", - tdata.last_interval, tdata.new_interval, ctdb_db->db_name)); - } - } else { - tdata.new_interval = tdata.last_interval / 2; - if (tdata.new_interval < ctdb_db->ctdb->tunable.vacuum_min_interval || - tdata.new_interval > ctdb_db->ctdb->tunable.vacuum_max_interval) { - tdata.new_interval = ctdb_db->ctdb->tunable.vacuum_min_interval; - } - DEBUG(DEBUG_INFO,("Decreasing vacuum interval %u -> %u for %s\n", - tdata.last_interval, tdata.new_interval, ctdb_db->db_name)); - } - tdata.last_interval = tdata.new_interval; - } else { - DEBUG(DEBUG_DEBUG,(__location__ " Cannot find tunedb record for %s. Using default interval\n", ctdb_db->db_name)); - tdata.last_num_repack = freelist; - tdata.last_num_empty = vdata->delete_count; - tdata.last_interval = ctdb_db->ctdb->tunable.vacuum_default_interval; - } - - if (value.dptr != NULL) { - free(value.dptr); - } - - tdata.last_start = vdata->start; - tdata.last_duration = timeval_elapsed(&vdata->start); - - value.dptr = (unsigned char *)&tdata; - value.dsize = sizeof(tdata); - - if (tdb_store(tune_tdb, key, value, 0) != 0) { - DEBUG(DEBUG_ERR,(__location__ " Unable to store tundb record for %s\n", ctdb_db->db_name)); - tdb_transaction_cancel(tune_tdb); - tdb_close(tune_tdb); - talloc_free(tmp_ctx); - return -1; - } - tdb_transaction_commit(tune_tdb); - tdb_close(tune_tdb); - talloc_free(tmp_ctx); - - return 0; -} - /* * repack and vaccum a db * called from the child context @@ -985,9 +1137,9 @@ static int ctdb_vacuum_and_repack_db(struct ctdb_db_context *ctdb_db, vdata->ctdb = ctdb_db->ctdb; vdata->vacuum_limit = vacuum_limit; vdata->repack_limit = repack_limit; - vdata->delete_tree = trbt_create(vdata, 0); + vdata->delete_list = trbt_create(vdata, 0); vdata->ctdb_db = ctdb_db; - if (vdata->delete_tree == NULL) { + if (vdata->delete_list == NULL) { DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); talloc_free(vdata); return -1; @@ -1005,82 +1157,31 @@ static int ctdb_vacuum_and_repack_db(struct ctdb_db_context *ctdb_db, /* * decide if a repack is necessary */ - if (freelist_size < repack_limit && vdata->delete_count < vacuum_limit) + if (freelist_size < repack_limit && vdata->delete_left < vacuum_limit) { - update_tuning_db(ctdb_db, vdata, freelist_size); talloc_free(vdata); return 0; } DEBUG(DEBUG_INFO,("Repacking %s with %u freelist entries and %u records to delete\n", - name, freelist_size, vdata->delete_count)); + name, freelist_size, vdata->delete_left)); /* * repack and implicitely get rid of the records we can delete */ if (ctdb_repack_tdb(ctdb_db->ltdb->tdb, mem_ctx, vdata) != 0) { DEBUG(DEBUG_ERR,(__location__ " Failed to repack '%s'\n", name)); - update_tuning_db(ctdb_db, vdata, freelist_size); talloc_free(vdata); return -1; } - update_tuning_db(ctdb_db, vdata, freelist_size); talloc_free(vdata); return 0; } -static int get_vacuum_interval(struct ctdb_db_context *ctdb_db) +static uint32_t get_vacuum_interval(struct ctdb_db_context *ctdb_db) { - TALLOC_CTX *tmp_ctx = talloc_new(NULL); - TDB_CONTEXT *tdb; - TDB_DATA key, value; - char *vac_dbname; - uint interval = ctdb_db->ctdb->tunable.vacuum_default_interval; - struct ctdb_context *ctdb = ctdb_db->ctdb; - int flags; - - vac_dbname = talloc_asprintf(tmp_ctx, "%s/%s.%u", ctdb->db_directory, TUNINGDBNAME, ctdb->pnn); - if (vac_dbname == NULL) { - DEBUG(DEBUG_CRIT,(__location__ " Out of memory error while allocating '%s'\n", vac_dbname)); - talloc_free(tmp_ctx); - return interval; - } - - flags = ctdb_db->ctdb->valgrinding ? TDB_NOMMAP : 0; - flags |= TDB_DISALLOW_NESTING; - tdb = tdb_open(vac_dbname, 0, - flags, - O_RDWR|O_CREAT, 0600); - if (!tdb) { - DEBUG(DEBUG_ERR,("Unable to open/create database %s using default interval. Errno : %s (%d)\n", vac_dbname, strerror(errno), errno)); - talloc_free(tmp_ctx); - return interval; - } - - key.dptr = discard_const(ctdb_db->db_name); - key.dsize = strlen(ctdb_db->db_name); - - value = tdb_fetch(tdb, key); - - if (value.dptr != NULL) { - if (value.dsize == sizeof(struct vacuum_tuning_data)) { - struct vacuum_tuning_data *tptr = (struct vacuum_tuning_data *)value.dptr; - - interval = tptr->new_interval; - - if (interval < ctdb->tunable.vacuum_min_interval) { - interval = ctdb->tunable.vacuum_min_interval; - } - if (interval > ctdb->tunable.vacuum_max_interval) { - interval = ctdb->tunable.vacuum_max_interval; - } - } - free(value.dptr); - } - tdb_close(tdb); - - talloc_free(tmp_ctx); + uint32_t interval = ctdb_db->ctdb->tunable.vacuum_interval; return interval; } @@ -1171,7 +1272,9 @@ ctdb_vacuum_event(struct event_context *ev, struct timed_event *te, : ctdb->freeze_mode[ctdb_db->priority] == CTDB_FREEZE_PENDING ? "freeze pending" : "frozen")); - event_add_timed(ctdb->ev, vacuum_handle, timeval_current_ofs(ctdb->tunable.vacuum_default_interval, 0), ctdb_vacuum_event, vacuum_handle); + event_add_timed(ctdb->ev, vacuum_handle, + timeval_current_ofs(get_vacuum_interval(ctdb_db), 0), + ctdb_vacuum_event, vacuum_handle); return; } @@ -1186,7 +1289,9 @@ ctdb_vacuum_event(struct event_context *ev, struct timed_event *te, if (ret != 0) { talloc_free(child_ctx); DEBUG(DEBUG_ERR, ("Failed to create pipe for vacuum child process.\n")); - event_add_timed(ctdb->ev, vacuum_handle, timeval_current_ofs(ctdb->tunable.vacuum_default_interval, 0), ctdb_vacuum_event, vacuum_handle); + event_add_timed(ctdb->ev, vacuum_handle, + timeval_current_ofs(get_vacuum_interval(ctdb_db), 0), + ctdb_vacuum_event, vacuum_handle); return; } @@ -1200,7 +1305,9 @@ ctdb_vacuum_event(struct event_context *ev, struct timed_event *te, close(child_ctx->fd[1]); talloc_free(child_ctx); DEBUG(DEBUG_ERR, ("Failed to fork vacuum child process.\n")); - event_add_timed(ctdb->ev, vacuum_handle, timeval_current_ofs(ctdb->tunable.vacuum_default_interval, 0), ctdb_vacuum_event, vacuum_handle); + event_add_timed(ctdb->ev, vacuum_handle, + timeval_current_ofs(get_vacuum_interval(ctdb_db), 0), + ctdb_vacuum_event, vacuum_handle); return; } @@ -1301,6 +1408,57 @@ int ctdb_vacuum_init(struct ctdb_db_context *ctdb_db) return 0; } +static void remove_record_from_delete_queue(struct ctdb_db_context *ctdb_db, + const struct ctdb_ltdb_header *hdr, + const TDB_DATA key) +{ + struct delete_record_data *kd; + uint32_t hash; + + hash = (uint32_t)ctdb_hash(&key); + + DEBUG(DEBUG_DEBUG, (__location__ + " remove_record_from_delete_queue: " + "db[%s] " + "db_id[0x%08x] " + "key_hash[0x%08x] " + "lmaster[%u] " + "migrated_with_data[%s]\n", + ctdb_db->db_name, ctdb_db->db_id, + hash, + ctdb_lmaster(ctdb_db->ctdb, &key), + hdr->flags & CTDB_REC_FLAG_MIGRATED_WITH_DATA ? "yes" : "no")); + + kd = (struct delete_record_data *)trbt_lookup32(ctdb_db->delete_queue, hash); + if (kd == NULL) { + DEBUG(DEBUG_DEBUG, (__location__ + " remove_record_from_delete_queue: " + "record not in queue (hash[0x%08x])\n.", + hash)); + return; + } + + if ((kd->key.dsize != key.dsize) || + (memcmp(kd->key.dptr, key.dptr, key.dsize) != 0)) + { + DEBUG(DEBUG_DEBUG, (__location__ + " remove_record_from_delete_queue: " + "hash collision for key with hash[0x%08x] " + "in db[%s] - skipping\n", + hash, ctdb_db->db_name)); + return; + } + + DEBUG(DEBUG_DEBUG, (__location__ + " remove_record_from_delete_queue: " + "removing key with hash[0x%08x]\n", + hash)); + + talloc_free(kd); + + return; +} + /** * Insert a record into the ctdb_db context's delete queue, * handling hash collisions. @@ -1315,7 +1473,7 @@ static int insert_record_into_delete_queue(struct ctdb_db_context *ctdb_db, hash = (uint32_t)ctdb_hash(&key); - DEBUG(DEBUG_INFO, (__location__ " Schedule for deletion: db[%s] " + DEBUG(DEBUG_INFO, (__location__ " schedule for deletion: db[%s] " "db_id[0x%08x] " "key_hash[0x%08x] " "lmaster[%u] " @@ -1331,13 +1489,15 @@ static int insert_record_into_delete_queue(struct ctdb_db_context *ctdb_db, (memcmp(kd->key.dptr, key.dptr, key.dsize) != 0)) { DEBUG(DEBUG_INFO, - ("schedule for deletion: Hash collision (0x%08x)." - " Skipping the record.\n", hash)); + (__location__ " schedule for deletion: " + "hash collision for key hash [0x%08x]. " + "Skipping the record.\n", hash)); return 0; } else { DEBUG(DEBUG_DEBUG, - ("schedule for deletion: Overwriting entry for " - "key with hash 0x%08x.\n", hash)); + (__location__ " schedule for deletion: " + "updating entry for key with hash [0x%08x].\n", + hash)); } } @@ -1345,6 +1505,10 @@ static int insert_record_into_delete_queue(struct ctdb_db_context *ctdb_db, ctdb_db->delete_queue, hdr, key); if (ret != 0) { + DEBUG(DEBUG_INFO, + (__location__ " schedule for deletion: error " + "inserting key with hash [0x%08x] into delete queue\n", + hash)); return -1; } @@ -1435,3 +1599,20 @@ int32_t ctdb_local_schedule_for_deletion(struct ctdb_db_context *ctdb_db, return ret; } + +void ctdb_local_remove_from_delete_queue(struct ctdb_db_context *ctdb_db, + const struct ctdb_ltdb_header *hdr, + const TDB_DATA key) +{ + if (ctdb_db->ctdb->ctdbd_pid != getpid()) { + /* + * Only remove the record from the delete queue if called + * in the main daemon. + */ + return; + } + + remove_record_from_delete_queue(ctdb_db, hdr, key); + + return; +} diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c index 77c45dc58e..40abb5c810 100644 --- a/ctdb/tools/ctdb.c +++ b/ctdb/tools/ctdb.c @@ -3902,7 +3902,7 @@ static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv return -1; } - printf("%-19s = %u\n", name, value); + printf("%-23s = %u\n", name, value); return 0; } @@ -5289,7 +5289,7 @@ static const struct { { "chksrvid", chksrvid, false, false, "check if a server id exists", "<pnn> <type> <id>" }, { "getsrvids", getsrvids, false, false, "get a list of all server ids"}, { "check_srvids", check_srvids, false, false, "check if a srvid exists", "<id>+" }, - { "vacuum", ctdb_vacuum, false, false, "vacuum the databases of empty records", "[max_records]"}, + { "vacuum", ctdb_vacuum, false, true, "vacuum the databases of empty records", "[max_records]"}, { "repack", ctdb_repack, false, false, "repack all databases", "[max_freelist]"}, { "listnodes", control_listnodes, false, true, "list all nodes in the cluster"}, { "reloadnodes", control_reload_nodes_file, false, false, "reload the nodes file and restart the transport on all nodes"}, diff --git a/ctdb/tools/ctdb_vacuum.c b/ctdb/tools/ctdb_vacuum.c index 3bde7b384d..e90e130acd 100644 --- a/ctdb/tools/ctdb_vacuum.c +++ b/ctdb/tools/ctdb_vacuum.c @@ -30,451 +30,12 @@ #define TIMELIMIT() timeval_current_ofs(10, 0) -/* - a list of records to possibly delete - */ -struct vacuum_data { - uint32_t vacuum_limit; - struct ctdb_context *ctdb; - struct ctdb_db_context *ctdb_db; - trbt_tree_t *delete_tree; - uint32_t delete_count; - struct ctdb_marshall_buffer **list; - bool traverse_error; - uint32_t total; -}; - -/* this structure contains the information for one record to be deleted */ -struct delete_record_data { - struct ctdb_context *ctdb; - struct ctdb_db_context *ctdb_db; - struct ctdb_ltdb_header hdr; - TDB_DATA key; -}; - -/* - traverse function for vacuuming - */ -static int vacuum_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private) -{ - struct vacuum_data *vdata = talloc_get_type(private, struct vacuum_data); - struct ctdb_context *ctdb = vdata->ctdb; - struct ctdb_db_context *ctdb_db = vdata->ctdb_db; - uint32_t lmaster; - struct ctdb_ltdb_header *hdr; - struct ctdb_rec_data *rec; - size_t old_size; - - lmaster = ctdb_lmaster(ctdb, &key); - if (lmaster >= ctdb->vnn_map->size) { - return 0; - } - - if (data.dsize != sizeof(struct ctdb_ltdb_header)) { - /* its not a deleted record */ - return 0; - } - - hdr = (struct ctdb_ltdb_header *)data.dptr; - - if (hdr->dmaster != ctdb->pnn) { - return 0; - } - - - /* is this a records we could possibly delete? I.e. - if the record is empty and also we are both lmaster - and dmaster for the record we should be able to delete it - */ - if ( (lmaster == ctdb->pnn) - &&( (vdata->delete_count < vdata->vacuum_limit) - ||(vdata->vacuum_limit == 0) ) ){ - uint32_t hash; - - hash = ctdb_hash(&key); - if (trbt_lookup32(vdata->delete_tree, hash)) { - DEBUG(DEBUG_INFO, (__location__ " Hash collission when vacuuming, skipping this record.\n")); - } else { - struct delete_record_data *dd; - - /* store key and header indexed by the key hash */ - dd = talloc_zero(vdata->delete_tree, struct delete_record_data); - if (dd == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - dd->ctdb = ctdb; - dd->ctdb_db = ctdb_db; - dd->key.dsize = key.dsize; - dd->key.dptr = talloc_memdup(dd, key.dptr, key.dsize); - if (dd->key.dptr == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - - dd->hdr = *hdr; - - - trbt_insert32(vdata->delete_tree, hash, dd); - - vdata->delete_count++; - } - } - - - /* add the record to the blob ready to send to the nodes */ - rec = ctdb_marshall_record(vdata->list[lmaster], ctdb->pnn, key, NULL, tdb_null); - if (rec == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - vdata->traverse_error = true; - return -1; - } - old_size = talloc_get_size(vdata->list[lmaster]); - vdata->list[lmaster] = talloc_realloc_size(NULL, vdata->list[lmaster], - old_size + rec->length); - if (vdata->list[lmaster] == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Failed to expand\n")); - vdata->traverse_error = true; - return -1; - } - vdata->list[lmaster]->count++; - memcpy(old_size+(uint8_t *)vdata->list[lmaster], rec, rec->length); - talloc_free(rec); - - vdata->total++; - - /* don't gather too many records */ - if (vdata->vacuum_limit != 0 && - vdata->total == vdata->vacuum_limit) { - return -1; - } - - return 0; -} - -struct delete_records_list { - struct ctdb_marshall_buffer *records; -}; - -/* - traverse the tree of records to delete and marshall them into - a blob -*/ -static int -delete_traverse(void *param, void *data) -{ - struct delete_record_data *dd = talloc_get_type(data, struct delete_record_data); - struct delete_records_list *recs = talloc_get_type(param, struct delete_records_list); - struct ctdb_rec_data *rec; - size_t old_size; - - rec = ctdb_marshall_record(dd, recs->records->db_id, dd->key, &dd->hdr, tdb_null); - if (rec == NULL) { - DEBUG(DEBUG_ERR, (__location__ " failed to marshall record\n")); - return 0; - } - - old_size = talloc_get_size(recs->records); - recs->records = talloc_realloc_size(NULL, recs->records, old_size + rec->length); - if (recs->records == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Failed to expand\n")); - return 0; - } - recs->records->count++; - memcpy(old_size+(uint8_t *)(recs->records), rec, rec->length); - return 0; -} - - -static int delete_record(void *param, void *d) -{ - struct delete_record_data *dd = talloc_get_type(d, struct delete_record_data); - struct ctdb_context *ctdb = dd->ctdb; - struct ctdb_db_context *ctdb_db = dd->ctdb_db; - uint32_t *count = (uint32_t *)param; - struct ctdb_ltdb_header *hdr; - TDB_DATA data; - - /* its deleted on all other nodes - refetch, check and delete */ - if (tdb_chainlock_nonblock(ctdb_db->ltdb->tdb, dd->key) != 0) { - /* the chain is busy - come back later */ - return 0; - } - - data = tdb_fetch(ctdb_db->ltdb->tdb, dd->key); - if (data.dptr == NULL) { - tdb_chainunlock(ctdb_db->ltdb->tdb, dd->key); - return 0; - } - if (data.dsize != sizeof(struct ctdb_ltdb_header)) { - free(data.dptr); - tdb_chainunlock(ctdb_db->ltdb->tdb, dd->key); - return 0; - } - - hdr = (struct ctdb_ltdb_header *)data.dptr; - - /* if we are not the lmaster and the dmaster then skip the record */ - if (hdr->dmaster != ctdb->pnn || - ctdb_lmaster(ctdb, &(dd->key)) != ctdb->pnn || - dd->hdr.rsn != hdr->rsn) { - tdb_chainunlock(ctdb_db->ltdb->tdb, dd->key); - free(data.dptr); - return 0; - } - - ctdb_block_signal(SIGALRM); - tdb_delete(ctdb_db->ltdb->tdb, dd->key); - ctdb_unblock_signal(SIGALRM); - tdb_chainunlock(ctdb_db->ltdb->tdb, dd->key); - free(data.dptr); - - (*count)++; - return 0; -} - -/* vacuum one database */ -static int ctdb_vacuum_db(struct ctdb_context *ctdb, uint32_t db_id, struct ctdb_node_map *map, - bool persistent, uint32_t vacuum_limit) -{ - struct ctdb_db_context *ctdb_db; - const char *name; - struct vacuum_data *vdata; - int i; - - vdata = talloc_zero(ctdb, struct vacuum_data); - if (vdata == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - - vdata->ctdb = ctdb; - vdata->vacuum_limit = vacuum_limit; - vdata->delete_tree = trbt_create(vdata, 0); - if (vdata->delete_tree == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - - if (ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, db_id, vdata, &name) != 0) { - DEBUG(DEBUG_ERR,(__location__ " Failed to get name of db 0x%x\n", db_id)); - talloc_free(vdata); - return -1; - } - - ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), name, persistent, 0); - if (ctdb_db == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Failed to attach to database '%s'\n", name)); - talloc_free(vdata); - return -1; - } - vdata->ctdb_db = ctdb_db; - - /* the list needs to be of length num_nodes */ - vdata->list = talloc_array(vdata, struct ctdb_marshall_buffer *, ctdb->vnn_map->size); - if (vdata->list == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - talloc_free(vdata); - return -1; - } - for (i=0;i<ctdb->vnn_map->size;i++) { - vdata->list[i] = (struct ctdb_marshall_buffer *) - talloc_zero_size(vdata->list, - offsetof(struct ctdb_marshall_buffer, data)); - if (vdata->list[i] == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - talloc_free(vdata); - return -1; - } - vdata->list[i]->db_id = db_id; - } - - /* traverse, looking for records that might be able to be vacuumed */ - if (tdb_traverse_read(ctdb_db->ltdb->tdb, vacuum_traverse, vdata) == -1 || - vdata->traverse_error) { - DEBUG(DEBUG_ERR,(__location__ " Traverse error in vacuuming '%s'\n", name)); - talloc_free(vdata); - return -1; - } - - - for (i=0;i<ctdb->vnn_map->size;i++) { - if (vdata->list[i]->count == 0) { - continue; - } - - /* for records where we are not the lmaster, tell the lmaster to fetch the record */ - if (ctdb->vnn_map->map[i] != ctdb->pnn) { - TDB_DATA data; - printf("Found %u records for lmaster %u in '%s'\n", vdata->list[i]->count, i, name); - - data.dsize = talloc_get_size(vdata->list[i]); - data.dptr = (void *)vdata->list[i]; - if (ctdb_client_send_message(ctdb, ctdb->vnn_map->map[i], CTDB_SRVID_VACUUM_FETCH, data) != 0) { - DEBUG(DEBUG_ERR,(__location__ " Failed to send vacuum fetch message to %u\n", - ctdb->vnn_map->map[i])); - talloc_free(vdata); - return -1; - } - continue; - } - } - - - /* Process all records we can delete (if any) */ - if (vdata->delete_count > 0) { - struct delete_records_list *recs; - TDB_DATA indata, outdata; - int ret; - int32_t res; - uint32_t count; - - recs = talloc_zero(vdata, struct delete_records_list); - if (recs == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - recs->records = (struct ctdb_marshall_buffer *) - talloc_zero_size(vdata, - offsetof(struct ctdb_marshall_buffer, data)); - if (recs->records == NULL) { - DEBUG(DEBUG_ERR,(__location__ " Out of memory\n")); - return -1; - } - recs->records->db_id = db_id; - - /* traverse the tree of all records we want to delete and - create a blob we can send to the other nodes. - */ - trbt_traversearray32(vdata->delete_tree, 1, delete_traverse, recs); - - indata.dsize = talloc_get_size(recs->records); - indata.dptr = (void *)recs->records; - - /* now tell all the other nodes to delete all these records - (if possible) - */ - for (i=0;i<ctdb->vnn_map->size;i++) { - struct ctdb_marshall_buffer *records; - struct ctdb_rec_data *rec; - - if (ctdb->vnn_map->map[i] == ctdb->pnn) { - /* we dont delete the records on the local node - just yet - */ - continue; - } - - ret = ctdb_control(ctdb, ctdb->vnn_map->map[i], 0, - CTDB_CONTROL_TRY_DELETE_RECORDS, 0, - indata, recs, &outdata, &res, - NULL, NULL); - if (ret != 0 || res != 0) { - DEBUG(DEBUG_ERR,("Failed to delete records on node %u\n", ctdb->vnn_map->map[i])); - exit(10); - } - - /* outdata countains the list of records coming back - from the node which the node could not delete - */ - records = (struct ctdb_marshall_buffer *)outdata.dptr; - rec = (struct ctdb_rec_data *)&records->data[0]; - while (records->count-- > 1) { - TDB_DATA reckey, recdata; - struct ctdb_ltdb_header *rechdr; - - reckey.dptr = &rec->data[0]; - reckey.dsize = rec->keylen; - recdata.dptr = &rec->data[reckey.dsize]; - recdata.dsize = rec->datalen; - - if (recdata.dsize < sizeof(struct ctdb_ltdb_header)) { - DEBUG(DEBUG_CRIT,(__location__ " bad ltdb record\n")); - exit(10); - } - rechdr = (struct ctdb_ltdb_header *)recdata.dptr; - recdata.dptr += sizeof(*rechdr); - recdata.dsize -= sizeof(*rechdr); - - /* that other node couldnt delete the record - so we shouldnt delete it either. - remove it from the tree. - */ - talloc_free(trbt_lookup32(vdata->delete_tree, ctdb_hash(&reckey))); - - rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec); - } - } - - - /* the only records remaining in the tree would be those - records where all other nodes could successfully - delete them, so we can now safely delete them on the - lmaster as well. - */ - count = 0; - trbt_traversearray32(vdata->delete_tree, 1, delete_record, &count); - if (vdata->delete_count != 0) { - printf("Deleted %u records out of %u on this node from '%s'\n", count, vdata->delete_count, name); - } - } - - /* this ensures we run our event queue */ - ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE); - - talloc_free(vdata); - - return 0; -} - - /* vacuum all our databases */ int ctdb_vacuum(struct ctdb_context *ctdb, int argc, const char **argv) { - struct ctdb_dbid_map *dbmap=NULL; - struct ctdb_node_map *nodemap=NULL; - int ret, i, pnn; - uint32_t vacuum_limit = 0; - - if (argc > 0) { - vacuum_limit = atoi(argv[0]); - } - - ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &dbmap); - if (ret != 0) { - DEBUG(DEBUG_ERR, ("Unable to get dbids from local node\n")); - return ret; - } - - ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &nodemap); - if (ret != 0) { - DEBUG(DEBUG_ERR, ("Unable to get nodemap from local node\n")); - return ret; - } - - ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE, ctdb, &ctdb->vnn_map); - if (ret != 0) { - DEBUG(DEBUG_ERR, ("Unable to get vnnmap from local node\n")); - return ret; - } - - pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE); - if (pnn == -1) { - DEBUG(DEBUG_ERR, ("Unable to get pnn from local node\n")); - return -1; - } - ctdb->pnn = pnn; - - for (i=0;i<dbmap->num;i++) { - if (ctdb_vacuum_db(ctdb, dbmap->dbs[i].dbid, nodemap, - dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT, vacuum_limit) != 0) { - DEBUG(DEBUG_ERR,("Failed to vacuum db 0x%x\n", dbmap->dbs[i].dbid)); - return -1; - } - } - + printf("\"ctdb vacuum\" is not implemented any more.\n"); return 0; } |