Make fetch_locked more scalable

This patch improves the handling of the fetch_lock operation on non-persistent
databases that ctdb clients have to do very frequently.

The normal flow how this goes is the following:

1. Client does a local fetch_lock on the database

2. Client looks if the local node is dmaster.
   If yes, everything is fine
   If no, continue here

3. Client unlocks the local record

4. Client issues a "get me the record" call to ctdbd

5. ctdbd goes out and fetches the dmaster role

6. ctdbd tells the client to retry

7. Client starts over again

The problem is between step 6 and 7: Before the client has had the chance to
retry (i.e. catch the record with a fetch_locked), another node might have come
asking ctdbd to migrate away the record again. This is a real problem, I've
seen >20 loops of this kind in real workloads.

This patch does the following: Whenever ctdb receives a record as result of
step 5, it puts the key on a "holdback list". As long as a key is on this list,
a request to migrate away the dmaster is put on hold. It is the client's duty
to issue the "CTDB_CONTROL_GOTIT" control when it has successfully done step 2
after having asked ctdb to fetch the record. This will release the key from the
"holdback list" and re-issue all dmaster migration requests.

As a safeguard against malicious clients, once a second (default 1000msecs,
tunable "HoldbackCleanupInterval" in milliseconds) ctdbd goes over the list of
held back keys, deletes them and releases all held back migration requests.

(This used to be ctdb commit 5736e17c139c9a8049e235429aeae0c6c9d0e93d)

1 files changed, 23 insertions, 0 deletions

diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index e4f4aba091..041c0e3f89 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -95,6 +95,7 @@ struct ctdb_tunable {
 	uint32_t traverse_timeout;
 	uint32_t keepalive_interval;
 	uint32_t keepalive_limit;
+	uint32_t holdback_cleanup_interval;
 	uint32_t max_lacount;
 	uint32_t recover_timeout;
 	uint32_t recover_interval;
@@ -397,6 +398,7 @@ struct ctdb_context {
 	uint32_t recovery_mode;
 	TALLOC_CTX *tickle_update_context;
 	TALLOC_CTX *keepalive_ctx;
+	struct timed_event *holdback_cleanup_te;
 	struct ctdb_tunable tunable;
 	enum ctdb_freeze_mode freeze_mode[NUM_DB_PRIORITIES+1];
 	struct ctdb_freeze_handle *freeze_handles[NUM_DB_PRIORITIES+1];
@@ -478,6 +480,17 @@ struct ctdb_db_context {
 	struct ctdb_traverse_local_handle *traverse;
 	bool transaction_active;
 	struct ctdb_vacuum_handle *vacuum_handle;
+
+	/*
+	 * The keys to hold back until CTDB_CONTROL_GOTIT is being
+	 * sent by a client having forced a migration to us.
+	 */
+	uint8_t **holdback_keys;
+
+	/*
+	 * The CTDB_REQ_CALLs held back according to "holdback_keys"
+	 */
+	struct ctdb_req_header **held_back;
 };
 
 
@@ -627,6 +640,7 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0,
 		    CTDB_CONTROL_CLEAR_LOG		 = 118,
 		    CTDB_CONTROL_TRANS3_COMMIT           = 119,
 		    CTDB_CONTROL_GET_DB_SEQNUM           = 120,
+		    CTDB_CONTROL_GOTIT			 = 121,
 };	
 
 /*
@@ -1170,6 +1184,11 @@ struct ctdb_control_wipe_database {
 	uint32_t transaction_id;
 };
 
+struct ctdb_control_gotit {
+	uint32_t db_id;
+	uint8_t key[1];
+};
+
 /*
   state of a in-progress ctdb call in client
 */
@@ -1238,6 +1257,10 @@ void ctdb_start_keepalive(struct ctdb_context *ctdb);
 void ctdb_stop_keepalive(struct ctdb_context *ctdb);
 int32_t ctdb_run_eventscripts(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA data, bool *async_reply);
 
+void ctdb_start_holdback_cleanup(struct ctdb_context *ctdb);
+void ctdb_stop_holdback_cleanup(struct ctdb_context *ctdb);
+int32_t ctdb_control_gotit(struct ctdb_context *ctdb, TDB_DATA indata);
+
 
 void ctdb_daemon_cancel_controls(struct ctdb_context *ctdb, struct ctdb_node *node);
 void ctdb_call_resend_all(struct ctdb_context *ctdb);