Niceify the readonlyrecord API. Dont force clients to be exposed to the featch_with_header function

We dont strictly need to force clients to use CTDB_FETCH_WITH_HEADER instead of CTDB_FETCH when they ask for readonly records.
Have ctdbd internally remap this internally to FETCH_WITH_HEADER and map the reply back to CTDB_FETCH_FUNC or CTDB_FETCH_WITH_HEADER_FUNC based on what the client initially asked for.

This removes the need for the client to know about the CTDB_FETCH_WITH_HEADER_FUNC function and simplifies the client code.
Clients that do not care what the header after the request is can just continue using the old CTDB_FETCH_FUNC call and ctdbd will do all the difficult stuff.

(This used to be ctdb commit 444a7bac4e9a854b06c1ad4cb36c2b58a72001fa)

3 files changed, 47 insertions, 8 deletions

diff --git a/ctdb/doc/readonlyrecords.txt b/ctdb/doc/readonlyrecords.txt
index acdab2e49b..f8f10954ed 100644
--- a/ctdb/doc/readonlyrecords.txt
+++ b/ctdb/doc/readonlyrecords.txt
@@ -89,7 +89,10 @@ This new database is used for tracking delegations for the records. A record in
 This tracking database is lockless, using TDB_NOLOCK, and is only ever accessed by the main ctdbd daemon.
 The lockless nature and the fact that no other process ever access this TDB means we are guranteed non-blocking access to records in the trcking database.
 
-The ctdb_call PDU is allocated with two new flags WANT_READONLY and WITH_HEADER.
+The ctdb_call PDU is allocated with a new flags WANT_READONLY and possibly also a new callid: CTDB_FETCH_WITH_HEADER_FUNC.
+This new function returns not only the record, as CTDB_FETCH_FUNC does, but also returns the HEADER prepended to the record.
+This function is optional, clients that do not care what the header is can continue using just CTDB_FETCH_FUNC
+
 This first flag is used to explicitely requesting a read-only record from the DMASTER/LMASTER.
 The second flag is used to request that the fetch operation will return not only the data for the record but also
 the record header. 
@@ -137,7 +140,7 @@ This will change to instead do
                 goto finished
             else
                 unlock record 
-                ask ctdb for read-only copy (WANT_READONLY|WITH_HEADER)
+                ask ctdb for read-only copy (WANT_READONLY[|WITH_HEADER])
                 if failed to get read-only copy (*A)
                     ask ctdb to migrate the record onto the node
                     goto try_again
diff --git a/ctdb/libctdb/ctdb.c b/ctdb/libctdb/ctdb.c
index 2f694a1c34..13ccf9e4de 100644
--- a/ctdb/libctdb/ctdb.c
+++ b/ctdb/libctdb/ctdb.c
@@ -832,13 +832,13 @@ static void readrecordlock_retry(struct ctdb_connection *ctdb,
 	struct ctdb_reply_call *reply;
 	TDB_DATA data;
 
-	/* OK, we've received reply to fetch-with-header migration */
-	reply = unpack_reply_call(req, CTDB_FETCH_WITH_HEADER_FUNC);
+	/* OK, we've received reply to fetch migration */
+	reply = unpack_reply_call(req, CTDB_FETCH_FUNC);
 	if (!reply || reply->status != 0) {
 		if (reply) {
 			DEBUG(ctdb, LOG_ERR,
 			      "ctdb_readrecordlock_async(async):"
-			      " FETCH_WITH_HEADER_FUNC returned %i", reply->status);
+			      " FETCH returned %i", reply->status);
 		}
 		lock->callback(lock->ctdb_db, NULL, tdb_null, private);
 		ctdb_request_free(req); /* Also frees lock. */
@@ -920,7 +920,7 @@ ctdb_readrecordlock_internal(struct ctdb_db *ctdb_db, TDB_DATA key,
 		req->hdr.call->flags = CTDB_IMMEDIATE_MIGRATION;
 	}
 	req->hdr.call->db_id = ctdb_db->id;
-	req->hdr.call->callid = CTDB_FETCH_WITH_HEADER_FUNC;
+	req->hdr.call->callid = CTDB_FETCH_FUNC;
 	req->hdr.call->hopcount = 0;
 	req->hdr.call->keylen = key.dsize;
 	req->hdr.call->calldatalen = 0;
diff --git a/ctdb/server/ctdb_daemon.c b/ctdb/server/ctdb_daemon.c
index 69488dfac4..53f47d67fa 100644
--- a/ctdb/server/ctdb_daemon.c
+++ b/ctdb/server/ctdb_daemon.c
@@ -311,6 +311,10 @@ struct daemon_call_state {
 	uint32_t reqid;
 	struct ctdb_call *call;
 	struct timeval start_time;
+
+	/* readonly request ? */
+	uint32_t readonly_fetch;
+	uint32_t client_callid;
 };
 
 /* 
@@ -339,6 +343,16 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
 	}
 
 	length = offsetof(struct ctdb_reply_call, data) + dstate->call->reply_data.dsize;
+	/* If the client asked for readonly FETCH, we remapped this to 
+	   FETCH_WITH_HEADER when calling the daemon. So we must
+	   strip the extra header off the reply data before passing
+	   it back to the client.
+	*/
+	if (dstate->readonly_fetch
+	&& dstate->client_callid == CTDB_FETCH_FUNC) {
+		length -= sizeof(struct ctdb_ltdb_header);
+	}
+
 	r = ctdbd_allocate_pkt(client->ctdb, dstate, CTDB_REPLY_CALL, 
 			       length, struct ctdb_reply_call);
 	if (r == NULL) {
@@ -348,9 +362,19 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
 		return;
 	}
 	r->hdr.reqid        = dstate->reqid;
-	r->datalen          = dstate->call->reply_data.dsize;
 	r->status           = dstate->call->status;
-	memcpy(&r->data[0], dstate->call->reply_data.dptr, r->datalen);
+
+	if (dstate->readonly_fetch
+	&& dstate->client_callid == CTDB_FETCH_FUNC) {
+		/* client only asked for a FETCH so we must strip off
+		   the extra ctdb_ltdb header
+		*/
+		r->datalen          = dstate->call->reply_data.dsize - sizeof(struct ctdb_ltdb_header);
+		memcpy(&r->data[0], dstate->call->reply_data.dptr + sizeof(struct ctdb_ltdb_header), r->datalen);
+	} else {
+		r->datalen          = dstate->call->reply_data.dsize;
+		memcpy(&r->data[0], dstate->call->reply_data.dptr, r->datalen);
+	}
 
 	res = daemon_queue_send(client, &r->hdr);
 	if (res == -1) {
@@ -738,12 +762,24 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
 		return;
 	}
 
+	dstate->readonly_fetch = 0;
 	call->call_id = c->callid;
 	call->key = key;
 	call->call_data.dptr = c->data + c->keylen;
 	call->call_data.dsize = c->calldatalen;
 	call->flags = c->flags;
 
+	if (c->flags & CTDB_WANT_READONLY) {
+		/* client wants readonly record, so translate this into a 
+		   fetch with header. remember what the client asked for
+		   so we can remap the reply back to the proper format for
+		   the client in the reply
+		 */
+		dstate->client_callid = call->call_id;
+		call->call_id = CTDB_FETCH_WITH_HEADER_FUNC;
+		dstate->readonly_fetch = 1;
+	}
+
 	if (header.dmaster == ctdb->pnn) {
 		state = ctdb_call_local_send(ctdb_db, call, &header, &data);
 	} else {