[PATCH 9/9] NFSD: Send referring calls with CB_RECALL

When CB_RECALL races ahead of the reply that granted the delegation,
the client has not yet recorded the delegation stateid and responds
NFS4ERR_BADHANDLE or NFS4ERR_BAD_STATEID. The slot that carried the
grant has not retired at that point, so NFSD cannot read the rejection
as proof that the client never held the delegation. It retries the
recall and, once the retries lapse, revokes a delegation the client is
by then able to return.

Remove the ambiguity with the referring call mechanism of RFC 8881
Section 2.10.6.3: until the slot that carried the grant retires, name
that request as a referring call in the CB_SEQUENCE of each recall. A
client that finds it still outstanding may respond NFS4ERR_DELAY, and
the recall is retried until the client has processed the grant.

A recall reuses one callback context across its retries, and ->prepare
does not run on every send. The granting request does not change, so a
send that inherits the previous list sends the right one. Retirement
of the granting slot drops the list, and nfs4_free_deleg() releases
what is left.

Signed-off-by: Chuck Lever <[email protected]>
---
 fs/nfsd/nfs4callback.c |  7 +++++--
 fs/nfsd/nfs4state.c    | 54 +++++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 509195d488c9..9afe2d78d39d 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -1530,12 +1530,14 @@ void nfsd41_cb_referring_call(struct nfsd4_callback *cb,
 
 /**
  * nfsd41_cb_destroy_referring_call_list - release referring call info
- * @cb: context of a callback that has completed
+ * @cb: context of callback to release referring calls from
  *
  * Callers who allocate referring calls using nfsd41_cb_referring_call() must
  * release those resources by calling nfsd41_cb_destroy_referring_call_list.
  *
- * Caller serializes access to @cb.
+ * Caller serializes access to @cb. No CB_COMPOUND for @cb may be in
+ * flight, because encode_cb_sequence4args() walks this list as it
+ * encodes.
  */
 void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb)
 {
@@ -1557,6 +1559,7 @@ void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb)
 		list_del(&rcl->__list);
 		kfree(rcl);
 	}
+	cb->cb_nr_referring_call_list = 0;
 }
 
 static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 5774c7a1b3de..510380b6aa7a 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1167,6 +1167,8 @@ static void nfs4_free_deleg(struct nfs4_stid *stid)
 	WARN_ON_ONCE(!list_empty(&dp->dl_perfile));
 	WARN_ON_ONCE(!list_empty(&dp->dl_perclnt));
 	WARN_ON_ONCE(!list_empty(&dp->dl_recall_lru));
+	/* The list outlives one recall, so ->release() cannot free it. */
+	nfsd41_cb_destroy_referring_call_list(&dp->dl_recall);
 	kmem_cache_free(deleg_slab, stid);
 	atomic_long_dec(&num_delegations);
 }
@@ -6091,6 +6093,18 @@ bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode)
 	return timeo > 0;
 }
 
+/*
+ * gen_sessionid() composes a sessionid from the client's clientid and a
+ * sequence counter, so the sequence alone identifies the granting session.
+ */
+static void nfsd4_recall_grant_sessionid(const struct nfs4_delegation *dp,
+					 struct nfsd4_sessionid *sid)
+{
+	sid->clientid = dp->dl_stid.sc_client->cl_clientid;
+	sid->sequence = dp->dl_recall_grant.sessionid_seq;
+	sid->reserved = 0;
+}
+
 static bool nfsd4_recall_grant_slot_retired(struct nfs4_delegation *dp)
 {
 	struct nfs4_client *clp = dp->dl_stid.sc_client;
@@ -6103,14 +6117,7 @@ static bool nfsd4_recall_grant_slot_retired(struct nfs4_delegation *dp)
 	if (!dp->dl_recall_grant.valid)
 		return false;
 
-	/*
-	 * gen_sessionid() composes a sessionid from the client's clientid
-	 * and a sequence counter, so the sequence alone identifies the
-	 * granting session.
-	 */
-	sid.clientid = clp->cl_clientid;
-	sid.sequence = dp->dl_recall_grant.sessionid_seq;
-	sid.reserved = 0;
+	nfsd4_recall_grant_sessionid(dp, &sid);
 
 	/*
 	 * A missing session does not prove the client saw the grant: a
@@ -6145,6 +6152,21 @@ static bool nfsd4_recall_grant_slot_retired(struct nfs4_delegation *dp)
 	return retired;
 }
 
+/*
+ * ->prepare does not run on every send: nfsd4_run_cb_work() skips it
+ * on a requeue, and a retry via rpc_restart_call_prepare() re-enters
+ * the RPC layer beneath it. The granting request does not change, so
+ * a send inherits a correct list. Retirement is the one transition
+ * the list has to follow.
+ */
+static void nfsd4_refresh_recall_grant(struct nfs4_delegation *dp)
+{
+	dp->dl_recall_grant.retired_at_send =
+			nfsd4_recall_grant_slot_retired(dp);
+	if (dp->dl_recall_grant.retired_at_send)
+		nfsd41_cb_destroy_referring_call_list(&dp->dl_recall);
+}
+
 static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 {
 	struct nfs4_delegation *dp = cb_to_delegation(cb);
@@ -6167,8 +6189,17 @@ static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 	}
 	spin_unlock(&nn->deleg_lock);
 
-	dp->dl_recall_grant.retired_at_send =
-			nfsd4_recall_grant_slot_retired(dp);
+	nfsd4_refresh_recall_grant(dp);
+
+	if (dp->dl_recall_grant.valid && !dp->dl_recall_grant.retired_at_send) {
+		struct nfsd4_sessionid sid;
+
+		nfsd4_recall_grant_sessionid(dp, &sid);
+		nfsd41_cb_referring_call(&dp->dl_recall,
+					 (struct nfs4_sessionid *)&sid,
+					 dp->dl_recall_grant.slotid,
+					 dp->dl_recall_grant.seqid);
+	}
 	return true;
 }
 
@@ -6225,8 +6256,7 @@ static int nfsd4_cb_recall_done(struct nfsd4_callback *cb,
 			return 1;
 		}
 		if (!dp->dl_stid.sc_status && dp->dl_retries--) {
-			dp->dl_recall_grant.retired_at_send =
-					nfsd4_recall_grant_slot_retired(dp);
+			nfsd4_refresh_recall_grant(dp);
 			rpc_delay(task, 2 * HZ);
 			return 0;
 		}

-- 
2.54.0