Re: nfs server issues

On Wed, Jul 8, 2026 at 7:57 AM Rick Macklem <[email protected]> wrote:
>
> On Wed, Jul 8, 2026 at 2:28 AM Richard Purdie
> <[email protected]> wrote:
> >
> > On Tue, 2026-07-07 at 15:54 -0700, Michael Halstead wrote:
> > > On Tue, Jul 7, 2026 at 1:52 AM Richard Purdie <[email protected]> wrote:
> > > > On Sat, 2026-07-04 at 12:41 -0700, Rick Macklem wrote:
> > > > > On Sat, Jul 4, 2026 at 1:05 AM Richard Purdie
> > > > > <[email protected]> wrote:
> > > > > >
> > > > > > nfsstat -E -s
> > > > > > Server Info:
> > > > > >       Getattr      Setattr       Lookup     Readlink         Read        Write
> > > > > >   17354100116   1674996998   5136769057        40544  18429670397    986443058
> > > > > >        Create       Remove       Rename         Link      Symlink        Mkdir
> > > > > >             0     98646613    119865835     17071853            0            0
> > > > > >         Rmdir      Readdir     RdirPlus       Access        Mknod       Fsstat
> > > > > >             0   1655544711            0   4129535844            0            0
> > > > > >        FSinfo     pathConf       Commit      LookupP      SetClId    SetClIdCf
> > > > > >             0            0      8723436            0            0            0
> > > > > One more thing to note. The # of Commits is much smaller than the # of Writes
> > > > > on the top line. This "hints" that a lot of the Writes are being done File_sync.
> > > > > (To check that, you'd need to capture packets and look at them in wireshark.)
> > >
> > >
> > > I've captured data
> > > at https://valkyrie.yocto.io/pub/non-release/mtest/nfs_capture2.pcap.
> > > zst from a fresh reboot running the testlock.py for about 100MB of
> > > data and umounting the filesystems. Please let me know if this
> > > reveals the issue or what else I can offer.
> >
> > Thanks Michael. I think that didn't have enough parallel test scripts
> > running. This is one I just captured with more parallelism (4 clients),
> > just tracing one of them:
> >
> > https://valkyrie.yocto.io/pub/non-release/mtest/rp_capture.pcap.zst
> >
> > This one has a lot of NFS4ERR_STALE in it which I suspect is what we
> > were looking for...
> Yep. This one definitely shows RPCs that would do Close failing when
> PutFH replies NFS4ERR_STALE. (I looked at the previous one and couldn't
> see any problems in it, although the previous one does have the weird case
> where the same Lock is acquired twice. Both succeed because it is the
> same client and lock_owner.  I am going to check to see what the RFCs
> say w.r.t. duplicate lock operations. Like Open, the Lock ops were modelled
> after what Windows wants, but were intended to handle POSIX lock semantics
> as well. Of course, no one implements mandatory locking and the protocol
> doesn't even allow the client to figure out which the server is doing, but
> that's another story.;-)
>
> I do think I can come up with a simple solution for this:
> - Since the operations that get rid of state all require a CFH, which
>   must be set by PutFH.. I cannot see any reason to keep state once
>   the PutFH is returning NFS4ERR_STALE for the file.
>   --> So, I think I can just free all the state for that FH at that time.
>       (The state is keyed on FH and not a vnode, so the PutFH has
>        the required FH.)
> --> I'll post a patch here later to-day. Hopefully you guys can run a
>       kernel from patched sources to test it.
I've attached a patch that I think fixes this (minimally tested).
If you can test this by applying it to your kernel sources and building a
kernel from sources, please do so and let us know how it goes.

If you've never done this, the following should work (assuming the
source tree is /usr/src).
# cd /usr/src
# patch -p0 < freefh.patch
# make buildkernel
# make installkernel
- reboot

rick

>
> Thanks for the packet trace and help tracking this down, rick
>
> >
> > Cheers,
> >
> > Richard

--- sys/fs/nfs/nfs_var.h.freefh	2026-07-08 13:44:13.102029000 -0700
+++ sys/fs/nfs/nfs_var.h	2026-07-08 13:59:13.162426000 -0700
@@ -169,6 +169,7 @@ void nfsrv_removedeleg(fhandle_t *, struct nfsrv_descr
     struct nfsdevice **);
 void nfsrv_marknospc(char *, bool);
 void nfsrv_removedeleg(fhandle_t *, struct nfsrv_descript *, NFSPROC_T *);
+void nfsrv_freestrandedstate(struct nfsrvfh *);
 
 /* nfs_nfsdserv.c */
 int nfsrvd_access(struct nfsrv_descript *, int,
--- sys/fs/nfsserver/nfs_nfsdstate.c.freefh	2026-07-08 08:52:44.075026000 -0700
+++ sys/fs/nfsserver/nfs_nfsdstate.c	2026-07-08 14:15:45.938151000 -0700
@@ -253,6 +253,7 @@ static void nfsrv_clientunlock(bool mlocked);
     nfsv4stateid_t *delegstateidp);
 static void nfsrv_clientlock(bool mlocked);
 static void nfsrv_clientunlock(bool mlocked);
+static void nfsrv_freelockifnotinuse(struct nfslockfile *lfp);
 
 /*
  * Lock the client structure, either with the mutex or the exclusive nfsd lock.
@@ -8878,6 +8879,79 @@ nfsrv_removedeleg(fhandle_t *fhp, struct nfsrv_descrip
 	if (error == 0) {
 		LIST_FOREACH_SAFE(stp, &lfp->lf_deleg, ls_file, nstp)
 			nfsrv_freedeleg(stp);
+	}
+	NFSUNLOCKSTATE();
+}
+
+/*
+ * Free the nfslockfile structure if not in use.
+ */
+static void
+nfsrv_freelockifnotinuse(struct nfslockfile *lfp)
+{
+
+	mtx_assert(NFSSTATEMUTEXPTR, MA_OWNED);
+	/*
+	 * The nfslockfile is freed here if there are no locks
+	 * associated with the open.
+	 * If there are locks associated with the open, the
+	 * nfslockfile structure can be freed via nfsrv_freelockowner().
+	 */
+	if (lfp != NULL && LIST_EMPTY(&lfp->lf_open) &&
+	    LIST_EMPTY(&lfp->lf_deleg) && LIST_EMPTY(&lfp->lf_lock) &&
+	    LIST_EMPTY(&lfp->lf_locallock) && LIST_EMPTY(&lfp->lf_rollback) &&
+	    lfp->lf_usecount == 0 &&
+	    nfsv4_testlock(&lfp->lf_locallock_lck) == 0)
+		nfsrv_freenfslockfile(lfp);
+}
+
+/*
+ * Free stranded open/lock/delegation/layouts.
+ * (These become stranded if the file has been deleted.)
+ */
+void
+nfsrv_freestrandedstate(struct nfsrvfh *nfp)
+{
+	struct nfslockfile *lfp;
+	struct nfsstate *stp, *nstp;
+	struct nfslayouthash *lhyp;
+	struct nfslayout *lyp, *nlyp;
+	fhandle_t *fhp;
+
+	if (nfp->nfsrvfh_len != NFSX_MYFH)
+		return;
+	fhp = (fhandle_t *)nfp->nfsrvfh_data;
+	NFSLOCKSTATE();
+	if (nfsrv_getlockfile(0, NULL, &lfp, fhp, 0) < 0) {
+		NFSUNLOCKSTATE();
+		return;
 	}
+	lfp->lf_usecount++;	/* So nfsrv_freeopen() does not free it. */
+	/* Note that nfsrv_freeopen() will also free the byte range locks. */
+	LIST_FOREACH_SAFE(stp, &lfp->lf_open, ls_file, nstp)
+		nfsrv_freeopen(stp, NULL, 0, curthread);
+
+	/*
+	 * Normally, a delegation will have been recalled when the file is
+	 * removed.  However, get rid of any that have somehow been
+	 * left stranded.
+	 */
+	LIST_FOREACH_SAFE(stp, &lfp->lf_deleg, ls_file, nstp)
+		nfsrv_freedeleg(stp);
+
+	/* Get rid of the nfslockfile, if no longer in use. */
+	lfp->lf_usecount--;
+	nfsrv_freelockifnotinuse(lfp);
 	NFSUNLOCKSTATE();
+
+	/* Free any layouts for the pNFS server case. */
+	if (nfsrv_devidcnt == 0)
+		return;
+	lhyp = NFSLAYOUTHASH(fhp);
+	NFSLOCKLAYOUT(lhyp);
+	TAILQ_FOREACH_SAFE(lyp, &lhyp->list, lay_list, nlyp) {
+		if (NFSBCMP(&lyp->lay_fh, fhp, sizeof(*fhp)) == 0)
+			nfsrv_freelayout(&lhyp->list, lyp);
+	}
+	NFSUNLOCKLAYOUT(lhyp);
 }
--- sys/fs/nfsserver/nfs_nfsdsocket.c.freefh	2026-07-08 13:45:25.897531000 -0700
+++ sys/fs/nfsserver/nfs_nfsdsocket.c	2026-07-08 13:58:42.960529000 -0700
@@ -1031,7 +1031,8 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram
 				cur_fsid = vp->v_mount->mnt_stat.f_fsid;
 				NFSVOPUNLOCK(vp);
 				vpnes = nes;
-			}
+			} else if (nd->nd_repstat == ESTALE)
+				nfsrv_freestrandedstate(&fh);
 			break;
 		case NFSV4OP_PUTPUBFH:
 			if (nfs_pubfhset) {