[PATCH] ceph: make nearfull sync writes opt-in

The kernel CephFS client has historically treated a cluster or pool
NEARFULL condition as a request to force successful writes through
generic_write_sync().  That effectively turns otherwise buffered writes
into synchronous writes and can cause a severe throughput drop as soon
as a single OSD or the file data pool crosses the nearfull threshold.

On modern large clusters, NEARFULL is primarily an operator health
signal rather than an immediate client-side capacity failure.  Operators
can still have substantial usable capacity while a cluster is
rebalancing, splitting PGs, or expanding onto new devices.  RBD, RGW and
the userspace CephFS client do not impose this extra client-side
sync-write throttle, so the kernel client behavior is surprising and
operationally painful.

Change the default behavior so NEARFULL no longer changes normal
write-sync semantics.  FULL and pool FULL still fail with -ENOSPC, and
explicitly synchronous writes continue to be synced by
generic_write_sync().

Add a nearfull_sync mount option for deployments that want the legacy
backpressure behavior.  When this option is set, successful writes are
promoted to IOCB_DSYNC if the cluster or file data pool is marked
NEARFULL, preserving the old behavior for conservative deployments.

Link: https://tracker.ceph.com/issues/74849
Signed-off-by: Alex Markuze <[email protected]>
---
 Documentation/filesystems/ceph.rst |  6 ++++++
 fs/ceph/file.c                     |  8 +++++---
 fs/ceph/super.c                    | 10 ++++++++++
 fs/ceph/super.h                    |  1 +
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/Documentation/filesystems/ceph.rst b/Documentation/filesystems/ceph.rst
index 6d2276a87a5a..ee2ca0c0c654 100644
--- a/Documentation/filesystems/ceph.rst
+++ b/Documentation/filesystems/ceph.rst
@@ -194,6 +194,12 @@ Mount Options
         copies.  Currently, it's only used in copy_file_range, which will revert
         to the default VFS implementation if this option is used.
 
+  nearfull_sync
+	Force written data to stable storage when the cluster or file data pool is
+	marked NEARFULL. This restores the legacy client-side backpressure
+	behavior. By default, CephFS writes are not forced synchronous solely
+	because of NEARFULL.
+
   recover_session=<no|clean>
 	Set auto reconnect mode in the case where the client is blocklisted. The
 	available modes are "no" and "clean". The default is "no".
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index 3823671ab95a..0ae9f09aa233 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -2400,7 +2400,8 @@ static ssize_t ceph_splice_read(struct file *in, loff_t *ppos,
  * dropping our cap refs and allowing the pending snap to logically
  * complete _before_ this write occurs.
  *
- * If we are near ENOSPC, write synchronously.
+ * If requested, nearfull writes are synced to preserve the legacy
+ * client-side backpressure behavior.
  */
 static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
@@ -2568,8 +2569,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 
 	if (written >= 0) {
-		if ((map_flags & CEPH_OSDMAP_NEARFULL) ||
-		    (pool_flags & CEPH_POOL_FLAG_NEARFULL))
+		if (ceph_test_mount_opt(fsc, NEARFULL_SYNC) &&
+		    ((map_flags & CEPH_OSDMAP_NEARFULL) ||
+		     (pool_flags & CEPH_POOL_FLAG_NEARFULL)))
 			iocb->ki_flags |= IOCB_DSYNC;
 		written = generic_write_sync(iocb, written);
 	}
diff --git a/fs/ceph/super.c b/fs/ceph/super.c
index 0bbd38933f0e..4d129483e345 100644
--- a/fs/ceph/super.c
+++ b/fs/ceph/super.c
@@ -178,6 +178,7 @@ enum {
 	Opt_pagecache,
 	Opt_sparseread,
 	Opt_lazyio,
+	Opt_nearfull_sync,
 };
 
 enum ceph_recover_session_mode {
@@ -207,6 +208,7 @@ static const struct fs_parameter_spec ceph_mount_parameters[] = {
 	fsparam_flag_no	("lazyio",			Opt_lazyio),
 	fsparam_string	("mds_namespace",		Opt_mds_namespace),
 	fsparam_string	("mon_addr",			Opt_mon_addr),
+	fsparam_flag_no	("nearfull_sync",		Opt_nearfull_sync),
 	fsparam_flag_no ("poolperm",			Opt_poolperm),
 	fsparam_flag_no ("quotadf",			Opt_quotadf),
 	fsparam_u32	("rasize",			Opt_rasize),
@@ -601,6 +603,12 @@ static int ceph_parse_mount_param(struct fs_context *fc,
 		else
 			fsopt->flags |= CEPH_MOUNT_OPT_LAZYIO;
 		break;
+	case Opt_nearfull_sync:
+		if (result.negated)
+			fsopt->flags &= ~CEPH_MOUNT_OPT_NEARFULL_SYNC;
+		else
+			fsopt->flags |= CEPH_MOUNT_OPT_NEARFULL_SYNC;
+		break;
 	case Opt_test_dummy_encryption:
 #ifdef CONFIG_FS_ENCRYPTION
 		fscrypt_free_dummy_policy(&fsopt->dummy_enc_policy);
@@ -759,6 +767,8 @@ static int ceph_show_options(struct seq_file *m, struct dentry *root)
 		seq_puts(m, ",sparseread");
 	if (fsopt->flags & CEPH_MOUNT_OPT_LAZYIO)
 		seq_puts(m, ",lazyio");
+	if (fsopt->flags & CEPH_MOUNT_OPT_NEARFULL_SYNC)
+		seq_puts(m, ",nearfull_sync");
 
 	fscrypt_show_test_dummy_encryption(m, ',', root->d_sb);
 
diff --git a/fs/ceph/super.h b/fs/ceph/super.h
index 3737ea7ed88b..632ffb1b401e 100644
--- a/fs/ceph/super.h
+++ b/fs/ceph/super.h
@@ -46,6 +46,7 @@
 #define CEPH_MOUNT_OPT_NOPAGECACHE     (1<<16) /* bypass pagecache altogether */
 #define CEPH_MOUNT_OPT_SPARSEREAD      (1<<17) /* always do sparse reads */
 #define CEPH_MOUNT_OPT_LAZYIO          (1<<18) /* force lazyio for all file opens */
+#define CEPH_MOUNT_OPT_NEARFULL_SYNC   (1<<19) /* sync writes when nearfull */
 
 #define CEPH_MOUNT_OPT_DEFAULT			\
 	(CEPH_MOUNT_OPT_DCACHE |		\
-- 
2.34.1