[RFC PATCH v3 09/11] xfs: Implement RWF_NOSERIAL to parallelize RWF_WRITETHROUGH writes

In xfs, buffered writethrough writes take an exclusive inode lock
similar to regular buffered write path. However, since writethrough also
submits the write under the inode lock, the increased critical section
really hurts performance when we have parallel writers writing to the
same file.

To mitigate this, implement RWF_NOSERIAL flag which allows us to perform
the write under a shared inode lock, instead of an exclusive lock. This
gives significant performance boost to single file, multiple writer
workloads at the cost of losing write-write serialization and read-write
serialization guarantees which XFS has historically provided.

Let's look at each of the guarantees and what changes with the NOSERIAL
flag:

Write-write guarantee:

Image 2 writers trying to write the same 3 folios. One is write As to
all 3 folios (denoted by AAA) and the other is writing BBB. Then under
exclusive lock the final state of the 3 folios could only be either AAA
or BBB.

However, with NOSERIAL writes, we could end up with mixed data in the
3 folios, like AAB, ABA, BAA etc. Note that this mixing will always
happen at the inter folio level, data contained within the same folio
will not be mixed as it is protected by the folio lock.

Read-write guarantee:

In XFS, reads also take a shared lock, however they don't take a folio
lock when copying from the folio to the user buffer. Due to the shared
lock of read and exclusive lock of write, we are able to guarantee that
the read always reads either completely old data or completely new data.

However, with the NOSERIAL flag, writethrough will take a shared lock for
writes, ie a read can race with a write which is still in middle of
copying data to the folio, hence the read can read a mix of old and new
data.

Despite the above changes in behavior, there might be applications
who would be okay to lose the guarantees because of the nature of their
workloads for example, if they never have multiple readers/writes doing
IO to the same range in the file. Such applications would benefit
significantly by using NOSERIAL writes.

Below are some fio performance numbers of RWF_WRITETHROUGH with and without
NOSERIAL writes.

** Multiple writes, single file (Pure overwrites, DSYNC) **
  Fio Workload:
    libaio, buffered randwrite, bs=4k, size=2G (pre written), O_DSYNC
    iodepth=32

  numjobs  baseline BW  RWF_NOSERIAL BW  Δ%
     1        350           392         +12.0%
     2        526           798         +51.7%
     4        597          1591        +166.5%
     8        630          1839        +191.9%
    16        570          1836        +222.1%

** Multiple writes, single file (Preallocated file, sync_file_range) **
  Fio Workload:
    libaio, buffered randwrite, bs=16k, size=5G (fallocated)
    sync_file_range=wait_before,write:16

  numjobs  baseline BW  RWF_NOSERIAL BW  Δ%
     1       1099          1279        +16.4%
     2       1482          2456        +65.7%
     4       2065          2479        +20.1%
     8       1778          2500        +40.6%
    16       1787          2508        +40.3%

* Multiple writes, single file (Truncated file, DSYNC) *
  Fio Workload:
    libaio, buffered randwrite, bs=4k, size=6.5G (truncated), O_DSYNC
    iodepth=32

  numjobs  baseline BW  RWF_NOSERIAL BW  Δ%
     1        78            80         +2.6%
     2        97            88         -9.3%
     4       100            96         -4.0%
     8       109            97        -11.0%
    16       106           107         +0.9%

Co-developed-by: Ritesh Harjani (IBM) <[email protected]>
Signed-off-by: Ritesh Harjani (IBM) <[email protected]>
Signed-off-by: Ojaswin Mujoo <[email protected]>
---
 fs/xfs/xfs_file.c  | 54 ++++++++++++++++++++++++++++++++++++++++------
 include/linux/fs.h |  7 ++++++
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index 4b45ceacd461..b79076c15c5f 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -520,6 +520,42 @@ xfs_file_write_checks(
 	return kiocb_modified(iocb);
 }
 
+STATIC ssize_t
+xfs_file_writethrough_checks(
+	struct kiocb		*iocb,
+	struct iov_iter		*from,
+	unsigned int		*iolock,
+	struct xfs_zone_alloc_ctx *ac)
+{
+	struct inode		*inode = iocb->ki_filp->f_mapping->host;
+	size_t			isize = i_size_read(inode);
+	size_t			count = iov_iter_count(from);
+	ssize_t			error;
+
+	error =  xfs_file_write_checks(iocb, from, iolock, ac);
+	if (error < 0)
+		return error;
+
+	if (*iolock == XFS_IOLOCK_EXCL)
+		return 0;
+
+	/*
+	 * Extending IO needs exclusive lock for i_size change
+	 */
+	if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize)
+		goto upgrade_excl;
+
+	return 0;
+
+upgrade_excl:
+	xfs_iunlock(XFS_I(inode), *iolock);
+	*iolock = XFS_IOLOCK_EXCL;
+	error = xfs_ilock_iocb(iocb, *iolock);
+	if (error)
+		*iolock = 0;
+	return error;
+}
+
 static ssize_t
 xfs_zoned_write_space_reserve(
 	struct xfs_mount		*mp,
@@ -1108,23 +1144,29 @@ xfs_file_buffered_write(
 	unsigned int		iolock;
 
 write_retry:
-	iolock = XFS_IOLOCK_EXCL;
+	if (iocb->ki_flags & IOCB_NOSERIAL)
+		iolock = XFS_IOLOCK_SHARED;
+	else
+		iolock = XFS_IOLOCK_EXCL;
 	ret = xfs_ilock_iocb(iocb, iolock);
 	if (ret)
 		return ret;
 
-	ret = xfs_file_write_checks(iocb, from, &iolock, NULL);
-	if (ret)
-		goto out;
-
 	trace_xfs_file_buffered_write(iocb, from);
 	if (iocb->ki_flags & IOCB_WRITETHROUGH) {
+		ret = xfs_file_writethrough_checks(iocb, from, &iolock, NULL);
+		if (ret)
+			goto out;
 		ret = iomap_file_writethrough_write(iocb, from,
 						    &xfs_writethrough_ops, NULL);
-	} else
+	} else {
+		ret = xfs_file_write_checks(iocb, from, &iolock, NULL);
+		if (ret)
+			goto out;
 		ret = iomap_file_buffered_write(iocb, from,
 						&xfs_buffered_write_iomap_ops,
 						&xfs_iomap_write_ops, NULL);
+	}
 
 	/*
 	 * If we hit a space limit, try to free up some lingering preallocated
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 685ffe8da6ea..ed5144512835 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -3495,6 +3495,13 @@ static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags,
 		ki->ki_flags &= ~IOCB_APPEND;
 	}
 
+	/*
+	 * Writethrough implies non-serial writes ie writes can go
+	 * parallelly.
+	 */
+	if (flags & RWF_WRITETHROUGH)
+		kiocb_flags |= IOCB_NOSERIAL;
+
 	ki->ki_flags |= kiocb_flags;
 	return 0;
 }
-- 
2.55.0