[PATCH v8 04/10] nvme-multipath: pass I/O type to nvme_find_path()

Currently, nvme_find_path() only accepts an nvme_ns_head argument.
However, the upcoming latency-aware I/O policy also needs to know
the I/O type (read/write/other) associated with the request in order
to make path selection decisions.

Update nvme_find_path() to accept an additional argument describing
the I/O type. Classify requests into three categories: READ, WRITE,
and OTHER. Admin commands and I/O requests that are neither reads nor
writes are classified as OTHER.

This patch does not introduce any functional change and only prepares
the interface for subsequent latency-policy changes.

Reviewed-by: Guixin Liu <[email protected]>
Reviewed-by: Hannes Reinecke <[email protected]>
Signed-off-by: Nilay Shroff <[email protected]>
---
 drivers/nvme/host/ioctl.c     | 44 +++++++++++++++++++++++++++---
 drivers/nvme/host/multipath.c |  9 ++++---
 drivers/nvme/host/nvme.h      | 50 ++++++++++++++++++++++++++++++++++-
 drivers/nvme/host/pr.c        |  6 +++--
 drivers/nvme/host/sysfs.c     |  2 +-
 5 files changed, 100 insertions(+), 11 deletions(-)

diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 6539d4750098..b0538ed2adbc 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -751,12 +751,25 @@ int nvme_ns_head_ioctl(struct block_device *bdev, blk_mode_t mode,
 	struct nvme_ns *ns;
 	int srcu_idx, ret = -EWOULDBLOCK;
 	unsigned int flags = 0;
+	unsigned int op_type = NVME_STAT_GROUP_OTHER;
 
 	if (bdev_is_partition(bdev))
 		flags |= NVME_IOCTL_PARTITION;
 
+	if (cmd == NVME_IOCTL_SUBMIT_IO) {
+		u8 opcode;
+
+		if (get_user(opcode, (u8 *)argp))
+			return -EFAULT;
+
+		if (opcode == nvme_cmd_write)
+			op_type = NVME_STAT_GROUP_WRITE;
+		else if (opcode == nvme_cmd_read)
+			op_type = NVME_STAT_GROUP_READ;
+	}
+
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, op_type);
 	if (!ns)
 		goto out_unlock;
 
@@ -785,9 +798,22 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
 	void __user *argp = (void __user *)arg;
 	struct nvme_ns *ns;
 	int srcu_idx, ret = -EWOULDBLOCK;
+	unsigned int op_type = NVME_STAT_GROUP_OTHER;
+
+	if (cmd == NVME_IOCTL_SUBMIT_IO) {
+		u8 opcode;
+
+		if (get_user(opcode, (u8 *)argp))
+			return -EFAULT;
+
+		if (opcode == nvme_cmd_write)
+			op_type = NVME_STAT_GROUP_WRITE;
+		else if (opcode == nvme_cmd_read)
+			op_type = NVME_STAT_GROUP_READ;
+	}
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, op_type);
 	if (!ns)
 		goto out_unlock;
 
@@ -804,12 +830,24 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
 		unsigned int issue_flags)
 {
+	struct nvme_ns *ns;
+	unsigned int op_type;
 	struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
 	struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
 	int srcu_idx = srcu_read_lock(&head->srcu);
-	struct nvme_ns *ns = nvme_find_path(head);
 	int ret = -EINVAL;
+	const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
+						struct nvme_uring_cmd);
+	__u8 opcode = READ_ONCE(cmd->opcode);
+
+	if (opcode == nvme_cmd_write)
+		op_type = NVME_STAT_GROUP_WRITE;
+	else if (opcode == nvme_cmd_read)
+		op_type = NVME_STAT_GROUP_READ;
+	else
+		op_type = NVME_STAT_GROUP_OTHER;
 
+	ns = nvme_find_path(head, op_type);
 	if (ns)
 		ret = nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
 	srcu_read_unlock(&head->srcu, srcu_idx);
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 75dbb58286a3..95768eaef843 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -484,7 +484,8 @@ static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
 	return ns;
 }
 
-inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
+inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head,
+		enum nvme_stat_group op_type)
 {
 	switch (READ_ONCE(head->subsys->iopolicy)) {
 	case NVME_IOPOLICY_QD:
@@ -547,7 +548,7 @@ static void nvme_ns_head_submit_bio(struct bio *bio)
 		return;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, __nvme_get_stat_group(bio_op(bio)));
 	if (likely(ns)) {
 		bio_set_dev(bio, ns->disk->part0);
 		/*
@@ -597,7 +598,7 @@ static int nvme_ns_head_get_unique_id(struct gendisk *disk, u8 id[16],
 	int srcu_idx, ret = -EWOULDBLOCK;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, NVME_STAT_GROUP_OTHER);
 	if (ns)
 		ret = nvme_ns_get_unique_id(ns, id, type);
 	srcu_read_unlock(&head->srcu, srcu_idx);
@@ -613,7 +614,7 @@ static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector,
 	int srcu_idx, ret = -EWOULDBLOCK;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, NVME_STAT_GROUP_OTHER);
 	if (ns)
 		ret = nvme_ns_report_zones(ns, sector, nr_zones, args);
 	srcu_read_unlock(&head->srcu, srcu_idx);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 75e5d5a8a77c..5af272cf2f49 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -531,6 +531,20 @@ struct nvme_ns_ids {
 	u8	csi;
 };
 
+/*
+ * Enum used to classify NVMe I/O type into a stat group. Read and write
+ * I/Os are classified as NVME_STAT_GROUP_READ and NVME_STAT_GROUP_WRITE
+ * respectively; all other I/Os or admin commands are classified as
+ * NVME_STAT_GROUP_OTHER.
+ */
+enum nvme_stat_group {
+	NVME_STAT_GROUP_READ,
+	NVME_STAT_GROUP_WRITE,
+	NVME_STAT_GROUP_OTHER,
+
+	NVME_NUM_STAT_GROUPS
+};
+
 /*
  * Anchor structure for namespaces.  There is one for each namespace in a
  * NVMe subsystem that any of our controllers can see, and the namespace
@@ -1046,8 +1060,42 @@ extern const struct attribute_group *nvme_dev_attr_groups[];
 extern const struct block_device_operations nvme_bdev_ops;
 
 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
-struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
+struct nvme_ns *nvme_find_path(struct nvme_ns_head *head,
+		enum nvme_stat_group op_type)
 	__must_hold_shared(&head->srcu);
+
+static inline enum nvme_stat_group __nvme_get_stat_group(const enum req_op op)
+{
+	if (op == REQ_OP_READ)
+		return NVME_STAT_GROUP_READ;
+	else if (op == REQ_OP_WRITE)
+		return NVME_STAT_GROUP_WRITE;
+	else
+		return NVME_STAT_GROUP_OTHER;
+}
+
+static inline enum nvme_stat_group __nvme_get_passthru_stat_group(
+		enum nvme_opcode op)
+{
+	if (op == nvme_cmd_read)
+		return NVME_STAT_GROUP_READ;
+	else if (op == nvme_cmd_write)
+		return NVME_STAT_GROUP_WRITE;
+	else
+		return NVME_STAT_GROUP_OTHER;
+}
+
+static inline enum nvme_stat_group nvme_get_stat_group(struct request *req)
+{
+	if (blk_rq_is_passthrough(req)) {
+		struct nvme_request *nr = nvme_req(req);
+
+		return __nvme_get_passthru_stat_group(nr->cmd->common.opcode);
+	}
+
+	return __nvme_get_stat_group(req_op(req));
+}
+
 #ifdef CONFIG_NVME_MULTIPATH
 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
 {
diff --git a/drivers/nvme/host/pr.c b/drivers/nvme/host/pr.c
index fe7dbe264815..715e6c242bd1 100644
--- a/drivers/nvme/host/pr.c
+++ b/drivers/nvme/host/pr.c
@@ -53,10 +53,12 @@ static int nvme_send_ns_head_pr_command(struct block_device *bdev,
 		struct nvme_command *c, void *data, unsigned int data_len)
 {
 	struct nvme_ns_head *head = bdev->bd_disk->private_data;
-	int srcu_idx = srcu_read_lock(&head->srcu);
-	struct nvme_ns *ns = nvme_find_path(head);
+	int srcu_idx;
+	struct nvme_ns *ns;
 	int ret = -EWOULDBLOCK;
 
+	srcu_idx = srcu_read_lock(&head->srcu);
+	ns = nvme_find_path(head, NVME_STAT_GROUP_OTHER);
 	if (ns) {
 		c->common.nsid = cpu_to_le32(ns->head->ns_id);
 		ret = nvme_submit_sync_cmd(ns->queue, c, data, data_len);
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index abf8edaae371..e95543fecb2a 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -195,7 +195,7 @@ static int ns_head_update_nuse(struct nvme_ns_head *head)
 		return 0;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, NVME_STAT_GROUP_OTHER);
 	if (!ns)
 		goto out_unlock;
 
-- 
2.53.0