[RFC PATCH v4 22/38] NTB: ntb_transport: Split core library and default NTB client

The transport code is being refactored to support multiple clients and
transport backends. As a first step, split the module into a core
library and a thin default client that binds to NTB ports.

Move module parameters and the legacy ntb_client glue into a new
ntb_transport.c. Export ntb_transport_attach()/ntb_transport_detach()
from the core so other clients can reuse the common transport
infrastructure.

No functional change intended for the default transport.

Signed-off-by: Koichiro Den <[email protected]>
---
 drivers/ntb/Makefile                 |  3 +-
 drivers/ntb/ntb_transport.c          | 81 ++++++++++++++++++++++++
 drivers/ntb/ntb_transport_core.c     | 93 ++++++++++------------------
 drivers/ntb/ntb_transport_internal.h | 15 +++++
 4 files changed, 128 insertions(+), 64 deletions(-)
 create mode 100644 drivers/ntb/ntb_transport.c

diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
index 9b66e5fafbc0..47e6b95ef7ce 100644
--- a/drivers/ntb/Makefile
+++ b/drivers/ntb/Makefile
@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_NTB) += ntb.o hw/ test/
 obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
+obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport_core.o
 
 ntb-y			:= core.o
 ntb-$(CONFIG_NTB_MSI)	+= msi.o
-
-ntb_transport-y		:= ntb_transport_core.o
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
new file mode 100644
index 000000000000..dafb97e38883
--- /dev/null
+++ b/drivers/ntb/ntb_transport.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright(c) 2012 Intel Corporation. All rights reserved.
+ * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *
+ * Default NTB transport client module.
+ *
+ * The transport core library and backend infrastructure are implemented in
+ * ntb_transport_core.c. This module provides the default client that binds
+ * to NTB ports and instantiates the default transport for consumers such
+ * as ntb_netdev.
+ */
+
+#include <linux/module.h>
+#include <linux/ntb.h>
+
+#include "ntb_transport_internal.h"
+
+static unsigned long max_mw_size;
+module_param(max_mw_size, ulong, 0644);
+MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
+
+static unsigned int transport_mtu = 0x10000;
+module_param(transport_mtu, uint, 0644);
+MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");
+
+static unsigned char max_num_clients;
+module_param(max_num_clients, byte, 0644);
+MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");
+
+static unsigned int copy_bytes = 1024;
+module_param(copy_bytes, uint, 0644);
+MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");
+
+static bool use_dma;
+module_param(use_dma, bool, 0644);
+MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");
+
+static bool use_msi;
+#ifdef CONFIG_NTB_MSI
+module_param(use_msi, bool, 0644);
+MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
+#endif
+
+#define NTB_QP_DEF_NUM_ENTRIES	100
+
+static int ntb_transport_host_probe(struct ntb_client *self,
+				    struct ntb_dev *ndev)
+{
+	return ntb_transport_attach(ndev, "default", use_msi, max_mw_size,
+				    transport_mtu, max_num_clients, copy_bytes,
+				    use_dma, NTB_QP_DEF_NUM_ENTRIES);
+}
+
+static void ntb_transport_host_remove(struct ntb_client *self, struct ntb_dev *ndev)
+{
+	ntb_transport_detach(ndev);
+}
+
+static struct ntb_client ntb_transport_host_client = {
+	.ops = {
+		.probe = ntb_transport_host_probe,
+		.remove = ntb_transport_host_remove,
+	},
+};
+
+static int __init ntb_transport_host_init(void)
+{
+	return ntb_register_client(&ntb_transport_host_client);
+}
+module_init(ntb_transport_host_init);
+
+static void __exit ntb_transport_host_exit(void)
+{
+	ntb_unregister_client(&ntb_transport_host_client);
+}
+module_exit(ntb_transport_host_exit);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Intel Corporation");
+MODULE_DESCRIPTION("Software Queue-Pair Transport over NTB");
diff --git a/drivers/ntb/ntb_transport_core.c b/drivers/ntb/ntb_transport_core.c
index 04a13fdce71c..86181fe1eadd 100644
--- a/drivers/ntb/ntb_transport_core.c
+++ b/drivers/ntb/ntb_transport_core.c
@@ -69,7 +69,7 @@
 #define NTB_TRANSPORT_VERSION	4
 #define NTB_TRANSPORT_VER	"4"
 #define NTB_TRANSPORT_NAME	"ntb_transport"
-#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"
+#define NTB_TRANSPORT_DESC	"NTB transport core library"
 #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
 
 MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
@@ -77,31 +77,6 @@ MODULE_VERSION(NTB_TRANSPORT_VER);
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel Corporation");
 
-static unsigned long max_mw_size;
-module_param(max_mw_size, ulong, 0644);
-MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
-
-static unsigned int transport_mtu = 0x10000;
-module_param(transport_mtu, uint, 0644);
-MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");
-
-static unsigned char max_num_clients;
-module_param(max_num_clients, byte, 0644);
-MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");
-
-static unsigned int copy_bytes = 1024;
-module_param(copy_bytes, uint, 0644);
-MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");
-
-static bool use_dma;
-module_param(use_dma, bool, 0644);
-MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");
-
-static bool use_msi;
-#ifdef CONFIG_NTB_MSI
-module_param(use_msi, bool, 0644);
-MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells");
-#endif
 
 static struct dentry *nt_debugfs_dir;
 
@@ -135,12 +110,10 @@ enum {
 #define drv_client(__drv) \
 	container_of((__drv), struct ntb_transport_client, driver)
 
-#define NTB_QP_DEF_NUM_ENTRIES	100
 #define NTB_LINK_DOWN_TIMEOUT	10
 
 static void ntb_transport_rxc_db(unsigned long data);
 static const struct ntb_ctx_ops ntb_transport_ops;
-static struct ntb_client ntb_transport_client;
 static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
 			       struct ntb_queue_entry *entry);
 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset);
@@ -456,8 +429,8 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
 
 	if (mw_size > mw->xlat_size)
 		mw_size = mw->xlat_size;
-	if (max_mw_size && mw_size > max_mw_size)
-		mw_size = max_mw_size;
+	if (nt->max_mw_size && mw_size > nt->max_mw_size)
+		mw_size = nt->max_mw_size;
 
 	tx_size = (unsigned int)mw_size / num_qps_mw;
 	qp_offset = tx_size * (qp_num / mw_count);
@@ -481,9 +454,9 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
 	qp->rx_info = qp->tx_mw + tx_size;
 
 	/* Due to housekeeping, there must be atleast 2 buffs */
-	qp->tx_max_frame = min(transport_mtu, tx_size / 2);
+	qp->tx_max_frame = min(nt->transport_mtu, tx_size / 2);
 	qp->tx_max_entry = tx_size / qp->tx_max_frame;
-	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
+	qp->rx_max_frame = min(nt->transport_mtu, rx_size / 2);
 	qp->rx_max_entry = rx_size / qp->rx_max_frame;
 	qp->rx_index = 0;
 
@@ -909,8 +882,8 @@ static void ntb_transport_link_work(struct work_struct *work)
 	for (i = 0; i < nt->mw_count; i++) {
 		size = nt->mw_vec[i].phys_size;
 
-		if (max_mw_size && size > max_mw_size)
-			size = max_mw_size;
+		if (nt->max_mw_size && size > nt->max_mw_size)
+			size = nt->max_mw_size;
 
 		spad = MW0_SZ_HIGH + (i * 2);
 		ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size));
@@ -1084,7 +1057,12 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
 	return 0;
 }
 
-static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
+int ntb_transport_attach(struct ntb_dev *ndev, const char *backend_name,
+			 bool use_msi, unsigned long max_mw_size,
+			 unsigned int transport_mtu,
+			 unsigned char max_num_clients,
+			 unsigned int copy_bytes, bool use_dma,
+			 unsigned int num_rx_entries)
 {
 	struct ntb_transport_ctx *nt;
 	struct ntb_transport_mw *mw;
@@ -1117,6 +1095,11 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
 		return -ENOMEM;
 
 	nt->ndev = ndev;
+	nt->max_mw_size = max_mw_size;
+	nt->transport_mtu = transport_mtu;
+	nt->copy_bytes = copy_bytes;
+	nt->use_dma = use_dma;
+	nt->num_rx_entries = num_rx_entries;
 
 	/*
 	 * If we are using MSI, and have at least one extra memory window,
@@ -1241,8 +1224,9 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
 	kfree(nt);
 	return rc;
 }
+EXPORT_SYMBOL_GPL(ntb_transport_attach);
 
-static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev)
+void ntb_transport_detach(struct ntb_dev *ndev)
 {
 	struct ntb_transport_ctx *nt = ndev->ctx;
 	struct ntb_transport_qp *qp;
@@ -1262,6 +1246,7 @@ static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev)
 			ntb_transport_free_queue(qp);
 		debugfs_remove_recursive(qp->debugfs_dir);
 	}
+	debugfs_remove(nt->debugfs_node_dir);
 
 	ntb_link_disable(ndev);
 	ntb_clear_ctx(ndev);
@@ -1277,6 +1262,7 @@ static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev)
 	kfree(nt->mw_vec);
 	kfree(nt);
 }
+EXPORT_SYMBOL_GPL(ntb_transport_detach);
 
 static void ntb_complete_rxc(struct ntb_transport_qp *qp)
 {
@@ -1438,7 +1424,7 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 	if (!chan)
 		goto err;
 
-	if (entry->len < copy_bytes)
+	if (entry->len < qp->transport->copy_bytes)
 		goto err;
 
 	res = ntb_async_rx_submit(entry, offset);
@@ -1718,7 +1704,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
 	if (!chan)
 		goto err;
 
-	if (entry->len < copy_bytes)
+	if (entry->len < qp->transport->copy_bytes)
 		goto err;
 
 	res = ntb_async_tx_submit(qp, entry);
@@ -1856,7 +1842,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
 	dma_cap_zero(dma_mask);
 	dma_cap_set(DMA_MEMCPY, dma_mask);
 
-	if (use_dma) {
+	if (nt->use_dma) {
 		qp->tx_dma_chan =
 			dma_request_channel(dma_mask, ntb_dma_filter_fn,
 					    (void *)(unsigned long)node);
@@ -1892,7 +1878,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
 	dev_dbg(&pdev->dev, "Using %s memcpy for RX\n",
 		qp->rx_dma_chan ? "DMA" : "CPU");
 
-	for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
+	for (i = 0; i < nt->num_rx_entries; i++) {
 		entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
 		if (!entry)
 			goto err1;
@@ -1901,7 +1887,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev,
 		ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
 			     &qp->rx_free_q);
 	}
-	qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES;
+	qp->rx_alloc_entry = nt->num_rx_entries;
 
 	for (i = 0; i < qp->tx_max_entry; i++) {
 		entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
@@ -2301,13 +2287,6 @@ static const struct ntb_ctx_ops ntb_transport_ops = {
 	.db_event = ntb_transport_doorbell_callback,
 };
 
-static struct ntb_client ntb_transport_client = {
-	.ops = {
-		.probe = ntb_transport_probe,
-		.remove = ntb_transport_free,
-	},
-};
-
 static int __init ntb_transport_init(void)
 {
 	int rc;
@@ -2318,26 +2297,16 @@ static int __init ntb_transport_init(void)
 		nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 
 	rc = bus_register(&ntb_transport_bus);
-	if (rc)
-		goto err_bus;
-
-	rc = ntb_register_client(&ntb_transport_client);
-	if (rc)
-		goto err_client;
-
-	return 0;
-
-err_client:
-	bus_unregister(&ntb_transport_bus);
-err_bus:
-	debugfs_remove_recursive(nt_debugfs_dir);
+	if (rc) {
+		bus_unregister(&ntb_transport_bus);
+		debugfs_remove_recursive(nt_debugfs_dir);
+	}
 	return rc;
 }
 module_init(ntb_transport_init);
 
 static void __exit ntb_transport_exit(void)
 {
-	ntb_unregister_client(&ntb_transport_client);
 	bus_unregister(&ntb_transport_bus);
 	debugfs_remove_recursive(nt_debugfs_dir);
 }
diff --git a/drivers/ntb/ntb_transport_internal.h b/drivers/ntb/ntb_transport_internal.h
index 6b45790cc88e..406033dbddb7 100644
--- a/drivers/ntb/ntb_transport_internal.h
+++ b/drivers/ntb/ntb_transport_internal.h
@@ -134,9 +134,17 @@ struct ntb_transport_ctx {
 	struct ntb_transport_qp *qp_vec;
 	unsigned int mw_count;
 	unsigned int qp_count;
+	unsigned int max_qp_count;
 	u64 qp_bitmap;
 	u64 qp_bitmap_free;
 
+	/* Parameters */
+	unsigned int num_rx_entries;
+	unsigned int transport_mtu;
+	unsigned long max_mw_size;
+	unsigned int copy_bytes;
+	bool use_dma;
+
 	bool use_msi;
 	unsigned int msi_spad_offset;
 	u64 msi_db_mask;
@@ -162,5 +170,12 @@ struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, struct list_head *list);
 struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, struct list_head *list,
 				    struct list_head *to_list);
 void ntb_qp_link_down(struct ntb_transport_qp *qp);
+int ntb_transport_attach(struct ntb_dev *ndev, const char *backend_name,
+			 bool use_msi, unsigned long max_mw_size,
+			 unsigned int transport_mtu,
+			 unsigned char max_num_clients,
+			 unsigned int copy_bytes, bool use_dma,
+			 unsigned int num_rx_entries);
+void ntb_transport_detach(struct ntb_dev *ndev);
 
 #endif /* _NTB_TRANSPORT_INTERNAL_H_ */
-- 
2.51.0