[PATCH rdma-next v5 1/2] RDMA/mana_ib: unify QP lookup table

From: Konstantin Taranov <[email protected]>

Add helpers to retrieve the send and receive queues of a QP.
Use these helpers when storing queue IDs in the lookup table.
MANA queue IDs are 2-bit aligned, allowing the two least
significant bits to be omitted when storing and looking up
queue IDs.

Signed-off-by: Konstantin Taranov <[email protected]>
---
v4->v5: inline store functions inside mana_table_store_qp() to explicitly
see the call of drain at the same level.
 drivers/infiniband/hw/mana/mana_ib.h | 30 +++++++++
 drivers/infiniband/hw/mana/qp.c      | 97 ++++++++++++----------------
 2 files changed, 71 insertions(+), 56 deletions(-)

diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index da05966af..18688072f 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -26,6 +26,8 @@
 
 /* Send queue ID mask */
 #define MANA_SENDQ_MASK	BIT(31)
+/* Queue ID encodes type in the lower 2 bits */
+#define MANA_QID_SUBTYPE_MASK 0x3
 
 /*
  * The hardware limit of number of MRs is greater than maximum number of MRs
@@ -582,12 +584,40 @@ static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
 	return mdev->gdma_dev->gdma_context;
 }
 
+static inline struct mana_ib_queue *mana_qp_get_sq(struct mana_ib_qp *qp)
+{
+	switch (qp->ibqp.qp_type) {
+	case IB_QPT_RC:
+		return &qp->rc_qp.queues[MANA_RC_SEND_QUEUE_REQUESTER];
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return &qp->ud_qp.queues[MANA_UD_SEND_QUEUE];
+	default:
+		return NULL;
+	}
+}
+
+static inline struct mana_ib_queue *mana_qp_get_rq(struct mana_ib_qp *qp)
+{
+	switch (qp->ibqp.qp_type) {
+	case IB_QPT_RC:
+		return &qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER];
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return &qp->ud_qp.queues[MANA_UD_RECV_QUEUE];
+	default:
+		return NULL;
+	}
+}
+
 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev,
 						 u32 qid, bool is_sq)
 {
 	struct mana_ib_qp *qp;
 	unsigned long flag;
 
+	/* Remove subtype bits */
+	qid &= ~MANA_QID_SUBTYPE_MASK;
 	if (is_sq)
 		qid |= MANA_SENDQ_MASK;
 
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index c52f3ec14..47e1fadbd 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -461,89 +461,74 @@ static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32
 	return type;
 }
 
-static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+static void mana_table_drain_qp_ref(struct mana_ib_qp *qp)
 {
-	return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp,
-			     GFP_KERNEL);
+	mana_put_qp_ref(qp);
+	wait_for_completion(&qp->free);
 }
 
-static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+static int mana_table_store_qp_qid(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
+				   u32 qid, bool is_sq)
 {
-	xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num);
+	/* Remove subtype bits */
+	qid &= ~MANA_QID_SUBTYPE_MASK;
+	if (is_sq)
+		qid |= MANA_SENDQ_MASK;
+
+	return xa_insert_irq(&mdev->qp_table_wq, qid, qp, GFP_KERNEL);
 }
 
-static void mana_table_drain_qp_ref(struct mana_ib_qp *qp)
+static void mana_table_remove_qp_qid(struct mana_ib_dev *mdev, u32 qid, bool is_sq)
 {
-	mana_put_qp_ref(qp);
-	wait_for_completion(&qp->free);
+	/* Remove subtype bits */
+	qid &= ~MANA_QID_SUBTYPE_MASK;
+	if (is_sq)
+		qid |= MANA_SENDQ_MASK;
+
+	xa_erase_irq(&mdev->qp_table_wq, qid);
 }
 
-static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
 {
-	u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;
-	u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
+	struct mana_ib_queue *sq, *rq;
 	int err;
 
-	err = xa_insert_irq(&mdev->qp_table_wq, qids, qp, GFP_KERNEL);
+	refcount_set(&qp->refcount, 1);
+	init_completion(&qp->free);
+	sq = mana_qp_get_sq(qp);
+	rq = mana_qp_get_rq(qp);
+
+	if (!sq || !rq)
+		return -EINVAL;
+
+	err = mana_table_store_qp_qid(mdev, qp, sq->id, true);
 	if (err)
 		return err;
 
-	err = xa_insert_irq(&mdev->qp_table_wq, qidr, qp, GFP_KERNEL);
+	err = mana_table_store_qp_qid(mdev, qp, rq->id, false);
 	if (err)
-		goto remove_sq;
+		goto err_remove_sq;
 
 	return 0;
 
-remove_sq:
-	xa_erase_irq(&mdev->qp_table_wq, qids);
+err_remove_sq:
+	mana_table_remove_qp_qid(mdev, sq->id, true);
 	mana_table_drain_qp_ref(qp);
 	return err;
 }
 
-static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
-{
-	u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;
-	u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
-
-	xa_erase_irq(&mdev->qp_table_wq, qids);
-	xa_erase_irq(&mdev->qp_table_wq, qidr);
-}
-
-static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+static void mana_table_remove_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
 {
-	refcount_set(&qp->refcount, 1);
-	init_completion(&qp->free);
-
-	switch (qp->ibqp.qp_type) {
-	case IB_QPT_RC:
-		return mana_table_store_rc_qp(mdev, qp);
-	case IB_QPT_UD:
-	case IB_QPT_GSI:
-		return mana_table_store_ud_qp(mdev, qp);
-	default:
-		ibdev_dbg(&mdev->ib_dev, "Unknown QP type for storing in mana table, %d\n",
-			  qp->ibqp.qp_type);
-	}
+	struct mana_ib_queue *sq, *rq;
 
-	return -EINVAL;
-}
+	sq = mana_qp_get_sq(qp);
+	rq = mana_qp_get_rq(qp);
 
-static void mana_table_remove_qp(struct mana_ib_dev *mdev,
-				 struct mana_ib_qp *qp)
-{
-	switch (qp->ibqp.qp_type) {
-	case IB_QPT_RC:
-		mana_table_remove_rc_qp(mdev, qp);
-		break;
-	case IB_QPT_UD:
-	case IB_QPT_GSI:
-		mana_table_remove_ud_qp(mdev, qp);
-		break;
-	default:
-		ibdev_dbg(&mdev->ib_dev, "Unknown QP type for removing from mana table, %d\n",
-			  qp->ibqp.qp_type);
+	if (!sq || !rq)
 		return;
-	}
+
+	mana_table_remove_qp_qid(mdev, sq->id, true);
+	mana_table_remove_qp_qid(mdev, rq->id, false);
 	mana_table_drain_qp_ref(qp);
 }
 
-- 
2.43.0