[PATCH v4 for-next 17/24] RDMA/hfi2: Add verbs core
Dennis Dalessandro <[email protected]>
| Newsgroups | org.kernel.vger.linux-rdma |
|---|---|
| Message-ID | <178612543217.3164235.3432580988258685182.stgit@awdrv-04> |
Add the core verbs implementation including queue pair management, reliable/unreliable connected and datagram send paths, and the OPFN protocol extension. Co-developed-by: Dean Luick <[email protected]> Signed-off-by: Dean Luick <[email protected]> Co-developed-by: Bendan Cunningham <[email protected]> Signed-off-by: Breandan Cunningham <[email protected]> Co-developed-by: Douglas Miller <[email protected]> Signed-off-by: Douglas Miller <[email protected]> Assisted-by: Claude:claude-sonnet-4-5 Signed-off-by: Dennis Dalessandro <[email protected]> --- Changes since v2: - uc.c: fix RDMA_WRITE_LAST tlen guard to use extra_bytes instead of pad+4, correctly handling 16B headers. - verbs.c: fix double-free in hfi2_verbs_send_dma() by setting ps->s_txreq=NULL after verbs_sdma_complete(); fix txreq leak on pkey check failure in SDMA path; use timer_shutdown_sync for mem_timer; use spin_lock_irqsave in verbs_sdma_complete() since it is called from interrupt context; fix AHG branch in build_verbs_tx_desc() to include extra_bytes in tlen passed to sdma_txinit_ahg(). - qp.c: add INIT_WORK for opfn_work in hfi2_qp_priv_alloc(). Changes since v1: - Replace cport_get_dev_fw_str accessor macros with inline functions (Arnd Bergmann). --- drivers/infiniband/hw/hfi2/opfn.c | 324 +++++ drivers/infiniband/hw/hfi2/qp.c | 1054 ++++++++++++++++ drivers/infiniband/hw/hfi2/ruc.c | 622 +++++++++ drivers/infiniband/hw/hfi2/uc.c | 536 ++++++++ drivers/infiniband/hw/hfi2/ud.c | 1037 +++++++++++++++ drivers/infiniband/hw/hfi2/verbs.c | 2038 ++++++++++++++++++++++++++++++ drivers/infiniband/hw/hfi2/verbs.h | 6 drivers/infiniband/hw/hfi2/verbs_txreq.c | 101 + 8 files changed, 5712 insertions(+), 6 deletions(-) create mode 100644 drivers/infiniband/hw/hfi2/opfn.c create mode 100644 drivers/infiniband/hw/hfi2/qp.c create mode 100644 drivers/infiniband/hw/hfi2/ruc.c create mode 100644 drivers/infiniband/hw/hfi2/uc.c create mode 100644 drivers/infiniband/hw/hfi2/ud.c create mode 100644 drivers/infiniband/hw/hfi2/verbs.c create mode 100644 drivers/infiniband/hw/hfi2/verbs_txreq.c diff --git a/drivers/infiniband/hw/hfi2/opfn.c b/drivers/infiniband/hw/hfi2/opfn.c new file mode 100644 index 000000000000..f24d28e404b8 --- /dev/null +++ b/drivers/infiniband/hw/hfi2/opfn.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright(c) 2018 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + * + */ +#include "hfi2.h" +#include "trace.h" +#include "qp.h" +#include "opfn.h" + +#define IB_BTHE_E BIT(IB_BTHE_E_SHIFT) + +#define OPFN_CODE(code) BIT((code) - 1) +#define OPFN_MASK(code) OPFN_CODE(STL_VERBS_EXTD_##code) + +struct hfi2_opfn_type { + bool (*request)(struct rvt_qp *qp, u64 *data); + bool (*response)(struct rvt_qp *qp, u64 *data); + bool (*reply)(struct rvt_qp *qp, u64 data); + void (*error)(struct rvt_qp *qp); +}; + +static struct hfi2_opfn_type hfi2_opfn_handlers[STL_VERBS_EXTD_MAX] = { + [STL_VERBS_EXTD_TID_RDMA] = { + .request = hfi2_tid_rdma_conn_req, + .response = hfi2_tid_rdma_conn_resp, + .reply = hfi2_tid_rdma_conn_reply, + .error = hfi2_tid_rdma_conn_error, + }, +}; + +static struct workqueue_struct *opfn_wq; + +static void opfn_schedule_conn_request(struct rvt_qp *qp); + +static bool hfi2_opfn_extended(u32 bth1) +{ + return !!(bth1 & IB_BTHE_E); +} + +static void opfn_conn_request(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct ib_atomic_wr wr; + u16 mask, capcode; + struct hfi2_opfn_type *extd; + u64 data; + unsigned long flags; + int ret = 0; + + trace_hfi2_opfn_state_conn_request(qp); + spin_lock_irqsave(&priv->opfn.lock, flags); + /* + * Exit if the extended bit is not set, or if nothing is requested, or + * if we have completed all requests, or if a previous request is in + * progress + */ + if (!priv->opfn.extended || !priv->opfn.requested || + priv->opfn.requested == priv->opfn.completed || priv->opfn.curr) + goto done; + + mask = priv->opfn.requested & ~priv->opfn.completed; + capcode = ilog2(mask & ~(mask - 1)) + 1; + if (capcode >= STL_VERBS_EXTD_MAX) { + priv->opfn.completed |= OPFN_CODE(capcode); + goto done; + } + + extd = &hfi2_opfn_handlers[capcode]; + if (!extd || !extd->request || !extd->request(qp, &data)) { + /* + * Either there is no handler for this capability or the request + * packet could not be generated. Either way, mark it as done so + * we don't keep attempting to complete it. + */ + priv->opfn.completed |= OPFN_CODE(capcode); + goto done; + } + + trace_hfi2_opfn_data_conn_request(qp, capcode, data); + data = (data & ~0xf) | capcode; + + memset(&wr, 0, sizeof(wr)); + wr.wr.opcode = IB_WR_OPFN; + wr.remote_addr = HFI2_VERBS_E_ATOMIC_VADDR; + wr.compare_add = data; + + priv->opfn.curr = capcode; /* A new request is now in progress */ + /* Drop opfn.lock before calling ib_post_send() */ + spin_unlock_irqrestore(&priv->opfn.lock, flags); + + ret = ib_post_send(&qp->ibqp, &wr.wr, NULL); + if (ret) + goto err; + trace_hfi2_opfn_state_conn_request(qp); + return; +err: + trace_hfi2_msg_opfn_conn_request(qp, "ib_ost_send failed: ret = ", + (u64)ret); + spin_lock_irqsave(&priv->opfn.lock, flags); + /* + * In case of an unexpected error return from ib_post_send + * clear opfn.curr and reschedule to try again + */ + priv->opfn.curr = STL_VERBS_EXTD_NONE; + opfn_schedule_conn_request(qp); +done: + spin_unlock_irqrestore(&priv->opfn.lock, flags); +} + +void hfi2_opfn_send_conn_request(struct work_struct *work) +{ + struct hfi2_opfn_data *od; + struct hfi2_qp_priv *qpriv; + + od = container_of(work, struct hfi2_opfn_data, opfn_work); + qpriv = container_of(od, struct hfi2_qp_priv, opfn); + + opfn_conn_request(qpriv->owner); +} + +/* + * When QP s_lock is held in the caller, the OPFN request must be scheduled + * to a different workqueue to avoid double locking QP s_lock in call to + * ib_post_send in opfn_conn_request + */ +static void opfn_schedule_conn_request(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + + trace_hfi2_opfn_state_sched_conn_request(qp); + queue_work(opfn_wq, &priv->opfn.opfn_work); +} + +void hfi2_opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e, + struct ib_atomic_eth *ateth) +{ + struct hfi2_qp_priv *priv = qp->priv; + u64 data = be64_to_cpu(ateth->compare_data); + struct hfi2_opfn_type *extd; + u8 capcode; + unsigned long flags; + + trace_hfi2_opfn_state_conn_response(qp); + capcode = data & 0xf; + trace_hfi2_opfn_data_conn_response(qp, capcode, data); + if (!capcode || capcode >= STL_VERBS_EXTD_MAX) + return; + + extd = &hfi2_opfn_handlers[capcode]; + + if (!extd || !extd->response) { + e->atomic_data = capcode; + return; + } + + spin_lock_irqsave(&priv->opfn.lock, flags); + if (priv->opfn.completed & OPFN_CODE(capcode)) { + /* + * We are receiving a request for a feature that has already + * been negotiated. This may mean that the other side has reset + */ + priv->opfn.completed &= ~OPFN_CODE(capcode); + if (extd->error) + extd->error(qp); + } + + if (extd->response(qp, &data)) + priv->opfn.completed |= OPFN_CODE(capcode); + e->atomic_data = (data & ~0xf) | capcode; + trace_hfi2_opfn_state_conn_response(qp); + spin_unlock_irqrestore(&priv->opfn.lock, flags); +} + +void hfi2_opfn_conn_reply(struct rvt_qp *qp, u64 data) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_opfn_type *extd; + u8 capcode; + unsigned long flags; + + trace_hfi2_opfn_state_conn_reply(qp); + capcode = data & 0xf; + trace_hfi2_opfn_data_conn_reply(qp, capcode, data); + if (!capcode || capcode >= STL_VERBS_EXTD_MAX) + return; + + spin_lock_irqsave(&priv->opfn.lock, flags); + /* + * Either there is no previous request or the reply is not for the + * current request + */ + if (!priv->opfn.curr || capcode != priv->opfn.curr) + goto done; + + extd = &hfi2_opfn_handlers[capcode]; + + if (!extd || !extd->reply) + goto clear; + + if (extd->reply(qp, data)) + priv->opfn.completed |= OPFN_CODE(capcode); +clear: + /* + * Clear opfn.curr to indicate that the previous request is no longer in + * progress + */ + priv->opfn.curr = STL_VERBS_EXTD_NONE; + trace_hfi2_opfn_state_conn_reply(qp); +done: + spin_unlock_irqrestore(&priv->opfn.lock, flags); +} + +void hfi2_opfn_conn_error(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_opfn_type *extd = NULL; + unsigned long flags; + u16 capcode; + + trace_hfi2_opfn_state_conn_error(qp); + trace_hfi2_msg_opfn_conn_error(qp, "error. qp state ", (u64)qp->state); + /* + * The QP has gone into the Error state. We have to invalidate all + * negotiated feature, including the one in progress (if any). The RC + * QP handling will clean the WQE for the connection request. + */ + spin_lock_irqsave(&priv->opfn.lock, flags); + while (priv->opfn.completed) { + capcode = priv->opfn.completed & ~(priv->opfn.completed - 1); + extd = &hfi2_opfn_handlers[ilog2(capcode) + 1]; + if (extd->error) + extd->error(qp); + priv->opfn.completed &= ~OPFN_CODE(capcode); + } + priv->opfn.extended = 0; + priv->opfn.requested = 0; + priv->opfn.curr = STL_VERBS_EXTD_NONE; + spin_unlock_irqrestore(&priv->opfn.lock, flags); +} + +void hfi2_opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask) +{ + struct ib_qp *ibqp = &qp->ibqp; + struct hfi2_qp_priv *priv = qp->priv; + unsigned long flags; + + if (attr_mask & IB_QP_RETRY_CNT) + priv->s_retry = attr->retry_cnt; + + spin_lock_irqsave(&priv->opfn.lock, flags); + if (ibqp->qp_type == IB_QPT_RC && HFI2_CAP_IS_KSET(TID_RDMA)) { + struct tid_rdma_params *local = &priv->tid_rdma.local; + + if (attr_mask & IB_QP_TIMEOUT) + priv->tid_retry_timeout_jiffies = qp->timeout_jiffies; + if (qp->pmtu == hfi2_enum_to_mtu(OPA_MTU_4096) || + qp->pmtu == hfi2_enum_to_mtu(OPA_MTU_8192)) { + hfi2_tid_rdma_opfn_init(qp, local); + /* + * We only want to set the OPFN requested bit when the + * QP transitions to RTS. + */ + if (attr_mask & IB_QP_STATE && + attr->qp_state == IB_QPS_RTS) { + priv->opfn.requested |= OPFN_MASK(TID_RDMA); + /* + * If the QP is transitioning to RTS and the + * opfn.completed for TID RDMA has already been + * set, the QP is being moved *back* into RTS. + * We can now renegotiate the TID RDMA + * parameters. + */ + if (priv->opfn.completed & + OPFN_MASK(TID_RDMA)) { + priv->opfn.completed &= + ~OPFN_MASK(TID_RDMA); + /* + * Since the opfn.completed bit was + * already set, it is safe to assume + * that the opfn.extended is also set. + */ + opfn_schedule_conn_request(qp); + } + } + } else { + memset(local, 0, sizeof(*local)); + } + } + spin_unlock_irqrestore(&priv->opfn.lock, flags); +} + +void hfi2_opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1) +{ + struct hfi2_qp_priv *priv = qp->priv; + + if (!priv->opfn.extended && hfi2_opfn_extended(bth1) && + HFI2_CAP_IS_KSET(OPFN)) { + priv->opfn.extended = 1; + if (qp->state == IB_QPS_RTS) + opfn_conn_request(qp); + } +} + +int hfi2_opfn_init(void) +{ + opfn_wq = alloc_workqueue("hfi_opfn", + WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | + WQ_PERCPU, + HFI2_MAX_ACTIVE_WORKQUEUE_ENTRIES); + if (!opfn_wq) + return -ENOMEM; + + return 0; +} + +void hfi2_opfn_exit(void) +{ + if (opfn_wq) { + destroy_workqueue(opfn_wq); + opfn_wq = NULL; + } +} diff --git a/drivers/infiniband/hw/hfi2/qp.c b/drivers/infiniband/hw/hfi2/qp.c new file mode 100644 index 000000000000..2dc222bf5963 --- /dev/null +++ b/drivers/infiniband/hw/hfi2/qp.c @@ -0,0 +1,1054 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright(c) 2015 - 2020 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + */ + +#include <linux/err.h> +#include <linux/vmalloc.h> +#include <linux/hash.h> +#include <linux/module.h> +#include <linux/seq_file.h> +#include <rdma/rdma_vt.h> +#include <rdma/rdmavt_qp.h> +#include <rdma/ib_verbs.h> + +#include "hfi2.h" +#include "qp.h" +#include "trace.h" +#include "verbs_txreq.h" + +unsigned int hfi2_qp_table_size = 256; + +static void flush_tx_list(struct rvt_qp *qp); +static int iowait_sleep(struct sdma_engine *sde, struct iowait_work *wait, + struct sdma_txreq *stx, unsigned int seq, + bool pkts_sent); +static void iowait_wakeup(struct iowait *wait, int reason); +static void iowait_sdma_drained(struct iowait *wait); +static void qp_pio_drain(struct rvt_qp *qp); + +const struct rvt_operation_params hfi2_post_parms[RVT_OPERATION_MAX] = { +[IB_WR_RDMA_WRITE] = { + .length = sizeof(struct ib_rdma_wr), + .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), +}, + +[IB_WR_RDMA_READ] = { + .length = sizeof(struct ib_rdma_wr), + .qpt_support = BIT(IB_QPT_RC), + .flags = RVT_OPERATION_ATOMIC, +}, + +[IB_WR_ATOMIC_CMP_AND_SWP] = { + .length = sizeof(struct ib_atomic_wr), + .qpt_support = BIT(IB_QPT_RC), + .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE, +}, + +[IB_WR_ATOMIC_FETCH_AND_ADD] = { + .length = sizeof(struct ib_atomic_wr), + .qpt_support = BIT(IB_QPT_RC), + .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE, +}, + +[IB_WR_RDMA_WRITE_WITH_IMM] = { + .length = sizeof(struct ib_rdma_wr), + .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), +}, + +[IB_WR_SEND] = { + .length = sizeof(struct ib_send_wr), + .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) | + BIT(IB_QPT_UC) | BIT(IB_QPT_RC), +}, + +[IB_WR_SEND_WITH_IMM] = { + .length = sizeof(struct ib_send_wr), + .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) | + BIT(IB_QPT_UC) | BIT(IB_QPT_RC), +}, + +[IB_WR_REG_MR] = { + .length = sizeof(struct ib_reg_wr), + .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), + .flags = RVT_OPERATION_LOCAL, +}, + +[IB_WR_LOCAL_INV] = { + .length = sizeof(struct ib_send_wr), + .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), + .flags = RVT_OPERATION_LOCAL, +}, + +[IB_WR_SEND_WITH_INV] = { + .length = sizeof(struct ib_send_wr), + .qpt_support = BIT(IB_QPT_RC), +}, + +[IB_WR_OPFN] = { + .length = sizeof(struct ib_atomic_wr), + .qpt_support = BIT(IB_QPT_RC), + .flags = RVT_OPERATION_USE_RESERVE, +}, + +[IB_WR_TID_RDMA_WRITE] = { + .length = sizeof(struct ib_rdma_wr), + .qpt_support = BIT(IB_QPT_RC), + .flags = RVT_OPERATION_IGN_RNR_CNT, +}, + +}; + +static void flush_list_head(struct list_head *l) +{ + while (!list_empty(l)) { + struct sdma_txreq *tx; + + tx = list_first_entry(l, struct sdma_txreq, list); + list_del_init(&tx->list); + hfi2_put_txreq(container_of(tx, struct verbs_txreq, txreq)); + } +} + +static void flush_tx_list(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + + flush_list_head(&iowait_get_ib_work(&priv->s_iowait)->tx_head); + flush_list_head(&iowait_get_tid_work(&priv->s_iowait)->tx_head); +} + +static void flush_iowait(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + unsigned long flags; + seqlock_t *lock = priv->s_iowait.lock; + + if (!lock) + return; + write_seqlock_irqsave(lock, flags); + if (!list_empty(&priv->s_iowait.list)) { + list_del_init(&priv->s_iowait.list); + priv->s_iowait.lock = NULL; + rvt_put_qp(qp); + } + write_sequnlock_irqrestore(lock, flags); +} + +/* + * This function is what we would push to the core layer if we wanted to be a + * "first class citizen". Instead we hide this here and rely on Verbs ULPs + * to blindly pass the MTU enum value from the PathRecord to us. + */ +static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu) +{ + /* Constraining 10KB packets to 8KB packets */ + if (mtu == (enum ib_mtu)OPA_MTU_10240) + mtu = (enum ib_mtu)OPA_MTU_8192; + return opa_mtu_enum_to_int((enum opa_mtu)mtu); +} + +int hfi2_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct ib_qp *ibqp = &qp->ibqp; + struct hfi2_ibdev *dev = to_idev(ibqp->device); + struct hfi2_devdata *dd = dd_from_dev(dev); + u8 sc; + + if (attr_mask & IB_QP_AV) { + sc = hfi2_ah_to_sc(ibqp->device, &attr->ah_attr); + if (sc == 0xf) + return -EINVAL; + + if (!hfi2_qp_to_sdma_engine(qp, sc) && + dd->flags & HFI2_HAS_SEND_DMA) + return -EINVAL; + + if (!hfi2_qp_to_send_context(qp, sc)) + return -EINVAL; + } + + if (attr_mask & IB_QP_ALT_PATH) { + sc = hfi2_ah_to_sc(ibqp->device, &attr->alt_ah_attr); + if (sc == 0xf) + return -EINVAL; + + if (!hfi2_qp_to_sdma_engine(qp, sc) && + dd->flags & HFI2_HAS_SEND_DMA) + return -EINVAL; + + if (!hfi2_qp_to_send_context(qp, sc)) + return -EINVAL; + } + + return 0; +} + +/* + * qp_set_16b - Set the hdr_type based on whether the slid or the + * dlid in the connection is extended. Only applicable for RC and UC + * QPs. UD QPs determine this on the fly from the ah in the wqe + */ +static inline void qp_set_16b(struct rvt_qp *qp) +{ + struct hfi2_pportdata *ppd; + struct hfi2_ibport *ibp; + struct hfi2_qp_priv *priv = qp->priv; + + /* Update ah_attr to account for extended LIDs */ + hfi2_update_ah_attr(qp->ibqp.device, &qp->remote_ah_attr); + + /* Create 32 bit LIDs */ + hfi2_make_opa_lid(&qp->remote_ah_attr); + + if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) + return; + + ibp = to_iport(qp->ibqp.device, qp->port_num); + ppd = ppd_from_ibp(ibp); + priv->hdr_type = hfi2_get_hdr_type(ppd->lid, &qp->remote_ah_attr); +} + +void hfi2_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + struct ib_qp *ibqp = &qp->ibqp; + struct hfi2_qp_priv *priv = qp->priv; + + if (attr_mask & IB_QP_AV) { + priv->s_sc = hfi2_ah_to_sc(ibqp->device, &qp->remote_ah_attr); + priv->s_sde = hfi2_qp_to_sdma_engine(qp, priv->s_sc); + priv->s_sendcontext = hfi2_qp_to_send_context(qp, priv->s_sc); + qp_set_16b(qp); + } + + if (attr_mask & IB_QP_PATH_MIG_STATE && + attr->path_mig_state == IB_MIG_MIGRATED && + qp->s_mig_state == IB_MIG_ARMED) { + qp->s_flags |= HFI2_S_AHG_CLEAR; + priv->s_sc = hfi2_ah_to_sc(ibqp->device, &qp->remote_ah_attr); + priv->s_sde = hfi2_qp_to_sdma_engine(qp, priv->s_sc); + priv->s_sendcontext = hfi2_qp_to_send_context(qp, priv->s_sc); + qp_set_16b(qp); + } + + if (attr_mask & IB_QP_PORT) { + /* + * Set or replace associated receive context based on the new + * port number. Note: At call time, qp->port_num is updated, + * but qp->ibqp.port is not updated. + */ + priv->rcd = hfi2_qp_to_rcd(qp); + } + + hfi2_opfn_qp_init(qp, attr, attr_mask); +} + +/** + * hfi2_setup_wqe - set up the wqe + * @qp: The qp + * @wqe: The built wqe + * @call_send: Determine if the send should be posted or scheduled. + * + * Perform setup of the wqe. This is called + * prior to inserting the wqe into the ring but after + * the wqe has been setup by RDMAVT. This function + * allows the driver the opportunity to perform + * validation and additional setup of the wqe. + * + * Returns 0 on success, -EINVAL on failure + * + */ +int hfi2_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) +{ + struct hfi2_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct rvt_ah *ah; + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + + switch (qp->ibqp.qp_type) { + case IB_QPT_RC: + hfi2_setup_tid_rdma_wqe(qp, wqe); + fallthrough; + case IB_QPT_UC: + if (wqe->length > 0x80000000U) + return -EINVAL; + if (wqe->length > qp->pmtu) + *call_send = false; + break; + case IB_QPT_SMI: + /* + * SM packets should exclusively use VL15 and their SL is + * ignored (IBTA v1.3, Section 3.5.8.2). Therefore, when ah + * is created, SL is 0 in most cases and as a result some + * fields (vl and pmtu) in ah may not be set correctly, + * depending on the SL2SC and SC2VL tables at the time. + */ + if (wqe->length > ppd->vld[15].mtu) + return -EINVAL; + break; + case IB_QPT_GSI: + case IB_QPT_UD: + ah = rvt_get_swqe_ah(wqe); + if (wqe->length > (1 << ah->log_pmtu)) + return -EINVAL; + if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf) + return -EINVAL; + break; + default: + break; + } + + /* + * Reject any non-SMP transmit attempts unless the link is ACTIVE + * or ARMED + NeighborNormal. + */ + if (qp->ibqp.qp_type != IB_QPT_SMI && + !(ppd->host_link_state == HLS_UP_ACTIVE || + (ppd->host_link_state == HLS_UP_ARMED && ppd->neighbor_normal))) { + return -EINVAL; + } + + /* + * System latency between send and schedule is large enough that + * forcing call_send to true for hfi2_piothreshold packets is necessary. + */ + if (wqe->length <= hfi2_piothreshold) + *call_send = true; + return 0; +} + +/** + * _hfi2_schedule_send - schedule progress + * @qp: the QP + * + * This schedules qp progress w/o regard to the s_flags. + * + * It is only used in the post send, which doesn't hold + * the s_lock. + */ +bool _hfi2_schedule_send(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct hfi2_devdata *dd = ppd->dd; + + if (dd->flags & HFI2_SHUTDOWN) + return true; + + return iowait_schedule( + &priv->s_iowait, dd->hfi2_wq, + priv->s_sde ? priv->s_sde->cpu : + cpumask_first(cpumask_of_node(dd->node))); +} + +static void qp_pio_drain(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + + if (!priv->s_sendcontext) + return; + while (iowait_pio_pending(&priv->s_iowait)) { + write_seqlock_irq(&priv->s_sendcontext->waitlock); + hfi2_sc_wantpiobuf_intr(priv->s_sendcontext, 1); + write_sequnlock_irq(&priv->s_sendcontext->waitlock); + iowait_pio_drain(&priv->s_iowait); + write_seqlock_irq(&priv->s_sendcontext->waitlock); + hfi2_sc_wantpiobuf_intr(priv->s_sendcontext, 0); + write_sequnlock_irq(&priv->s_sendcontext->waitlock); + } +} + +/** + * hfi2_schedule_send - schedule progress + * @qp: the QP + * + * This schedules qp progress and caller should hold + * the s_lock. + * @return true if the first leg is scheduled; + * false if the first leg is not scheduled. + */ +bool hfi2_schedule_send(struct rvt_qp *qp) +{ + lockdep_assert_held(&qp->s_lock); + if (hfi2_send_ok(qp)) { + _hfi2_schedule_send(qp); + return true; + } + if (qp->s_flags & HFI2_S_ANY_WAIT_IO) + hfi2_iowait_set_flag( + &((struct hfi2_qp_priv *)qp->priv)->s_iowait, + IOWAIT_PENDING_IB); + return false; +} + +static void hfi2_qp_schedule(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + bool ret; + + if (hfi2_iowait_flag_set(&priv->s_iowait, IOWAIT_PENDING_IB)) { + ret = hfi2_schedule_send(qp); + if (ret) + hfi2_iowait_clear_flag(&priv->s_iowait, + IOWAIT_PENDING_IB); + } + if (hfi2_iowait_flag_set(&priv->s_iowait, IOWAIT_PENDING_TID)) { + ret = hfi2_schedule_tid_send(qp); + if (ret) + hfi2_iowait_clear_flag(&priv->s_iowait, + IOWAIT_PENDING_TID); + } +} + +void hfi2_qp_wakeup(struct rvt_qp *qp, u32 flag) +{ + unsigned long flags; + + spin_lock_irqsave(&qp->s_lock, flags); + if (qp->s_flags & flag) { + qp->s_flags &= ~flag; + trace_hfi2_qpwakeup(qp, flag); + hfi2_qp_schedule(qp); + } + spin_unlock_irqrestore(&qp->s_lock, flags); + rvt_put_qp(qp); +} + +void hfi2_qp_unbusy(struct rvt_qp *qp, struct iowait_work *wait) +{ + struct hfi2_qp_priv *priv = qp->priv; + + if (hfi2_iowait_set_work_flag(wait) == IOWAIT_IB_SE) { + qp->s_flags &= ~RVT_S_BUSY; + /* + * If we are sending a first-leg packet from the second leg, + * we need to clear the busy flag from priv->s_flags to + * avoid a race condition when the qp wakes up before + * the call to hfi2_verbs_send() returns to the second + * leg. In that case, the second leg will terminate without + * being re-scheduled, resulting in failure to send TID RDMA + * WRITE DATA and TID RDMA ACK packets. + */ + if (priv->s_flags & HFI2_S_TID_BUSY_SET) { + priv->s_flags &= ~(HFI2_S_TID_BUSY_SET | RVT_S_BUSY); + hfi2_iowait_set_flag(&priv->s_iowait, + IOWAIT_PENDING_TID); + } + } else { + priv->s_flags &= ~RVT_S_BUSY; + } +} + +static int iowait_sleep(struct sdma_engine *sde, struct iowait_work *wait, + struct sdma_txreq *stx, uint seq, bool pkts_sent) +{ + struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq); + struct rvt_qp *qp; + struct hfi2_qp_priv *priv; + unsigned long flags; + int ret = 0; + + qp = tx->qp; + priv = qp->priv; + + spin_lock_irqsave(&qp->s_lock, flags); + if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { + /* + * If we couldn't queue the DMA request, save the info + * and try again later rather than destroying the + * buffer and undoing the side effects of the copy. + */ + /* Make a common routine? */ + list_add_tail(&stx->list, &wait->tx_head); + write_seqlock(&sde->waitlock); + if (sdma_progress(sde, seq, stx)) + goto eagain; + if (list_empty(&priv->s_iowait.list)) { + struct hfi2_ibport *ibp = + to_iport(qp->ibqp.device, qp->port_num); + + ibp->rvp.n_dmawait++; + qp->s_flags |= RVT_S_WAIT_DMA_DESC; + iowait_get_priority(&priv->s_iowait); + iowait_queue(pkts_sent, &priv->s_iowait, &sde->dmawait); + priv->s_iowait.lock = &sde->waitlock; + trace_hfi2_qpsleep(qp, RVT_S_WAIT_DMA_DESC); + rvt_get_qp(qp); + } + write_sequnlock(&sde->waitlock); + hfi2_qp_unbusy(qp, wait); + spin_unlock_irqrestore(&qp->s_lock, flags); + ret = -EIOCBQUEUED; + } else { + spin_unlock_irqrestore(&qp->s_lock, flags); + /* + * Return a unique error so the caller can identify the + * unqueued case. + */ + ret = -EIO; + } + return ret; +eagain: + write_sequnlock(&sde->waitlock); + spin_unlock_irqrestore(&qp->s_lock, flags); + list_del_init(&stx->list); + return -EAGAIN; +} + +static void iowait_wakeup(struct iowait *wait, int reason) +{ + struct rvt_qp *qp = iowait_to_qp(wait); + + WARN_ON(reason != SDMA_AVAIL_REASON); + hfi2_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC); +} + +static void iowait_sdma_drained(struct iowait *wait) +{ + struct rvt_qp *qp = iowait_to_qp(wait); + unsigned long flags; + + /* + * This happens when the send engine notes + * a QP in the error state and cannot + * do the flush work until that QP's + * sdma work has finished. + */ + spin_lock_irqsave(&qp->s_lock, flags); + if (qp->s_flags & RVT_S_WAIT_DMA) { + qp->s_flags &= ~RVT_S_WAIT_DMA; + hfi2_schedule_send(qp); + } + spin_unlock_irqrestore(&qp->s_lock, flags); +} + +static void hfi2_init_priority(struct iowait *w) +{ + struct rvt_qp *qp = iowait_to_qp(w); + struct hfi2_qp_priv *priv = qp->priv; + + if (qp->s_flags & RVT_S_ACK_PENDING) + w->priority++; + if (priv->s_flags & RVT_S_ACK_PENDING) + w->priority++; +} + +/** + * hfi2_qp_to_sdma_engine - map a qp to a send engine + * @qp: the QP + * @sc5: the 5 bit sc + * + * Return: + * A send engine for the qp or NULL for SMI type qp. + */ +struct sdma_engine *hfi2_qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5) +{ + struct hfi2_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct hfi2_devdata *dd = ppd->dd; + struct sdma_engine *sde; + + if (!(dd->flags & HFI2_HAS_SEND_DMA)) + return NULL; + switch (qp->ibqp.qp_type) { + case IB_QPT_SMI: + return NULL; + default: + break; + } + sde = hfi2_sdma_select_engine_sc(ppd, qp->ibqp.qp_num >> ppd->qos_shift, + sc5); + return sde; +} + +/** + * hfi2_qp_to_send_context - map a qp to a send context + * @qp: the QP + * @sc5: the 5 bit sc + * + * Return: + * A send context for the qp + */ +struct send_context *hfi2_qp_to_send_context(struct rvt_qp *qp, u8 sc5) +{ + struct hfi2_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + + switch (qp->ibqp.qp_type) { + case IB_QPT_SMI: + /* SMA packets to VL15 */ + return ppd->vld[15].sc; + default: + break; + } + + return hfi2_pio_select_send_context_sc( + ppd, qp->ibqp.qp_num >> ppd->qos_shift, sc5); +} + +static const char *const qp_type_str[] = { + "SMI", "GSI", "RC", "UC", "UD", +}; + +static int qp_idle(struct rvt_qp *qp) +{ + return qp->s_last == qp->s_acked && qp->s_acked == qp->s_cur && + qp->s_cur == qp->s_tail && qp->s_tail == qp->s_head; +} + +/** + * hfi2_qp_iter_print - print the qp information to seq_file + * @s: the seq_file to emit the qp information on + * @iter: the iterator for the qp hash list + */ +void hfi2_qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) +{ + struct rvt_swqe *wqe; + struct rvt_qp *qp = iter->qp; + struct hfi2_qp_priv *priv = qp->priv; + struct sdma_engine *sde; + struct send_context *send_context; + struct rvt_ack_entry *e = NULL; + struct rvt_srq *srq = qp->ibqp.srq ? ibsrq_to_rvtsrq(qp->ibqp.srq) : + NULL; + + sde = hfi2_qp_to_sdma_engine(qp, priv->s_sc); + wqe = rvt_get_swqe_ptr(qp, qp->s_last); + send_context = hfi2_qp_to_send_context(qp, priv->s_sc); + if (qp->s_ack_queue) + e = &qp->s_ack_queue[qp->s_tail_ack_queue]; + seq_printf( + s, + "N %d %s QP %x R %u %s %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x RNR %d %s %d\n", + iter->n, qp_idle(qp) ? "I" : "B", qp->ibqp.qp_num, + atomic_read(&qp->refcount), qp_type_str[qp->ibqp.qp_type], + qp->state, wqe ? wqe->wr.opcode : 0, qp->s_flags, + iowait_sdma_pending(&priv->s_iowait), + iowait_pio_pending(&priv->s_iowait), + !list_empty(&priv->s_iowait.list), qp->timeout, + wqe ? wqe->ssn : 0, qp->s_lsn, qp->s_last_psn, qp->s_psn, + qp->s_next_psn, qp->s_sending_psn, qp->s_sending_hpsn, + qp->r_psn, qp->s_last, qp->s_acked, qp->s_cur, qp->s_tail, + qp->s_head, qp->s_size, qp->s_avail, + /* ack_queue ring pointers, size */ + qp->s_tail_ack_queue, qp->r_head_ack_queue, + rvt_max_atomic(&to_idev(qp->ibqp.device)->rdi), + /* remote QP info */ + qp->remote_qpn, rdma_ah_get_dlid(&qp->remote_ah_attr), + rdma_ah_get_sl(&qp->remote_ah_attr), qp->pmtu, qp->s_retry, + qp->s_retry_cnt, qp->s_rnr_retry_cnt, qp->s_rnr_retry, sde, + sde ? sde->this_idx : 0, send_context, + send_context ? send_context->sw_index : 0, + ib_cq_head(qp->ibqp.send_cq), ib_cq_tail(qp->ibqp.send_cq), + qp->pid, qp->s_state, qp->s_ack_state, + /* ack queue information */ + e ? e->opcode : 0, e ? e->psn : 0, e ? e->lpsn : 0, + qp->r_min_rnr_timer, srq ? "SRQ" : "RQ", + srq ? srq->rq.size : qp->r_rq.size); +} + +void *hfi2_qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv; + + priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, rdi->dparms.node); + if (!priv) + return ERR_PTR(-ENOMEM); + + priv->owner = qp; + + priv->s_ahg = kzalloc_node(sizeof(*priv->s_ahg), GFP_KERNEL, + rdi->dparms.node); + if (!priv->s_ahg) { + kfree(priv); + return ERR_PTR(-ENOMEM); + } + hfi2_iowait_init(&priv->s_iowait, 1, _hfi2_do_send, _hfi2_do_tid_send, + iowait_sleep, iowait_wakeup, iowait_sdma_drained, + hfi2_init_priority); + INIT_WORK(&priv->opfn.opfn_work, hfi2_opfn_send_conn_request); + /* Init to a value to start the running average correctly */ + priv->s_running_pkt_size = hfi2_piothreshold / 2; + return priv; +} + +void hfi2_qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + + hfi2_qp_priv_tid_free(rdi, qp); + kfree(priv->s_ahg); + kfree(priv); +} + +unsigned int hfi2_free_all_qps(struct rvt_dev_info *rdi) +{ + struct hfi2_ibdev *verbs_dev = + container_of(rdi, struct hfi2_ibdev, rdi); + struct hfi2_devdata *dd = + container_of(verbs_dev, struct hfi2_devdata, verbs_dev); + int n; + unsigned int qp_inuse = 0; + + for (n = 0; n < dd->num_pports; n++) { + struct hfi2_ibport *ibp = &dd->pport[n].ibport_data; + + rcu_read_lock(); + if (rcu_dereference(ibp->rvp.qp[0])) + qp_inuse++; + if (rcu_dereference(ibp->rvp.qp[1])) + qp_inuse++; + rcu_read_unlock(); + } + + return qp_inuse; +} + +/* this is a duplicate of sw/rdmavt/qp.c */ +static void get_map_page(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map) +{ + unsigned long page = get_zeroed_page(GFP_KERNEL); + + /* Free the page if someone raced with us installing it. */ + + spin_lock(&qpt->lock); + if (map->page) + free_page(page); + else + map->page = (void *)page; + spin_unlock(&qpt->lock); +} + +/* + * Ensure the context number part of QPN remains within bounds of + * recv contexts assigned to this unit. This context number has no + * relationship to how the QPN is used, it is only to differentiate + * "ours" vs. "theirs" for QPNs associated with the same physical + * adapter port (SLID == DLID). + */ +static u32 fixup_qpn(u32 qpn, struct rvt_qpn_table *qpt, + struct hfi2_devdata *dd) +{ + struct hfi2_devrsrcs *dr = &dd->rsrcs; + u32 qpm = dd->rctxt_mask << 1; + u32 qpi = dd->rctxt_mask + 1; /* incrementer for context part */ + + if (((qpn & qpm) >> 1) >= dr->c.last_rcv_context) { + qpn = (qpn & ~qpm) + ((qpi | dr->c.first_rcv_context) << 1); + /* wrap at end of current maps or end of number space */ + if (qpn / RVT_BITS_PER_PAGE >= qpt->nmaps || qpn >= RVT_QPN_MAX) + qpn = (dr->c.first_rcv_context << 1) | ((qpn & 1) ^ 1); + } + if (qpn == qpt->last) { + /* one full pass completed */ + qpn = RVT_QPN_MAX; + } + return qpn; +} + +static u32 next_qpn(u32 qpn, struct rvt_qpn_table *qpt, struct hfi2_devdata *dd) +{ + return fixup_qpn(qpn + qpt->incr, qpt, dd); +} + +static u32 first_qpn(struct rvt_qpn_table *qpt, struct hfi2_devdata *dd) +{ + return next_qpn(qpt->last, qpt, dd); +} + +static u32 new_map_qpn(struct rvt_qpn_table *qpt, struct hfi2_devdata *dd) +{ + struct hfi2_devrsrcs *dr = &dd->rsrcs; + u32 qpn; + + if (qpt->nmaps == RVT_QPNMAP_ENTRIES) + return RVT_QPN_MAX; + qpn = qpt->nmaps++ * RVT_BITS_PER_PAGE + + (dr->c.first_rcv_context << 1) + (qpt->last & 1); + return qpn; +} + +/* + * use only qpn[8:1] that fall inside our context number range, + * to avoid conflicts with other VFs or PF0. + * + * cloned from sw/rdmavt/qp.c + */ +int hfi2_sriov_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, + enum ib_qp_type type, u32 port_num) +{ + struct hfi2_ibdev *ibdev = dev_from_rdi(rdi); + struct hfi2_devdata *dd = dd_from_dev(ibdev); + u32 offset, qpn; + struct rvt_qpn_map *map; + u32 ret; + + /* Must not be using QoS... ? also qpt->incr == 2? */ + if (rdi->dparms.qos_shift != 1) { + ret = -EINVAL; + goto bail; + } + + /* These are the same no matter what */ + if (type == IB_QPT_SMI || type == IB_QPT_GSI) { + unsigned int n; + + ret = type == IB_QPT_GSI; + n = 1 << (ret + 2 * (port_num - 1)); + spin_lock(&qpt->lock); + if (qpt->flags & n) + ret = -EINVAL; + else + qpt->flags |= n; + spin_unlock(&qpt->lock); + goto bail; + } + + /* + * qpn[8:1] will always match a valid ctxt number for the SI. + * This does not mean bits [8:1] indicate the receiving ctxt, + * it only avoids conflicts with other SIs and allows + * easier setup of RcvQPMapTable CSRs. + */ + qpn = first_qpn(qpt, dd); + /* + * First, scan existing maps once (qpt->last..(qpt->nmaps)..qpt->last). + * If none found, allocate a new map - which is guaranteed + * to have freespace. If RVT_QPNMAP_ENTRIES is reached then fail. + */ + for (;;) { + if (unlikely(qpn >= RVT_QPN_MAX)) { + qpn = new_map_qpn(qpt, dd); + if (unlikely(qpn >= RVT_QPN_MAX)) + goto nomem; + } + offset = qpn & RVT_BITS_PER_PAGE_MASK; + map = &qpt->map[qpn / RVT_BITS_PER_PAGE]; + if (unlikely(!map->page)) { + get_map_page(qpt, map); + if (unlikely(!map->page)) + goto nomem; + } + if (!test_and_set_bit(offset, map->page)) { + qpt->last = qpn; + ret = qpn; + goto bail; + } + qpn = next_qpn(qpn, qpt, dd); + } +nomem: + ret = -ENOMEM; +bail: + return ret; +} + +/* + * This determines QPN validity for use with 'ppd' based on hfi2_sriov_alloc_qpn(), + * specifically the algorithm of fixup_qpn(). + */ +int hfi2_valid_qp(struct hfi2_pportdata *ppd, u32 qpn) +{ + struct hfi2_devrsrcs *dr = &ppd->dd->rsrcs; + u32 qpm = ppd->dd->rctxt_mask << 1; + u16 ctxt; + + if (!ppd->dd->is_sriov) + return 1; + ctxt = (qpn & qpm) >> 1; + return (ctxt >= dr->c.first_rcv_context && + ctxt < dr->c.last_rcv_context); +} + +void hfi2_flush_qp_waiters(struct rvt_qp *qp) +{ + lockdep_assert_held(&qp->s_lock); + flush_iowait(qp); + hfi2_tid_rdma_flush_wait(qp); +} + +void hfi2_stop_send_queue(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + + hfi2_iowait_cancel_work(&priv->s_iowait); + if (cancel_work_sync(&priv->tid_rdma.trigger_work)) + rvt_put_qp(qp); +} + +void hfi2_quiesce_qp(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + + hfi2_del_tid_reap_timer(qp); + hfi2_del_tid_retry_timer(qp); + iowait_sdma_drain(&priv->s_iowait); + qp_pio_drain(qp); + flush_tx_list(qp); +} + +void hfi2_notify_qp_reset(struct rvt_qp *qp) +{ + hfi2_qp_kern_exp_rcv_clear_all(qp); + qp->r_adefered = 0; + clear_ahg(qp); + + /* Clear any OPFN state */ + if (qp->ibqp.qp_type == IB_QPT_RC) + hfi2_opfn_conn_error(qp); +} + +/* + * Switch to alternate path. + * The QP s_lock should be held and interrupts disabled. + */ +void hfi2_migrate_qp(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct ib_event ev; + + qp->s_mig_state = IB_MIG_MIGRATED; + qp->remote_ah_attr = qp->alt_ah_attr; + qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); + qp->s_pkey_index = qp->s_alt_pkey_index; + qp->s_flags |= HFI2_S_AHG_CLEAR; + priv->s_sc = hfi2_ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr); + priv->s_sde = hfi2_qp_to_sdma_engine(qp, priv->s_sc); + qp_set_16b(qp); + + ev.device = qp->ibqp.device; + ev.element.qp = &qp->ibqp; + ev.event = IB_EVENT_PATH_MIG; + qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); +} + +int hfi2_mtu_to_path_mtu(u32 mtu) +{ + return hfi2_mtu_to_enum(mtu, OPA_MTU_8192); +} + +u32 hfi2_mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) +{ + u32 mtu; + struct hfi2_ibdev *verbs_dev = + container_of(rdi, struct hfi2_ibdev, rdi); + struct hfi2_devdata *dd = + container_of(verbs_dev, struct hfi2_devdata, verbs_dev); + struct hfi2_pportdata *ppd = &dd->pport[qp->port_num - 1]; + struct hfi2_ibport *ibp = &ppd->ibport_data; + u8 sc, vl; + + sc = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; + vl = sc_to_vlt(ppd, sc); + + mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu); + if (vl < PER_VL_SEND_CONTEXTS) + mtu = min_t(u32, mtu, ppd->vld[vl].mtu); + return mtu; +} + +int hfi2_get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, + struct ib_qp_attr *attr) +{ + int mtu, pidx = qp->port_num - 1; + struct hfi2_ibdev *verbs_dev = + container_of(rdi, struct hfi2_ibdev, rdi); + struct hfi2_devdata *dd = + container_of(verbs_dev, struct hfi2_devdata, verbs_dev); + mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu); + if (mtu == -1) + return -1; /* values less than 0 are error */ + + if (mtu > dd->pport[pidx].ibmtu) + return hfi2_mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048); + else + return attr->path_mtu; +} + +void hfi2_notify_error_qp(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + seqlock_t *lock = priv->s_iowait.lock; + + if (lock) { + write_seqlock(lock); + if (!list_empty(&priv->s_iowait.list) && + !(qp->s_flags & RVT_S_BUSY) && + !(priv->s_flags & RVT_S_BUSY)) { + qp->s_flags &= ~HFI2_S_ANY_WAIT_IO; + hfi2_iowait_clear_flag(&priv->s_iowait, + IOWAIT_PENDING_IB); + hfi2_iowait_clear_flag(&priv->s_iowait, + IOWAIT_PENDING_TID); + list_del_init(&priv->s_iowait.list); + priv->s_iowait.lock = NULL; + rvt_put_qp(qp); + } + write_sequnlock(lock); + } + + if (!(qp->s_flags & RVT_S_BUSY) && !(priv->s_flags & RVT_S_BUSY)) { + qp->s_hdrwords = 0; + if (qp->s_rdma_mr) { + rvt_put_mr(qp->s_rdma_mr); + qp->s_rdma_mr = NULL; + } + flush_tx_list(qp); + } +} + +/** + * hfi2_qp_iter_cb - callback for iterator + * @qp: the qp + * @v: the sl in low bits of v + * + * This is called from the iterator callback to work + * on an individual qp. + */ +static void hfi2_qp_iter_cb(struct rvt_qp *qp, u64 v) +{ + int lastwqe; + struct ib_event ev; + struct hfi2_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + u8 sl = (u8)v; + + if (qp->port_num != ppd->port || + (qp->ibqp.qp_type != IB_QPT_UC && qp->ibqp.qp_type != IB_QPT_RC) || + rdma_ah_get_sl(&qp->remote_ah_attr) != sl || + !(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK)) + return; + + spin_lock_irq(&qp->r_lock); + spin_lock(&qp->s_hlock); + spin_lock(&qp->s_lock); + lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); + spin_unlock(&qp->s_lock); + spin_unlock(&qp->s_hlock); + spin_unlock_irq(&qp->r_lock); + if (lastwqe) { + ev.device = qp->ibqp.device; + ev.element.qp = &qp->ibqp; + ev.event = IB_EVENT_QP_LAST_WQE_REACHED; + qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); + } +} + +/** + * hfi2_error_port_qps - put a port's RC/UC qps into error state + * @ibp: the ibport. + * @sl: the service level. + * + * This function places all RC/UC qps with a given service level into error + * state. It is generally called to force upper lay apps to abandon stale qps + * after an sl->sc mapping change. + */ +void hfi2_error_port_qps(struct hfi2_ibport *ibp, u8 sl) +{ + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct hfi2_ibdev *dev = &ppd->dd->verbs_dev; + + rvt_qp_iter(&dev->rdi, sl, hfi2_qp_iter_cb); +} diff --git a/drivers/infiniband/hw/hfi2/ruc.c b/drivers/infiniband/hw/hfi2/ruc.c new file mode 100644 index 000000000000..7d20a2a5c0db --- /dev/null +++ b/drivers/infiniband/hw/hfi2/ruc.c @@ -0,0 +1,622 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + */ + +#include <linux/spinlock.h> + +#include "hfi2.h" +#include "mad.h" +#include "qp.h" +#include "verbs_txreq.h" +#include "chip_gen.h" +#include "trace.h" + +static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id) +{ + return (gid->global.interface_id == id && + (gid->global.subnet_prefix == gid_prefix || + gid->global.subnet_prefix == IB_DEFAULT_GID_PREFIX)); +} + +/* + * + * This should be called with the QP r_lock held. + * + * The s_lock will be acquired around the hfi2_migrate_qp() call. + */ +int hfi2_ruc_check_hdr(struct hfi2_ibport *ibp, struct hfi2_packet *packet) +{ + __be64 guid; + unsigned long flags; + struct rvt_qp *qp = packet->qp; + u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; + u32 dlid = packet->dlid; + u32 slid = packet->slid; + u32 sl = packet->sl; + bool migrated = packet->migrated; + u16 pkey = packet->pkey; + + if (qp->s_mig_state == IB_MIG_ARMED && migrated) { + if (!packet->grh) { + if ((rdma_ah_get_ah_flags(&qp->alt_ah_attr) & + IB_AH_GRH) && + (packet->etype != RHF_RCV_TYPE_BYPASS)) + return 1; + } else { + const struct ib_global_route *grh; + + if (!(rdma_ah_get_ah_flags(&qp->alt_ah_attr) & + IB_AH_GRH)) + return 1; + grh = rdma_ah_read_grh(&qp->alt_ah_attr); + guid = get_sguid(ibp, grh->sgid_index); + if (!gid_ok(&packet->grh->dgid, ibp->rvp.gid_prefix, + guid)) + return 1; + if (!gid_ok( + &packet->grh->sgid, + grh->dgid.global.subnet_prefix, + grh->dgid.global.interface_id)) + return 1; + } + if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), pkey, + sc5, slid))) { + hfi2_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num, + slid, dlid); + return 1; + } + /* Validate the SLID. See Ch. 9.6.1.5 and 17.2.8 */ + if (slid != rdma_ah_get_dlid(&qp->alt_ah_attr) || + ppd_from_ibp(ibp)->port != + rdma_ah_get_port_num(&qp->alt_ah_attr)) + return 1; + spin_lock_irqsave(&qp->s_lock, flags); + hfi2_migrate_qp(qp); + spin_unlock_irqrestore(&qp->s_lock, flags); + } else { + if (!packet->grh) { + if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & + IB_AH_GRH) && + (packet->etype != RHF_RCV_TYPE_BYPASS)) + return 1; + } else { + const struct ib_global_route *grh; + + if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & + IB_AH_GRH)) + return 1; + grh = rdma_ah_read_grh(&qp->remote_ah_attr); + guid = get_sguid(ibp, grh->sgid_index); + if (!gid_ok(&packet->grh->dgid, ibp->rvp.gid_prefix, + guid)) + return 1; + if (!gid_ok( + &packet->grh->sgid, + grh->dgid.global.subnet_prefix, + grh->dgid.global.interface_id)) + return 1; + } + if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), pkey, + sc5, slid))) { + hfi2_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num, + slid, dlid); + return 1; + } + /* Validate the SLID. See Ch. 9.6.1.5 */ + if ((slid != rdma_ah_get_dlid(&qp->remote_ah_attr)) || + ppd_from_ibp(ibp)->port != qp->port_num) + return 1; + if (qp->s_mig_state == IB_MIG_REARM && !migrated) + qp->s_mig_state = IB_MIG_ARMED; + } + + return 0; +} + +/** + * hfi2_make_grh - construct a GRH header + * @ibp: a pointer to the IB port + * @hdr: a pointer to the GRH header being constructed + * @grh: the global route address to send to + * @hwords: size of header after grh being sent in dwords + * @nwords: the number of 32 bit words of data being sent + * + * Return the size of the header in 32 bit words. + */ +u32 hfi2_make_grh(struct hfi2_ibport *ibp, struct ib_grh *hdr, + const struct ib_global_route *grh, u32 hwords, u32 nwords) +{ + hdr->version_tclass_flow = + cpu_to_be32((IB_GRH_VERSION << IB_GRH_VERSION_SHIFT) | + (grh->traffic_class << IB_GRH_TCLASS_SHIFT) | + (grh->flow_label << IB_GRH_FLOW_SHIFT)); + hdr->paylen = cpu_to_be16((hwords + nwords) << 2); + /* next_hdr is defined by C8-7 in ch. 8.4.1 */ + hdr->next_hdr = IB_GRH_NEXT_HDR; + hdr->hop_limit = grh->hop_limit; + /* The SGID is 32-bit aligned. */ + hdr->sgid.global.subnet_prefix = ibp->rvp.gid_prefix; + hdr->sgid.global.interface_id = + grh->sgid_index < HFI2_GUIDS_PER_PORT ? + get_sguid(ibp, grh->sgid_index) : + get_sguid(ibp, HFI2_PORT_GUID_INDEX); + hdr->dgid = grh->dgid; + + /* GRH header size in 32-bit words. */ + return sizeof(struct ib_grh) / sizeof(u32); +} + +#define BTH2_OFFSET (offsetof(struct hfi2_sdma_header, \ + hdr.ibh.u.oth.bth[2]) / 4) + +/** + * build_ahg - create ahg in s_ahg + * @qp: a pointer to QP + * @npsn: the next PSN for the request/response + * + * This routine handles the AHG by allocating an ahg entry and causing the + * copy of the first middle. + * + * Subsequent middles use the copied entry, editing the + * PSN with 1 or 2 edits. + */ +static inline void build_ahg(struct rvt_qp *qp, u32 npsn) +{ + struct hfi2_qp_priv *priv = qp->priv; + + if (unlikely(qp->s_flags & HFI2_S_AHG_CLEAR)) + clear_ahg(qp); + if (!(qp->s_flags & HFI2_S_AHG_VALID)) { + /* first middle that needs copy */ + if (qp->s_ahgidx < 0) + qp->s_ahgidx = hfi2_sdma_ahg_alloc(priv->s_sde); + if (qp->s_ahgidx >= 0) { + qp->s_ahgpsn = npsn; + priv->s_ahg->tx_flags |= SDMA_TXREQ_F_AHG_COPY; + /* save to protect a change in another thread */ + priv->s_ahg->ahgidx = qp->s_ahgidx; + qp->s_flags |= HFI2_S_AHG_VALID; + } + } else { + /* subsequent middle after valid */ + if (qp->s_ahgidx >= 0) { + priv->s_ahg->tx_flags |= SDMA_TXREQ_F_USE_AHG; + priv->s_ahg->ahgidx = qp->s_ahgidx; + priv->s_ahg->ahgcount++; + priv->s_ahg->ahgdesc[0] = + sdma_build_ahg_descriptor( + (__force u16)cpu_to_be16((u16)npsn), + BTH2_OFFSET, + 16, + 16); + if ((npsn & 0xffff0000) != + (qp->s_ahgpsn & 0xffff0000)) { + priv->s_ahg->ahgcount++; + priv->s_ahg->ahgdesc[1] = + sdma_build_ahg_descriptor( + (__force u16)cpu_to_be16( + (u16)(npsn >> 16)), + BTH2_OFFSET, + 0, + 16); + } + } + } +} + +static inline void hfi2_make_ruc_bth(struct rvt_qp *qp, + struct ib_other_headers *ohdr, + u32 bth0, u32 bth1, u32 bth2) +{ + ohdr->bth[0] = cpu_to_be32(bth0); + ohdr->bth[1] = cpu_to_be32(bth1); + ohdr->bth[2] = cpu_to_be32(bth2); +} + +/** + * hfi2_make_ruc_header_16B - build a 16B header + * @qp: the queue pair + * @ohdr: a pointer to the destination header memory + * @bth0: bth0 passed in from the RC/UC builder + * @bth1: bth1 passed in from the RC/UC builder + * @bth2: bth2 passed in from the RC/UC builder + * @middle: non zero implies indicates ahg "could" be used + * @ps: the current packet state + * + * This routine may disarm ahg under these situations: + * - packet needs a GRH + * - BECN needed + * - migration state not IB_MIG_MIGRATED + */ +static inline void hfi2_make_ruc_header_16B(struct rvt_qp *qp, + struct ib_other_headers *ohdr, + u32 bth0, u32 bth1, u32 bth2, + int middle, + struct hfi2_pkt_state *ps) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_ibport *ibp = ps->ibp; + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + u32 slid; + u16 pkey = hfi2_get_pkey(ibp, qp->s_pkey_index); + u8 l4 = OPA_16B_L4_IB_LOCAL; + u8 extra_bytes; + u32 nwords; + bool becn = false; + + if (ppd->dd->params->chip_type == CHIP_WFR) { + extra_bytes = hfi2_get_16b_padding((ps->s_txreq->hdr_dwords << 2), + ps->s_txreq->s_cur_size); + nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size + + extra_bytes + SIZE_OF_LT) >> 2); + } else { + /* round up to multiple of 8 */ + extra_bytes = hfi2_pad8((ps->s_txreq->hdr_dwords << 2) + + ps->s_txreq->s_cur_size); + /* add in ICRC QW */ + nwords = (ps->s_txreq->s_cur_size + extra_bytes + 8) >> 2; + } + + if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && + hfi2_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { + struct ib_grh *grh; + struct ib_global_route *grd = + rdma_ah_retrieve_grh(&qp->remote_ah_attr); + /* + * Ensure OPA GIDs are transformed to IB gids + * before creating the GRH. + */ + if (grd->sgid_index == OPA_GID_INDEX) + grd->sgid_index = 0; + grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; + l4 = OPA_16B_L4_IB_GLOBAL; + ps->s_txreq->hdr_dwords += + hfi2_make_grh(ibp, grh, grd, + ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, + nwords); + middle = 0; + } + + if (qp->s_mig_state == IB_MIG_MIGRATED) + bth1 |= OPA_BTH_MIG_REQ; + else + middle = 0; + + if (qp->s_flags & RVT_S_ECN) { + qp->s_flags &= ~RVT_S_ECN; + /* we recently received a FECN, so return a BECN */ + becn = true; + middle = 0; + } + if (middle) + build_ahg(qp, bth2); + else + qp->s_flags &= ~HFI2_S_AHG_VALID; + + bth0 |= pkey; + bth0 |= extra_bytes << 20; + hfi2_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); + + if (!ppd->lid) + slid = be32_to_cpu(OPA_LID_PERMISSIVE); + else + slid = ppd->lid | + (rdma_ah_get_path_bits(&qp->remote_ah_attr) & + ((1 << ppd->lmc) - 1)); + + hfi2_make_16b_hdr(&ps->s_txreq->phdr.hdr.opah, + slid, + opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), + 16B), + (ps->s_txreq->hdr_dwords + nwords) >> 1, + pkey, becn, 0, l4, priv->s_sc); +} + +/** + * hfi2_make_ruc_header_9B - build a 9B header + * @qp: the queue pair + * @ohdr: a pointer to the destination header memory + * @bth0: bth0 passed in from the RC/UC builder + * @bth1: bth1 passed in from the RC/UC builder + * @bth2: bth2 passed in from the RC/UC builder + * @middle: non zero implies indicates ahg "could" be used + * @ps: the current packet state + * + * This routine may disarm ahg under these situations: + * - packet needs a GRH + * - BECN needed + * - migration state not IB_MIG_MIGRATED + */ +static inline void hfi2_make_ruc_header_9B(struct rvt_qp *qp, + struct ib_other_headers *ohdr, + u32 bth0, u32 bth1, u32 bth2, + int middle, + struct hfi2_pkt_state *ps) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_ibport *ibp = ps->ibp; + u16 pkey = hfi2_get_pkey(ibp, qp->s_pkey_index); + u16 lrh0 = HFI2_LRH_BTH; + u8 extra_bytes = -ps->s_txreq->s_cur_size & 3; + u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size + + extra_bytes) >> 2); + + if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { + struct ib_grh *grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; + + lrh0 = HFI2_LRH_GRH; + ps->s_txreq->hdr_dwords += + hfi2_make_grh(ibp, grh, + rdma_ah_read_grh(&qp->remote_ah_attr), + ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, + nwords); + middle = 0; + } + lrh0 |= (priv->s_sc & 0xf) << 12 | + (rdma_ah_get_sl(&qp->remote_ah_attr) & 0xf) << 4; + + if (qp->s_mig_state == IB_MIG_MIGRATED) + bth0 |= IB_BTH_MIG_REQ; + else + middle = 0; + + if (qp->s_flags & RVT_S_ECN) { + qp->s_flags &= ~RVT_S_ECN; + /* we recently received a FECN, so return a BECN */ + bth1 |= (IB_BECN_MASK << IB_BECN_SHIFT); + middle = 0; + } + if (middle) + build_ahg(qp, bth2); + else + qp->s_flags &= ~HFI2_S_AHG_VALID; + + bth0 |= pkey; + bth0 |= extra_bytes << 20; + hfi2_make_ruc_bth(qp, ohdr, bth0, bth1, bth2); + hfi2_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh, + lrh0, + ps->s_txreq->hdr_dwords + nwords, + opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), + ppd_from_ibp(ibp)->lid | + rdma_ah_get_path_bits(&qp->remote_ah_attr)); +} + +typedef void (*hfi2_make_ruc_hdr)(struct rvt_qp *qp, + struct ib_other_headers *ohdr, + u32 bth0, u32 bth1, u32 bth2, int middle, + struct hfi2_pkt_state *ps); + +/* We support only two types - 9B and 16B for now */ +static const hfi2_make_ruc_hdr hfi2_ruc_header_tbl[2] = { + [HFI2_PKT_TYPE_9B] = &hfi2_make_ruc_header_9B, + [HFI2_PKT_TYPE_16B] = &hfi2_make_ruc_header_16B +}; + +void hfi2_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr, + u32 bth0, u32 bth1, u32 bth2, int middle, + struct hfi2_pkt_state *ps) +{ + struct hfi2_qp_priv *priv = qp->priv; + + /* + * reset s_ahg/AHG fields + * + * This insures that the ahgentry/ahgcount + * are at a non-AHG default to protect + * build_verbs_tx_desc() from using + * an include ahgidx. + * + * build_ahg() will modify as appropriate + * to use the AHG feature. + */ + priv->s_ahg->tx_flags = 0; + priv->s_ahg->ahgcount = 0; + priv->s_ahg->ahgidx = 0; + + /* Make the appropriate header */ + hfi2_ruc_header_tbl[priv->hdr_type](qp, ohdr, bth0, bth1, bth2, middle, + ps); +} + +/* when sending, force a reschedule every one of these periods */ +#define SEND_RESCHED_TIMEOUT (5 * HZ) /* 5s in jiffies */ + +/** + * hfi2_schedule_send_yield - test for a yield required for QP + * send engine + * @qp: a pointer to QP + * @ps: a pointer to a structure with commonly lookup values for + * the send engine progress + * @tid: true if it is the tid leg + * + * This routine checks if the time slice for the QP has expired + * for RC QPs, if so an additional work entry is queued. At this + * point, other QPs have an opportunity to be scheduled. It + * returns true if a yield is required, otherwise, false + * is returned. + */ +bool hfi2_schedule_send_yield(struct rvt_qp *qp, struct hfi2_pkt_state *ps, + bool tid) +{ + ps->pkts_sent = true; + + if (unlikely(time_after(jiffies, ps->timeout))) { + if (!ps->in_thread || + workqueue_congested(ps->cpu, ps->ppd->dd->hfi2_wq)) { + spin_lock_irqsave(&qp->s_lock, ps->flags); + if (!tid) { + qp->s_flags &= ~RVT_S_BUSY; + hfi2_schedule_send(qp); + } else { + struct hfi2_qp_priv *priv = qp->priv; + + if (priv->s_flags & + HFI2_S_TID_BUSY_SET) { + qp->s_flags &= ~RVT_S_BUSY; + priv->s_flags &= + ~(HFI2_S_TID_BUSY_SET | + RVT_S_BUSY); + } else { + priv->s_flags &= ~RVT_S_BUSY; + } + hfi2_schedule_tid_send(qp); + } + + spin_unlock_irqrestore(&qp->s_lock, ps->flags); + this_cpu_inc(*ps->ppd->dd->send_schedule); + trace_hfi2_rc_expired_time_slice(qp, true); + return true; + } + + cond_resched(); + this_cpu_inc(*ps->ppd->dd->send_schedule); + ps->timeout = jiffies + ps->timeout_int; + } + + trace_hfi2_rc_expired_time_slice(qp, false); + return false; +} + +void hfi2_do_send_from_rvt(struct rvt_qp *qp) +{ + hfi2_do_send(qp, false); +} + +void _hfi2_do_send(struct work_struct *work) +{ + struct iowait_work *w = container_of(work, struct iowait_work, iowork); + struct rvt_qp *qp = iowait_to_qp(w->iow); + + hfi2_do_send(qp, true); +} + +/** + * hfi2_do_send - perform a send on a QP + * @qp: a pointer to the QP + * @in_thread: true if in a workqueue thread + * + * Process entries in the send work queue until credit or queue is + * exhausted. Only allow one CPU to send a packet per QP. + * Otherwise, two threads could send packets out of order. + */ +void hfi2_do_send(struct rvt_qp *qp, bool in_thread) +{ + struct hfi2_pkt_state ps; + struct hfi2_qp_priv *priv = qp->priv; + int (*make_req)(struct rvt_qp *qp, struct hfi2_pkt_state *ps); + + ps.dev = to_idev(qp->ibqp.device); + ps.ibp = to_iport(qp->ibqp.device, qp->port_num); + ps.ppd = ppd_from_ibp(ps.ibp); + /* complete with error any sends that arrive on an unavailable port */ + if (!port_available_ppd(ps.ppd)) { + struct rvt_swqe *wqe; + + if (qp->s_last != READ_ONCE(qp->s_head)) { + wqe = rvt_get_swqe_ptr(qp, qp->s_last); + rvt_send_complete(qp, wqe, IB_WC_GENERAL_ERR); + } + return; + } + ps.in_thread = in_thread; + ps.wait = iowait_get_ib_work(&priv->s_iowait); + ps.hfi2_loopback = false; + + trace_hfi2_rc_do_send(qp, in_thread); + + switch (qp->ibqp.qp_type) { + case IB_QPT_RC: + if (!hfi2_loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) & + ~((1 << ps.ppd->lmc) - 1)) == + ps.ppd->lid)) { + if (hfi2_valid_qp(ps.ppd, qp->remote_qpn)) { + rvt_ruc_loopback(qp); + return; + } + if (ps.ppd->dd->params->chip_type == CHIP_JKR) { + /* + * recv context to receive from more than one port. + */ + ppd_dev_warn_ratelimited(ps.ppd, "VF-VF RC send not supported on same port\n"); + rvt_send_complete(qp, rvt_get_swqe_ptr(qp, qp->s_last), + IB_WC_RETRY_EXC_ERR); + return; + } + ps.hfi2_loopback = true; + } + make_req = hfi2_make_rc_req; + ps.timeout_int = qp->timeout_jiffies; + break; + case IB_QPT_UC: + if (!hfi2_loopback && ((rdma_ah_get_dlid(&qp->remote_ah_attr) & + ~((1 << ps.ppd->lmc) - 1)) == + ps.ppd->lid)) { + if (hfi2_valid_qp(ps.ppd, qp->remote_qpn)) { + rvt_ruc_loopback(qp); + return; + } + if (ps.ppd->dd->params->chip_type == CHIP_JKR) { + /* + * recv context to receive from more than one port. + */ + ppd_dev_warn_ratelimited(ps.ppd, "VF-VF UC send not supported on same port\n"); + rvt_send_complete(qp, rvt_get_swqe_ptr(qp, qp->s_last), + IB_WC_RETRY_EXC_ERR); + return; + } + ps.hfi2_loopback = true; + } + make_req = hfi2_make_uc_req; + ps.timeout_int = SEND_RESCHED_TIMEOUT; + break; + default: + make_req = hfi2_make_ud_req; + ps.timeout_int = SEND_RESCHED_TIMEOUT; + } + + spin_lock_irqsave(&qp->s_lock, ps.flags); + + /* Return if we are already busy processing a work request. */ + if (!hfi2_send_ok(qp)) { + if (qp->s_flags & HFI2_S_ANY_WAIT_IO) + hfi2_iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_IB); + spin_unlock_irqrestore(&qp->s_lock, ps.flags); + return; + } + + qp->s_flags |= RVT_S_BUSY; + + ps.timeout_int = ps.timeout_int / 8; + ps.timeout = jiffies + ps.timeout_int; + ps.cpu = priv->s_sde ? priv->s_sde->cpu : + cpumask_first(cpumask_of_node(ps.ppd->dd->node)); + ps.pkts_sent = false; + + /* insure a pre-built packet is handled */ + ps.s_txreq = get_waiting_verbs_txreq(ps.wait); + do { + /* Check for a constructed packet to be sent. */ + if (ps.s_txreq) { + if (priv->s_flags & HFI2_S_TID_BUSY_SET) + qp->s_flags |= RVT_S_BUSY; + spin_unlock_irqrestore(&qp->s_lock, ps.flags); + /* + * If the packet cannot be sent now, return and + * the send engine will be woken up later. + */ + if (hfi2_verbs_send(qp, &ps)) + return; + + /* allow other tasks to run */ + if (hfi2_schedule_send_yield(qp, &ps, false)) + return; + + spin_lock_irqsave(&qp->s_lock, ps.flags); + } + } while (make_req(qp, &ps)); + iowait_starve_clear(ps.pkts_sent, &priv->s_iowait); + spin_unlock_irqrestore(&qp->s_lock, ps.flags); +} diff --git a/drivers/infiniband/hw/hfi2/uc.c b/drivers/infiniband/hw/hfi2/uc.c new file mode 100644 index 000000000000..662092a6e3dc --- /dev/null +++ b/drivers/infiniband/hw/hfi2/uc.c @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + */ + +#include "hfi2.h" +#include "verbs_txreq.h" +#include "qp.h" + +/* cut down ridiculously long IB macro names */ +#define OP(x) UC_OP(x) + +/** + * hfi2_make_uc_req - construct a request packet (SEND, RDMA write) + * @qp: a pointer to the QP + * @ps: the current packet state + * + * Assume s_lock is held. + * + * Return 1 if constructed; otherwise, return 0. + */ +int hfi2_make_uc_req(struct rvt_qp *qp, struct hfi2_pkt_state *ps) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct ib_other_headers *ohdr; + struct rvt_swqe *wqe; + u32 hwords; + u32 bth0 = 0; + u32 len; + u32 pmtu = qp->pmtu; + int middle = 0; + + ps->s_txreq = alloc_txreq(ps->dev, qp); + if (!ps->s_txreq) + goto bail_no_tx; + + if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { + if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) + goto bail; + /* We are in the error state, flush the work request. */ + if (qp->s_last == READ_ONCE(qp->s_head)) + goto bail; + /* If DMAs are in progress, we can't flush immediately. */ + if (iowait_sdma_pending(&priv->s_iowait)) { + qp->s_flags |= RVT_S_WAIT_DMA; + goto bail; + } + clear_ahg(qp); + wqe = rvt_get_swqe_ptr(qp, qp->s_last); + rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); + goto done_free_tx; + } + + if (priv->hdr_type == HFI2_PKT_TYPE_9B) { + /* header size in 32-bit words LRH+BTH = (8+12)/4. */ + hwords = 5; + if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; + else + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; + } else { + /* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */ + hwords = 7; + if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && + (hfi2_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr)))) + ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth; + else + ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth; + } + + /* Get the next send request. */ + wqe = rvt_get_swqe_ptr(qp, qp->s_cur); + qp->s_wqe = NULL; + switch (qp->s_state) { + default: + if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) + goto bail; + /* Check if send work queue is empty. */ + if (qp->s_cur == READ_ONCE(qp->s_head)) { + clear_ahg(qp); + goto bail; + } + /* + * Local operations are processed immediately + * after all prior requests have completed. + */ + if (wqe->wr.opcode == IB_WR_REG_MR || + wqe->wr.opcode == IB_WR_LOCAL_INV) { + int local_ops = 0; + int err = 0; + + if (qp->s_last != qp->s_cur) + goto bail; + if (++qp->s_cur == qp->s_size) + qp->s_cur = 0; + if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { + err = rvt_invalidate_rkey( + qp, wqe->wr.ex.invalidate_rkey); + local_ops = 1; + } + rvt_send_complete(qp, wqe, + err ? IB_WC_LOC_PROT_ERR : + IB_WC_SUCCESS); + if (local_ops) + atomic_dec(&qp->local_ops_pending); + goto done_free_tx; + } + /* + * Start a new request. + */ + qp->s_psn = wqe->psn; + qp->s_sge.sge = wqe->sg_list[0]; + qp->s_sge.sg_list = wqe->sg_list + 1; + qp->s_sge.num_sge = wqe->wr.num_sge; + qp->s_sge.total_len = wqe->length; + len = wqe->length; + qp->s_len = len; + switch (wqe->wr.opcode) { + case IB_WR_SEND: + case IB_WR_SEND_WITH_IMM: + if (len > pmtu) { + qp->s_state = OP(SEND_FIRST); + len = pmtu; + break; + } + if (wqe->wr.opcode == IB_WR_SEND) { + qp->s_state = OP(SEND_ONLY); + } else { + qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); + /* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.ex.imm_data; + hwords += 1; + } + if (wqe->wr.send_flags & IB_SEND_SOLICITED) + bth0 |= IB_BTH_SOLICITED; + qp->s_wqe = wqe; + if (++qp->s_cur >= qp->s_size) + qp->s_cur = 0; + break; + + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + ohdr->u.rc.reth.vaddr = + cpu_to_be64(wqe->rdma_wr.remote_addr); + ohdr->u.rc.reth.rkey = cpu_to_be32(wqe->rdma_wr.rkey); + ohdr->u.rc.reth.length = cpu_to_be32(len); + hwords += sizeof(struct ib_reth) / 4; + if (len > pmtu) { + qp->s_state = OP(RDMA_WRITE_FIRST); + len = pmtu; + break; + } + if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { + qp->s_state = OP(RDMA_WRITE_ONLY); + } else { + qp->s_state = + OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); + /* Immediate data comes after the RETH */ + ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; + hwords += 1; + if (wqe->wr.send_flags & IB_SEND_SOLICITED) + bth0 |= IB_BTH_SOLICITED; + } + qp->s_wqe = wqe; + if (++qp->s_cur >= qp->s_size) + qp->s_cur = 0; + break; + + default: + goto bail; + } + break; + + case OP(SEND_FIRST): + qp->s_state = OP(SEND_MIDDLE); + fallthrough; + case OP(SEND_MIDDLE): + len = qp->s_len; + if (len > pmtu) { + len = pmtu; + middle = HFI2_CAP_IS_KSET(SDMA_AHG); + break; + } + if (wqe->wr.opcode == IB_WR_SEND) { + qp->s_state = OP(SEND_LAST); + } else { + qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); + /* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.ex.imm_data; + hwords += 1; + } + if (wqe->wr.send_flags & IB_SEND_SOLICITED) + bth0 |= IB_BTH_SOLICITED; + qp->s_wqe = wqe; + if (++qp->s_cur >= qp->s_size) + qp->s_cur = 0; + break; + + case OP(RDMA_WRITE_FIRST): + qp->s_state = OP(RDMA_WRITE_MIDDLE); + fallthrough; + case OP(RDMA_WRITE_MIDDLE): + len = qp->s_len; + if (len > pmtu) { + len = pmtu; + middle = HFI2_CAP_IS_KSET(SDMA_AHG); + break; + } + if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { + qp->s_state = OP(RDMA_WRITE_LAST); + } else { + qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); + /* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.ex.imm_data; + hwords += 1; + if (wqe->wr.send_flags & IB_SEND_SOLICITED) + bth0 |= IB_BTH_SOLICITED; + } + qp->s_wqe = wqe; + if (++qp->s_cur >= qp->s_size) + qp->s_cur = 0; + break; + } + qp->s_len -= len; + ps->s_txreq->hdr_dwords = hwords; + ps->s_txreq->sde = priv->s_sde; + ps->s_txreq->ss = &qp->s_sge; + ps->s_txreq->s_cur_size = len; + hfi2_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), + qp->remote_qpn, mask_psn(qp->s_psn++), middle, ps); + return 1; + +done_free_tx: + hfi2_put_txreq(ps->s_txreq); + ps->s_txreq = NULL; + return 1; + +bail: + hfi2_put_txreq(ps->s_txreq); + +bail_no_tx: + ps->s_txreq = NULL; + qp->s_flags &= ~RVT_S_BUSY; + return 0; +} + +/** + * hfi2_uc_rcv - handle an incoming UC packet + * @packet: the packet structure + * + * This is called from qp_rcv() to process an incoming UC packet + * for the given QP. + * Called at interrupt level. + */ +void hfi2_uc_rcv(struct hfi2_packet *packet) +{ + struct hfi2_ibport *ibp = rcd_to_iport(packet->rcd); + void *data = packet->payload; + u32 tlen = packet->tlen; + struct rvt_qp *qp = packet->qp; + struct ib_other_headers *ohdr = packet->ohdr; + u32 opcode = packet->opcode; + u32 hdrsize = packet->hlen; + u32 psn; + u32 pad = packet->pad; + struct ib_wc wc; + u32 pmtu = qp->pmtu; + struct ib_reth *reth; + int ret; + u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2); + + if (hfi2_ruc_check_hdr(ibp, packet)) + return; + + process_ecn(qp, packet); + + psn = ib_bth_get_psn(ohdr); + /* Compare the PSN verses the expected PSN. */ + if (unlikely(cmp_psn(psn, qp->r_psn) != 0)) { + /* + * Handle a sequence error. + * Silently drop any current message. + */ + qp->r_psn = psn; +inv: + if (qp->r_state == OP(SEND_FIRST) || + qp->r_state == OP(SEND_MIDDLE)) { + set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); + qp->r_sge.num_sge = 0; + } else { + rvt_put_ss(&qp->r_sge); + } + qp->r_state = OP(SEND_LAST); + switch (opcode) { + case OP(SEND_FIRST): + case OP(SEND_ONLY): + case OP(SEND_ONLY_WITH_IMMEDIATE): + goto send_first; + + case OP(RDMA_WRITE_FIRST): + case OP(RDMA_WRITE_ONLY): + case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): + goto rdma_first; + + default: + goto drop; + } + } + + /* Check for opcode sequence errors. */ + switch (qp->r_state) { + case OP(SEND_FIRST): + case OP(SEND_MIDDLE): + if (opcode == OP(SEND_MIDDLE) || opcode == OP(SEND_LAST) || + opcode == OP(SEND_LAST_WITH_IMMEDIATE)) + break; + goto inv; + + case OP(RDMA_WRITE_FIRST): + case OP(RDMA_WRITE_MIDDLE): + if (opcode == OP(RDMA_WRITE_MIDDLE) || + opcode == OP(RDMA_WRITE_LAST) || + opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) + break; + goto inv; + + default: + if (opcode == OP(SEND_FIRST) || opcode == OP(SEND_ONLY) || + opcode == OP(SEND_ONLY_WITH_IMMEDIATE) || + opcode == OP(RDMA_WRITE_FIRST) || + opcode == OP(RDMA_WRITE_ONLY) || + opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) + break; + goto inv; + } + + if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) + rvt_comm_est(qp); + + /* OK, process the packet. */ + switch (opcode) { + case OP(SEND_FIRST): + case OP(SEND_ONLY): + case OP(SEND_ONLY_WITH_IMMEDIATE): +send_first: + if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) { + qp->r_sge = qp->s_rdma_read_sge; + } else { + ret = rvt_get_rwqe(qp, false); + if (ret < 0) + goto op_err; + if (!ret) + goto drop; + /* + * qp->s_rdma_read_sge will be the owner + * of the mr references. + */ + qp->s_rdma_read_sge = qp->r_sge; + } + qp->r_rcv_len = 0; + if (opcode == OP(SEND_ONLY)) + goto no_immediate_data; + else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) + goto send_last_imm; + fallthrough; + case OP(SEND_MIDDLE): + /* Check for invalid length PMTU or posted rwqe len. */ + /* + * There will be no padding for 9B packet but 16B packets + * will come in with some padding since we always add + * CRC and LT bytes which will need to be flit aligned + */ + if (unlikely(tlen != (hdrsize + pmtu + extra_bytes))) + goto rewind; + qp->r_rcv_len += pmtu; + if (unlikely(qp->r_rcv_len > qp->r_len)) + goto rewind; + rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false); + break; + + case OP(SEND_LAST_WITH_IMMEDIATE): +send_last_imm: + wc.ex.imm_data = ohdr->u.imm_data; + wc.wc_flags = IB_WC_WITH_IMM; + goto send_last; + case OP(SEND_LAST): +no_immediate_data: + wc.ex.imm_data = 0; + wc.wc_flags = 0; +send_last: + /* Check for invalid length. */ + /* LAST len should be >= 1 */ + if (unlikely(tlen < (hdrsize + extra_bytes))) + goto rewind; + /* Don't count the CRC. */ + tlen -= (hdrsize + extra_bytes); + wc.byte_len = tlen + qp->r_rcv_len; + if (unlikely(wc.byte_len > qp->r_len)) + goto rewind; + wc.opcode = IB_WC_RECV; + rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false); + rvt_put_ss(&qp->s_rdma_read_sge); +last_imm: + wc.wr_id = qp->r_wr_id; + wc.status = IB_WC_SUCCESS; + wc.qp = &qp->ibqp; + wc.src_qp = qp->remote_qpn; + wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; + /* + * It seems that IB mandates the presence of an SL in a + * work completion only for the UD transport (see section + * 11.4.2 of IBTA Vol. 1). + * + * However, the way the SL is chosen below is consistent + * with the way that IB/qib works and is trying avoid + * introducing incompatibilities. + * + * See also OPA Vol. 1, section 9.7.6, and table 9-17. + */ + wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); + /* zero fields that are N/A */ + wc.vendor_err = 0; + wc.pkey_index = 0; + wc.dlid_path_bits = 0; + wc.port_num = 0; + /* Signal completion event if the solicited bit is set. */ + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); + break; + + case OP(RDMA_WRITE_FIRST): + case OP(RDMA_WRITE_ONLY): + case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */ +rdma_first: + if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) { + goto drop; + } + reth = &ohdr->u.rc.reth; + qp->r_len = be32_to_cpu(reth->length); + qp->r_rcv_len = 0; + qp->r_sge.sg_list = NULL; + if (qp->r_len != 0) { + u32 rkey = be32_to_cpu(reth->rkey); + u64 vaddr = be64_to_cpu(reth->vaddr); + int ok; + + /* Check rkey */ + ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, + rkey, IB_ACCESS_REMOTE_WRITE); + if (unlikely(!ok)) + goto drop; + qp->r_sge.num_sge = 1; + } else { + qp->r_sge.num_sge = 0; + qp->r_sge.sge.mr = NULL; + qp->r_sge.sge.vaddr = NULL; + qp->r_sge.sge.length = 0; + qp->r_sge.sge.sge_length = 0; + } + if (opcode == OP(RDMA_WRITE_ONLY)) { + goto rdma_last; + } else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) { + wc.ex.imm_data = ohdr->u.rc.imm_data; + goto rdma_last_imm; + } + fallthrough; + case OP(RDMA_WRITE_MIDDLE): + /* Check for invalid length PMTU or posted rwqe len. */ + if (unlikely(tlen != (hdrsize + pmtu + 4))) + goto drop; + qp->r_rcv_len += pmtu; + if (unlikely(qp->r_rcv_len > qp->r_len)) + goto drop; + rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); + break; + + case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): + wc.ex.imm_data = ohdr->u.imm_data; +rdma_last_imm: + wc.wc_flags = IB_WC_WITH_IMM; + + /* Check for invalid length. */ + /* LAST len should be >= 1 */ + if (unlikely(tlen < (hdrsize + extra_bytes))) + goto drop; + /* Don't count the CRC. */ + tlen -= (hdrsize + extra_bytes); + if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) + goto drop; + if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) { + rvt_put_ss(&qp->s_rdma_read_sge); + } else { + ret = rvt_get_rwqe(qp, true); + if (ret < 0) + goto op_err; + if (!ret) + goto drop; + } + wc.byte_len = qp->r_len; + wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; + rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); + rvt_put_ss(&qp->r_sge); + goto last_imm; + + case OP(RDMA_WRITE_LAST): +rdma_last: + /* Check for invalid length. */ + /* LAST len should be >= 1 */ + if (unlikely(tlen < (hdrsize + extra_bytes))) + goto drop; + /* Don't count the CRC. */ + tlen -= (hdrsize + extra_bytes); + if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) + goto drop; + rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); + rvt_put_ss(&qp->r_sge); + break; + + default: + /* Drop packet for unknown opcodes. */ + goto drop; + } + qp->r_psn++; + qp->r_state = opcode; + return; + +rewind: + set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); + qp->r_sge.num_sge = 0; +drop: + ibp->rvp.n_pkt_drops++; + return; + +op_err: + rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); +} diff --git a/drivers/infiniband/hw/hfi2/ud.c b/drivers/infiniband/hw/hfi2/ud.c new file mode 100644 index 000000000000..20e465ca1b20 --- /dev/null +++ b/drivers/infiniband/hw/hfi2/ud.c @@ -0,0 +1,1037 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright(c) 2015 - 2019 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + */ + +#include <linux/net.h> +#include <rdma/ib_smi.h> + +#include "hfi2.h" +#include "mad.h" +#include "verbs_txreq.h" +#include "trace_ibhdrs.h" +#include "qp.h" + +/* We support only two types - 9B and 16B for now */ +static const hfi2_make_req hfi2_make_ud_req_tbl[2] = { + [HFI2_PKT_TYPE_9B] = &hfi2_make_ud_req_9B, + [HFI2_PKT_TYPE_16B] = &hfi2_make_ud_req_16B +}; + +/** + * ud_loopback - handle send on hfi2_loopback QPs + * @sqp: the sending QP + * @swqe: the send work request + * + * This is called from hfi2_make_ud_req() to forward a WQE addressed + * to the same HFI. + * Note that the receive interrupt handler may be calling hfi2_ud_rcv() + * while this is being called. + */ +static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) +{ + struct hfi2_ibport *ibp = to_iport(sqp->ibqp.device, sqp->port_num); + struct hfi2_pportdata *ppd; + struct hfi2_qp_priv *priv = sqp->priv; + struct rvt_qp *qp; + struct rdma_ah_attr *ah_attr; + unsigned long flags; + struct rvt_sge_state ssge; + struct rvt_sge *sge; + struct ib_wc wc; + u32 length; + enum ib_qp_type sqptype, dqptype; + + rcu_read_lock(); + + qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp, + rvt_get_swqe_remote_qpn(swqe)); + if (!qp) { + ibp->rvp.n_pkt_drops++; + rcu_read_unlock(); + return; + } + + sqptype = sqp->ibqp.qp_type == IB_QPT_GSI ? + IB_QPT_UD : sqp->ibqp.qp_type; + dqptype = qp->ibqp.qp_type == IB_QPT_GSI ? + IB_QPT_UD : qp->ibqp.qp_type; + + if (dqptype != sqptype || + !(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { + ibp->rvp.n_pkt_drops++; + goto drop; + } + + ah_attr = rvt_get_swqe_ah_attr(swqe); + ppd = ppd_from_ibp(ibp); + + if (qp->ibqp.qp_num > 1) { + u16 pkey; + u32 slid; + u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)]; + + pkey = hfi2_get_pkey(ibp, sqp->s_pkey_index); + slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & + ((1 << ppd->lmc) - 1)); + if (unlikely(ingress_pkey_check(ppd, pkey, sc5, + qp->s_pkey_index, + slid, false))) { + hfi2_bad_pkey(ibp, pkey, + rdma_ah_get_sl(ah_attr), + sqp->ibqp.qp_num, qp->ibqp.qp_num, + slid, rdma_ah_get_dlid(ah_attr)); + goto drop; + } + } + + /* + * Check that the qkey matches (except for QP0, see 9.6.1.4.1). + * Qkeys with the high order bit set mean use the + * qkey from the QP context instead of the WR (see 10.2.5). + */ + if (qp->ibqp.qp_num) { + u32 qkey; + + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); + if (unlikely(qkey != qp->qkey)) + goto drop; /* silently drop per IBTA spec */ + } + + /* + * A GRH is expected to precede the data even if not + * present on the wire. + */ + length = swqe->length; + memset(&wc, 0, sizeof(wc)); + wc.byte_len = length + sizeof(struct ib_grh); + + if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) { + wc.wc_flags = IB_WC_WITH_IMM; + wc.ex.imm_data = swqe->wr.ex.imm_data; + } + + spin_lock_irqsave(&qp->r_lock, flags); + + /* + * Get the next work request entry to find where to put the data. + */ + if (qp->r_flags & RVT_R_REUSE_SGE) { + qp->r_flags &= ~RVT_R_REUSE_SGE; + } else { + int ret; + + ret = rvt_get_rwqe(qp, false); + if (ret < 0) { + rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); + goto bail_unlock; + } + if (!ret) { + if (qp->ibqp.qp_num == 0) + ibp->rvp.n_vl15_dropped++; + goto bail_unlock; + } + } + /* Silently drop packets which are too big. */ + if (unlikely(wc.byte_len > qp->r_len)) { + qp->r_flags |= RVT_R_REUSE_SGE; + ibp->rvp.n_pkt_drops++; + goto bail_unlock; + } + + if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { + struct ib_grh grh; + struct ib_global_route grd = *(rdma_ah_read_grh(ah_attr)); + + /* + * For hfi2_loopback packets with extended LIDs, the + * sgid_index in the GRH is 0 and the dgid is + * OPA GID of the sender. While creating a response + * to the hfi2_loopback packet, IB core creates the new + * sgid_index from the DGID and that will be the + * OPA_GID_INDEX. The new dgid is from the sgid + * index and that will be in the IB GID format. + * + * We now have a case where the sent packet had a + * different sgid_index and dgid compared to the + * one that was received in response. + * + * Fix this inconsistency. + */ + if (priv->hdr_type == HFI2_PKT_TYPE_16B) { + if (grd.sgid_index == 0) + grd.sgid_index = OPA_GID_INDEX; + + if (ib_is_opa_gid(&grd.dgid)) + grd.dgid.global.interface_id = + cpu_to_be64(ppd->guids[HFI2_PORT_GUID_INDEX]); + } + + hfi2_make_grh(ibp, &grh, &grd, 0, 0); + rvt_copy_sge(qp, &qp->r_sge, &grh, + sizeof(grh), true, false); + wc.wc_flags |= IB_WC_GRH; + } else { + rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); + } + ssge.sg_list = swqe->sg_list + 1; + ssge.sge = *swqe->sg_list; + ssge.num_sge = swqe->wr.num_sge; + sge = &ssge.sge; + while (length) { + u32 len = rvt_get_sge_length(sge, length); + + WARN_ON_ONCE(len == 0); + rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false); + rvt_update_sge(&ssge, len, false); + length -= len; + } + rvt_put_ss(&qp->r_sge); + if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) + goto bail_unlock; + wc.wr_id = qp->r_wr_id; + wc.status = IB_WC_SUCCESS; + wc.opcode = IB_WC_RECV; + wc.qp = &qp->ibqp; + wc.src_qp = sqp->ibqp.qp_num; + if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) { + if (sqp->ibqp.qp_type == IB_QPT_GSI || + sqp->ibqp.qp_type == IB_QPT_SMI) + wc.pkey_index = rvt_get_swqe_pkey_index(swqe); + else + wc.pkey_index = sqp->s_pkey_index; + } else { + wc.pkey_index = 0; + } + wc.slid = (ppd->lid | (rdma_ah_get_path_bits(ah_attr) & + ((1 << ppd->lmc) - 1))) & U16_MAX; + /* Check for hfi2_loopback when the port lid is not set */ + if (wc.slid == 0 && sqp->ibqp.qp_type == IB_QPT_GSI) + wc.slid = be16_to_cpu(IB_LID_PERMISSIVE); + wc.sl = rdma_ah_get_sl(ah_attr); + wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); + wc.port_num = qp->port_num; + /* Signal completion event if the solicited bit is set. */ + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); + ibp->rvp.n_loop_pkts++; +bail_unlock: + spin_unlock_irqrestore(&qp->r_lock, flags); +drop: + rcu_read_unlock(); +} + +static void hfi2_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe, + struct ib_other_headers *ohdr, + u16 *pkey, u32 extra_bytes, bool bypass) +{ + u32 bth0; + struct hfi2_ibport *ibp; + + ibp = to_iport(qp->ibqp.device, qp->port_num); + if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { + ohdr->u.ud.imm_data = wqe->wr.ex.imm_data; + bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24; + } else { + bth0 = IB_OPCODE_UD_SEND_ONLY << 24; + } + + if (wqe->wr.send_flags & IB_SEND_SOLICITED) + bth0 |= IB_BTH_SOLICITED; + bth0 |= extra_bytes << 20; + if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) + *pkey = hfi2_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); + else + *pkey = hfi2_get_pkey(ibp, qp->s_pkey_index); + if (!bypass) + bth0 |= *pkey; + ohdr->bth[0] = cpu_to_be32(bth0); + ohdr->bth[1] = cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); + ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn)); + /* + * Qkeys with the high order bit set mean use the + * qkey from the QP context instead of the WR (see 10.2.5). + */ + ohdr->u.ud.deth[0] = + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : + rvt_get_swqe_remote_qkey(wqe)); + ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); +} + +void hfi2_make_ud_req_9B(struct rvt_qp *qp, struct hfi2_pkt_state *ps, + struct rvt_swqe *wqe) +{ + u32 nwords, extra_bytes; + u16 len, slid, dlid, pkey; + u16 lrh0 = 0; + u8 sc5; + struct hfi2_qp_priv *priv = qp->priv; + struct ib_other_headers *ohdr; + struct rdma_ah_attr *ah_attr; + struct hfi2_pportdata *ppd; + struct hfi2_ibport *ibp; + struct ib_grh *grh; + + ibp = to_iport(qp->ibqp.device, qp->port_num); + ppd = ppd_from_ibp(ibp); + ah_attr = rvt_get_swqe_ah_attr(wqe); + + extra_bytes = -wqe->length & 3; + nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; + /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */ + ps->s_txreq->hdr_dwords = 7; + if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) + ps->s_txreq->hdr_dwords++; + + if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { + grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh; + ps->s_txreq->hdr_dwords += + hfi2_make_grh(ibp, grh, rdma_ah_read_grh(ah_attr), + ps->s_txreq->hdr_dwords - LRH_9B_DWORDS, + nwords); + lrh0 = HFI2_LRH_GRH; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; + } else { + lrh0 = HFI2_LRH_BTH; + ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; + } + + sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)]; + lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4; + if (qp->ibqp.qp_type == IB_QPT_SMI) { + lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */ + priv->s_sc = 0xf; + } else { + lrh0 |= (sc5 & 0xf) << 12; + priv->s_sc = sc5; + } + + dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B); + if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) { + slid = be16_to_cpu(IB_LID_PERMISSIVE); + } else { + u16 lid = (u16)ppd->lid; + + if (lid) { + lid |= rdma_ah_get_path_bits(ah_attr) & + ((1 << ppd->lmc) - 1); + slid = lid; + } else { + slid = be16_to_cpu(IB_LID_PERMISSIVE); + } + } + hfi2_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false); + len = ps->s_txreq->hdr_dwords + nwords; + + /* Setup the packet */ + ps->s_txreq->phdr.hdr.hdr_type = HFI2_PKT_TYPE_9B; + hfi2_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh, + lrh0, len, dlid, slid); +} + +void hfi2_make_ud_req_16B(struct rvt_qp *qp, struct hfi2_pkt_state *ps, + struct rvt_swqe *wqe) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct ib_other_headers *ohdr; + struct rdma_ah_attr *ah_attr; + struct hfi2_pportdata *ppd; + struct hfi2_ibport *ibp; + u32 dlid, slid, nwords, extra_bytes; + u32 dest_qp = rvt_get_swqe_remote_qpn(wqe); + u32 src_qp = qp->ibqp.qp_num; + u16 len, pkey; + u8 l4, sc5; + bool is_mgmt = false; + + ibp = to_iport(qp->ibqp.device, qp->port_num); + ppd = ppd_from_ibp(ibp); + ah_attr = rvt_get_swqe_ah_attr(wqe); + + /* + * Build 16B Management Packet if either the destination + * or source queue pair number is 0 or 1. + */ + if (dest_qp == 0 || src_qp == 0 || dest_qp == 1 || src_qp == 1) { + /* header size in dwords 16B LRH+L4_FM = (16+8)/4. */ + ps->s_txreq->hdr_dwords = 6; + is_mgmt = true; + } else { + /* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */ + ps->s_txreq->hdr_dwords = 9; + if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) + ps->s_txreq->hdr_dwords++; + } + + if (ppd->dd->params->chip_type == CHIP_WFR) { + /* SW provides space for CRC and LT for bypass packets. */ + extra_bytes = hfi2_get_16b_padding((ps->s_txreq->hdr_dwords << 2), + wqe->length); + nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC; + } else { + /* pad total message to a multiple of 8 */ + extra_bytes = hfi2_pad8((ps->s_txreq->hdr_dwords << 2) + + wqe->length); + /* add in ICRC QW */ + nwords = (wqe->length + extra_bytes + 8) >> 2; + } + + if ((rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) && + hfi2_check_mcast(rdma_ah_get_dlid(ah_attr))) { + struct ib_grh *grh; + struct ib_global_route *grd = rdma_ah_retrieve_grh(ah_attr); + /* + * Ensure OPA GIDs are transformed to IB gids + * before creating the GRH. + */ + if (grd->sgid_index == OPA_GID_INDEX) { + ppd_dev_warn(ppd, "Bad sgid_index. sgid_index: %d\n", + grd->sgid_index); + grd->sgid_index = 0; + } + grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh; + ps->s_txreq->hdr_dwords += hfi2_make_grh( + ibp, grh, grd, + ps->s_txreq->hdr_dwords - LRH_16B_DWORDS, + nwords); + ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth; + l4 = OPA_16B_L4_IB_GLOBAL; + } else { + ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth; + l4 = OPA_16B_L4_IB_LOCAL; + } + + sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)]; + if (qp->ibqp.qp_type == IB_QPT_SMI) + priv->s_sc = 0xf; + else + priv->s_sc = sc5; + + dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 16B); + if (!ppd->lid) + slid = be32_to_cpu(OPA_LID_PERMISSIVE); + else + slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & + ((1 << ppd->lmc) - 1)); + + if (is_mgmt) { + l4 = OPA_16B_L4_FM; + pkey = hfi2_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); + hfi2_16B_set_qpn(&ps->s_txreq->phdr.hdr.opah.u.mgmt, + dest_qp, src_qp); + } else { + hfi2_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true); + } + /* Convert dwords to flits */ + len = (ps->s_txreq->hdr_dwords + nwords) >> 1; + + /* Setup the packet */ + ps->s_txreq->phdr.hdr.hdr_type = HFI2_PKT_TYPE_16B; + hfi2_make_16b_hdr(&ps->s_txreq->phdr.hdr.opah, + slid, dlid, len, pkey, 0, 0, l4, priv->s_sc); +} + +/** + * hfi2_make_ud_req - construct a UD request packet + * @qp: the QP + * @ps: the current packet state + * + * Assume s_lock is held. + * + * Return 1 if constructed; otherwise, return 0. + */ +int hfi2_make_ud_req(struct rvt_qp *qp, struct hfi2_pkt_state *ps) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct rdma_ah_attr *ah_attr; + struct hfi2_pportdata *ppd; + struct hfi2_ibport *ibp; + struct rvt_swqe *wqe; + int next_cur; + u32 lid; + + ps->s_txreq = alloc_txreq(ps->dev, qp); + if (!ps->s_txreq) + goto bail_no_tx; + + if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) { + if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) + goto bail; + /* We are in the error state, flush the work request. */ + if (qp->s_last == READ_ONCE(qp->s_head)) + goto bail; + /* If DMAs are in progress, we can't flush immediately. */ + if (iowait_sdma_pending(&priv->s_iowait)) { + qp->s_flags |= RVT_S_WAIT_DMA; + goto bail; + } + wqe = rvt_get_swqe_ptr(qp, qp->s_last); + rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); + goto done_free_tx; + } + + /* see post_one_send() */ + if (qp->s_cur == READ_ONCE(qp->s_head)) + goto bail; + + wqe = rvt_get_swqe_ptr(qp, qp->s_cur); + next_cur = qp->s_cur + 1; + if (next_cur >= qp->s_size) + next_cur = 0; + + /* Construct the header. */ + ibp = to_iport(qp->ibqp.device, qp->port_num); + ppd = ppd_from_ibp(ibp); + ah_attr = rvt_get_swqe_ah_attr(wqe); + priv->hdr_type = hfi2_get_hdr_type(ppd->lid, ah_attr); + if ((!hfi2_check_mcast(rdma_ah_get_dlid(ah_attr))) || + (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) { + lid = rdma_ah_get_dlid(ah_attr) & ~((1 << ppd->lmc) - 1); + if (unlikely(!hfi2_loopback && + ((lid == ppd->lid) || + ((lid == be32_to_cpu(OPA_LID_PERMISSIVE)) && + (qp->ibqp.qp_type == IB_QPT_GSI))))) { + unsigned long tflags = ps->flags; + /* + * If DMAs are in progress, we can't generate + * a completion for the hfi2_loopback packet since + * it would be out of order. + * Instead of waiting, we could queue a + * zero length descriptor so we get a callback. + */ + if (iowait_sdma_pending(&priv->s_iowait)) { + qp->s_flags |= RVT_S_WAIT_DMA; + goto bail; + } + qp->s_cur = next_cur; + spin_unlock_irqrestore(&qp->s_lock, tflags); + ud_loopback(qp, wqe); + spin_lock_irqsave(&qp->s_lock, tflags); + ps->flags = tflags; + rvt_send_complete(qp, wqe, IB_WC_SUCCESS); + goto done_free_tx; + } + } + + qp->s_cur = next_cur; + ps->s_txreq->s_cur_size = wqe->length; + ps->s_txreq->ss = &qp->s_sge; + qp->s_srate = rdma_ah_get_static_rate(ah_attr); + qp->srate_mbps = ib_rate_to_mbps(qp->s_srate); + qp->s_wqe = wqe; + qp->s_sge.sge = wqe->sg_list[0]; + qp->s_sge.sg_list = wqe->sg_list + 1; + qp->s_sge.num_sge = wqe->wr.num_sge; + qp->s_sge.total_len = wqe->length; + + /* Make the appropriate header */ + hfi2_make_ud_req_tbl[priv->hdr_type](qp, ps, qp->s_wqe); + priv->s_sde = hfi2_qp_to_sdma_engine(qp, priv->s_sc); + ps->s_txreq->sde = priv->s_sde; + priv->s_sendcontext = hfi2_qp_to_send_context(qp, priv->s_sc); + ps->s_txreq->psc = priv->s_sendcontext; + /* disarm any ahg */ + priv->s_ahg->ahgcount = 0; + priv->s_ahg->ahgidx = 0; + priv->s_ahg->tx_flags = 0; + + return 1; + +done_free_tx: + hfi2_put_txreq(ps->s_txreq); + ps->s_txreq = NULL; + return 1; + +bail: + hfi2_put_txreq(ps->s_txreq); + +bail_no_tx: + ps->s_txreq = NULL; + qp->s_flags &= ~RVT_S_BUSY; + return 0; +} + +/* + * Hardware can't check this so we do it here. + * + * This is a slightly different algorithm than the standard pkey check. It + * special cases the management keys and allows for 0x7fff and 0xffff to be in + * the table at the same time. + * + * @returns the index found or -1 if not found + */ +int hfi2_lookup_pkey_idx(struct hfi2_ibport *ibp, u16 pkey) +{ + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + unsigned int i; + + if (pkey == FULL_MGMT_P_KEY || pkey == LIM_MGMT_P_KEY) { + unsigned int lim_idx = -1; + + for (i = 0; i < ppd->dd->params->pkey_table_size; ++i) { + /* here we look for an exact match */ + if (ppd->pkeys[i] == pkey) + return i; + if (ppd->pkeys[i] == LIM_MGMT_P_KEY) + lim_idx = i; + } + + /* did not find 0xffff return 0x7fff idx if found */ + if (pkey == FULL_MGMT_P_KEY) + return lim_idx; + + /* no match... */ + return -1; + } + + pkey &= 0x7fff; /* remove limited/full membership bit */ + + for (i = 0; i < ppd->dd->params->pkey_table_size; ++i) + if ((ppd->pkeys[i] & 0x7fff) == pkey) + return i; + + /* + * Should not get here, this means hardware failed to validate pkeys. + */ + return -1; +} + +void hfi2_return_cnp_16B(struct hfi2_ibport *ibp, struct rvt_qp *qp, + u32 remote_qpn, u16 pkey, u32 slid, u32 dlid, + u8 sc5, const struct ib_grh *old_grh) +{ + u64 pbc; + u32 bth0, plen, vl, hwords = 7; + u32 extra_bytes; + u16 len; + u8 l4; + struct hfi2_opa_header hdr; + struct ib_other_headers *ohdr; + struct pio_buf *pbuf; + struct send_context *ctxt = hfi2_qp_to_send_context(qp, sc5); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct hfi2_devdata *dd = ppd->dd; + u32 nwords; + + /* should only be called for WFR, creat_pbc ignores hfi2_loopback param */ + WARN_ON(dd->params->chip_type != CHIP_WFR); + + hdr.hdr_type = HFI2_PKT_TYPE_16B; + /* Populate length */ + if (dd->params->chip_type == CHIP_WFR) { + extra_bytes = hfi2_get_16b_padding(hwords << 2, 0); + nwords = ((extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC; + } else { + /* pad to multiple of 8 */ + extra_bytes = hfi2_pad8(hwords); + /* add ICRC QW */ + nwords = (extra_bytes + 8) >> 2; + } + if (old_grh) { + struct ib_grh *grh = &hdr.opah.u.l.grh; + + grh->version_tclass_flow = old_grh->version_tclass_flow; + grh->paylen = cpu_to_be16( + (hwords - LRH_16B_DWORDS + nwords) << 2); + grh->hop_limit = 0xff; + grh->sgid = old_grh->dgid; + grh->dgid = old_grh->sgid; + ohdr = &hdr.opah.u.l.oth; + l4 = OPA_16B_L4_IB_GLOBAL; + hwords += sizeof(struct ib_grh) / sizeof(u32); + } else { + ohdr = &hdr.opah.u.oth; + l4 = OPA_16B_L4_IB_LOCAL; + } + + /* BIT 16 to 19 is TVER. Bit 20 to 22 is pad cnt */ + bth0 = (IB_OPCODE_CNP << 24) | (1 << 16) | (extra_bytes << 20); + ohdr->bth[0] = cpu_to_be32(bth0); + + ohdr->bth[1] = cpu_to_be32(remote_qpn); + ohdr->bth[2] = 0; /* PSN 0 */ + + /* Convert dwords to flits */ + len = (hwords + nwords) >> 1; + hfi2_make_16b_hdr(&hdr.opah, slid, dlid, len, pkey, 1, 0, l4, sc5); + + plen = 2 /* PBC */ + hwords + nwords; + vl = sc_to_vlt(ppd, sc5); + if (ctxt) { + pbc = dd->params->create_pbc(ppd, false, 0, qp->srate_mbps, vl, plen, + PBC_L2_16B, dlid, + ctxt->hw_context); + pbuf = hfi2_sc_buffer_alloc(ctxt, plen, NULL, NULL); + if (!IS_ERR_OR_NULL(pbuf)) { + trace_pio_output_ibhdr(dd, &hdr, sc5, 0); + dd->pio_inline_send(dd, pbuf, pbc, &hdr, hwords); + } + } +} + +void hfi2_return_cnp(struct hfi2_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn, + u16 pkey, u32 slid, u32 dlid, u8 sc5, + const struct ib_grh *old_grh) +{ + u64 pbc, pbc_flags = 0; + u32 bth0, plen, vl, hwords = 5; + u16 lrh0; + u8 sl = ibp->sc_to_sl[sc5]; + struct hfi2_opa_header hdr; + struct ib_other_headers *ohdr; + struct pio_buf *pbuf; + struct send_context *ctxt = hfi2_qp_to_send_context(qp, sc5); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct hfi2_devdata *dd = ppd->dd; + + /* should only be called for WFR, creat_pbc ignores hfi2_loopback param */ + WARN_ON(dd->params->chip_type != CHIP_WFR); + + hdr.hdr_type = HFI2_PKT_TYPE_9B; + if (old_grh) { + struct ib_grh *grh = &hdr.ibh.u.l.grh; + + grh->version_tclass_flow = old_grh->version_tclass_flow; + grh->paylen = cpu_to_be16( + (hwords - LRH_9B_DWORDS + SIZE_OF_CRC) << 2); + grh->hop_limit = 0xff; + grh->sgid = old_grh->dgid; + grh->dgid = old_grh->sgid; + ohdr = &hdr.ibh.u.l.oth; + lrh0 = HFI2_LRH_GRH; + hwords += sizeof(struct ib_grh) / sizeof(u32); + } else { + ohdr = &hdr.ibh.u.oth; + lrh0 = HFI2_LRH_BTH; + } + + lrh0 |= (sc5 & 0xf) << 12 | sl << 4; + + bth0 = pkey | (IB_OPCODE_CNP << 24); + ohdr->bth[0] = cpu_to_be32(bth0); + + ohdr->bth[1] = cpu_to_be32(remote_qpn | (1 << IB_BECN_SHIFT)); + ohdr->bth[2] = 0; /* PSN 0 */ + + hfi2_make_ib_hdr(&hdr.ibh, lrh0, hwords + SIZE_OF_CRC, dlid, slid); + plen = 2 /* PBC */ + hwords; + pbc_flags |= pbc_sc4_flag(sc5); + vl = sc_to_vlt(ppd, sc5); + if (ctxt) { + pbc = dd->params->create_pbc(ppd, false, pbc_flags, qp->srate_mbps, vl, + plen, PBC_L2_9B, dlid, + ctxt->hw_context); + pbuf = hfi2_sc_buffer_alloc(ctxt, plen, NULL, NULL); + if (!IS_ERR_OR_NULL(pbuf)) { + trace_pio_output_ibhdr(dd, &hdr, sc5, 0); + dd->pio_inline_send(dd, pbuf, pbc, &hdr, hwords); + } + } +} + +/* + * opa_smp_check() - Do the regular pkey checking, and the additional + * checks for SMPs specified in OPAv1 rev 1.0, 9/19/2016 update, section + * 9.10.25 ("SMA Packet Checks"). + * + * Note that: + * - Checks are done using the pkey directly from the packet's BTH, + * and specifically _not_ the pkey that we attach to the completion, + * which may be different. + * - These checks are specifically for "non-local" SMPs (i.e., SMPs + * which originated on another node). SMPs which are sent from, and + * destined to this node are checked in opa_local_smp_check(). + * + * At the point where opa_smp_check() is called, we know: + * - destination QP is QP0 + * + * opa_smp_check() returns 0 if all checks succeed, 1 otherwise. + */ +static int opa_smp_check(struct hfi2_ibport *ibp, u16 pkey, u8 sc5, + struct rvt_qp *qp, u16 slid, struct opa_smp *smp) +{ + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + + /* + * I don't think it's possible for us to get here with sc != 0xf, + * but check it to be certain. + */ + if (sc5 != 0xf) + return 1; + + if (rcv_pkey_check(ppd, pkey, sc5, slid)) + return 1; + + /* + * At this point we know (and so don't need to check again) that + * the pkey is either LIM_MGMT_P_KEY, or FULL_MGMT_P_KEY + * (see ingress_pkey_check). + */ + if (smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE && + smp->mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED) { + ingress_pkey_table_fail(ppd, pkey, slid); + return 1; + } + + /* + * SMPs fall into one of four (disjoint) categories: + * SMA request, SMA response, SMA trap, or SMA trap repress. + * Our response depends, in part, on which type of SMP we're + * processing. + * + * If this is an SMA response, skip the check here. + * + * If this is an SMA request or SMA trap repress: + * - pkey != FULL_MGMT_P_KEY => + * increment port recv constraint errors, drop MAD + * + * Otherwise: + * - accept if the port is running an SM + * - drop MAD if it's an SMA trap + * - pkey == FULL_MGMT_P_KEY => + * reply with unsupported method + * - pkey != FULL_MGMT_P_KEY => + * increment port recv constraint errors, drop MAD + */ + switch (smp->method) { + case IB_MGMT_METHOD_GET_RESP: + case IB_MGMT_METHOD_REPORT_RESP: + break; + case IB_MGMT_METHOD_GET: + case IB_MGMT_METHOD_SET: + case IB_MGMT_METHOD_REPORT: + case IB_MGMT_METHOD_TRAP_REPRESS: + if (pkey != FULL_MGMT_P_KEY) { + ingress_pkey_table_fail(ppd, pkey, slid); + return 1; + } + break; + default: + if (ibp->rvp.port_cap_flags & IB_PORT_SM) + return 0; + if (smp->method == IB_MGMT_METHOD_TRAP) + return 1; + if (pkey == FULL_MGMT_P_KEY) { + smp->status |= IB_SMP_UNSUP_METHOD; + return 0; + } + ingress_pkey_table_fail(ppd, pkey, slid); + return 1; + } + return 0; +} + +/** + * hfi2_ud_rcv - receive an incoming UD packet + * @packet: the packet structure + * + * This is called from qp_rcv() to process an incoming UD packet + * for the given QP. + * Called at interrupt level. + */ +void hfi2_ud_rcv(struct hfi2_packet *packet) +{ + u32 hdrsize = packet->hlen; + struct ib_wc wc; + u32 src_qp; + u16 pkey; + int mgmt_pkey_idx = -1; + struct hfi2_ibport *ibp = rcd_to_iport(packet->rcd); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + void *data = packet->payload; + u32 tlen = packet->tlen; + struct rvt_qp *qp = packet->qp; + u8 sc5 = packet->sc; + u8 sl_from_sc; + u8 opcode = packet->opcode; + u8 sl = packet->sl; + u32 dlid = packet->dlid; + u32 slid = packet->slid; + u8 extra_bytes; + u8 l4 = 0; + bool dlid_is_permissive; + bool slid_is_permissive; + bool solicited = false; + + extra_bytes = packet->pad + packet->extra_byte + (SIZE_OF_CRC << 2); + + if (packet->etype == RHF_RCV_TYPE_BYPASS) { + u32 permissive_lid = + opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B); + + l4 = hfi2_16B_get_l4(packet->hdr); + pkey = hfi2_16B_get_pkey(packet->hdr); + dlid_is_permissive = (dlid == permissive_lid); + slid_is_permissive = (slid == permissive_lid); + } else { + pkey = ib_bth_get_pkey(packet->ohdr); + dlid_is_permissive = (dlid == be16_to_cpu(IB_LID_PERMISSIVE)); + slid_is_permissive = (slid == be16_to_cpu(IB_LID_PERMISSIVE)); + } + sl_from_sc = ibp->sc_to_sl[sc5]; + + if (likely(l4 != OPA_16B_L4_FM)) { + src_qp = ib_get_sqpn(packet->ohdr); + solicited = ib_bth_is_solicited(packet->ohdr); + } else { + src_qp = hfi2_16B_get_src_qpn(packet->mgmt); + } + + process_ecn(qp, packet); + /* + * Get the number of bytes the message was padded by + * and drop incomplete packets. + */ + if (unlikely(tlen < (hdrsize + extra_bytes))) + goto drop; + + tlen -= hdrsize + extra_bytes; + + /* + * Check that the permissive LID is only used on QP0 + * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1). + */ + if (qp->ibqp.qp_num) { + if (unlikely(dlid_is_permissive || slid_is_permissive)) + goto drop; + if (qp->ibqp.qp_num > 1) { + if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) { + /* + * Traps will not be sent for packets dropped + * by the HW. This is fine, as sending trap + * for invalid pkeys is optional according to + * IB spec (release 1.3, section 10.9.4) + */ + hfi2_bad_pkey(ibp, + pkey, sl, + src_qp, qp->ibqp.qp_num, + slid, dlid); + return; + } + } else { + /* GSI packet */ + mgmt_pkey_idx = hfi2_lookup_pkey_idx(ibp, pkey); + if (mgmt_pkey_idx < 0) + goto drop; + } + if (unlikely(l4 != OPA_16B_L4_FM && + ib_get_qkey(packet->ohdr) != qp->qkey)) + return; /* Silent drop */ + + /* Drop invalid MAD packets (see 13.5.3.1). */ + if (unlikely(qp->ibqp.qp_num == 1 && + (tlen > 2048 || (sc5 == 0xF)))) + goto drop; + } else { + /* Received on QP0, and so by definition, this is an SMP */ + struct opa_smp *smp = (struct opa_smp *)data; + + if (opa_smp_check(ibp, pkey, sc5, qp, slid, smp)) + goto drop; + + if (tlen > 2048) + goto drop; + if ((dlid_is_permissive || slid_is_permissive) && + smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) + goto drop; + + /* look up SMI pkey */ + mgmt_pkey_idx = hfi2_lookup_pkey_idx(ibp, pkey); + if (mgmt_pkey_idx < 0) + goto drop; + } + + if (qp->ibqp.qp_num > 1 && + opcode == IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE) { + wc.ex.imm_data = packet->ohdr->u.ud.imm_data; + wc.wc_flags = IB_WC_WITH_IMM; + } else if (opcode == IB_OPCODE_UD_SEND_ONLY) { + wc.ex.imm_data = 0; + wc.wc_flags = 0; + } else { + goto drop; + } + + /* + * A GRH is expected to precede the data even if not + * present on the wire. + */ + wc.byte_len = tlen + sizeof(struct ib_grh); + + /* + * Get the next work request entry to find where to put the data. + */ + if (qp->r_flags & RVT_R_REUSE_SGE) { + qp->r_flags &= ~RVT_R_REUSE_SGE; + } else { + int ret; + + ret = rvt_get_rwqe(qp, false); + if (ret < 0) { + rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); + return; + } + if (!ret) { + if (qp->ibqp.qp_num == 0) + ibp->rvp.n_vl15_dropped++; + return; + } + } + /* Silently drop packets which are too big. */ + if (unlikely(wc.byte_len > qp->r_len)) { + qp->r_flags |= RVT_R_REUSE_SGE; + goto drop; + } + if (packet->grh) { + rvt_copy_sge(qp, &qp->r_sge, packet->grh, + sizeof(struct ib_grh), true, false); + wc.wc_flags |= IB_WC_GRH; + } else { + rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); + } + rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), + true, false); + rvt_put_ss(&qp->r_sge); + if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) + return; + wc.wr_id = qp->r_wr_id; + wc.status = IB_WC_SUCCESS; + wc.opcode = IB_WC_RECV; + wc.vendor_err = 0; + wc.qp = &qp->ibqp; + wc.src_qp = src_qp; + + if (qp->ibqp.qp_type == IB_QPT_GSI || + qp->ibqp.qp_type == IB_QPT_SMI) { + if (mgmt_pkey_idx < 0) { + if (net_ratelimit()) { + struct hfi2_devdata *dd = ppd->dd; + + dd_dev_err(dd, "QP type %d mgmt_pkey_idx < 0 and packet not dropped???\n", + qp->ibqp.qp_type); + mgmt_pkey_idx = 0; + } + } + wc.pkey_index = (unsigned int)mgmt_pkey_idx; + } else { + wc.pkey_index = 0; + } + if (slid_is_permissive) + slid = be32_to_cpu(OPA_LID_PERMISSIVE); + wc.slid = slid & U16_MAX; + wc.sl = sl_from_sc; + + /* + * Save the LMC lower bits if the destination LID is a unicast LID. + */ + wc.dlid_path_bits = hfi2_check_mcast(dlid) ? 0 : + dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); + wc.port_num = qp->port_num; + /* Signal completion event if the solicited bit is set. */ + rvt_recv_cq(qp, &wc, solicited); + return; + +drop: + ibp->rvp.n_pkt_drops++; +} diff --git a/drivers/infiniband/hw/hfi2/verbs.c b/drivers/infiniband/hw/hfi2/verbs.c new file mode 100644 index 000000000000..b86920eb270a --- /dev/null +++ b/drivers/infiniband/hw/hfi2/verbs.c @@ -0,0 +1,2038 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright(c) 2015 - 2020 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + */ + +#include <rdma/ib_mad.h> +#include <rdma/ib_user_verbs.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/utsname.h> +#include <linux/rculist.h> +#include <linux/mm.h> +#include <linux/vmalloc.h> +#include <rdma/opa_addr.h> +#include <linux/nospec.h> + +#include "hfi2.h" +#include "common.h" +#include "trace.h" +#include "qp.h" +#include "verbs_txreq.h" +#include "debugfs.h" +#include "fault.h" +#include "affinity.h" +#include "ipoib.h" +#include "uverbs.h" +#include "sriov.h" +#include "chip_gen.h" + +extern int hfi2_sriov_alloc_qpn(struct rvt_dev_info *rdi, + struct rvt_qpn_table *qpt, enum ib_qp_type type, + u32 port_num); + +static unsigned int hfi2_lkey_table_size = 16; + +static unsigned int hfi2_max_pds = 0xFFFF; + +static unsigned int hfi2_max_ahs = 0xFFFF; + +static unsigned int hfi2_max_cqes = 0x2FFFFF; + +static unsigned int hfi2_max_cqs = 0x1FFFF; + +static unsigned int hfi2_max_qp_wrs = 0x3FFF; + +static unsigned int hfi2_max_qps = 32768; + +static unsigned int hfi2_max_sges = 0x60; + +static unsigned int hfi2_max_mcast_grps = 16384; + +static unsigned int hfi2_max_mcast_qp_attached = 16; + +static unsigned int hfi2_max_srqs = 1024; + +static unsigned int hfi2_max_srq_sges = 128; + +static unsigned int hfi2_max_srq_wrs = 0x1FFFF; + +unsigned short hfi2_piothreshold = 256; + +static unsigned int sge_copy_mode; + +static void verbs_sdma_complete(struct sdma_txreq *cookie, int status); + +static int pio_wait(struct rvt_qp *qp, struct send_context *sc, + struct hfi2_pkt_state *ps, u32 flag); + +/* Length of buffer to create verbs txreq cache name */ +#define TXREQ_NAME_LEN 24 + +static uint wss_threshold = 80; +static uint wss_clean_period = 256; + +/* + * Translate ib_wr_opcode into ib_wc_opcode. + */ +const enum ib_wc_opcode hfi2_ib_hfi2_wc_opcode[] = { + [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE, + [IB_WR_TID_RDMA_WRITE] = IB_WC_RDMA_WRITE, + [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, + [IB_WR_SEND] = IB_WC_SEND, + [IB_WR_SEND_WITH_IMM] = IB_WC_SEND, + [IB_WR_RDMA_READ] = IB_WC_RDMA_READ, + [IB_WR_TID_RDMA_READ] = IB_WC_RDMA_READ, + [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP, + [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD, + [IB_WR_SEND_WITH_INV] = IB_WC_SEND, + [IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV, + [IB_WR_REG_MR] = IB_WC_REG_MR +}; + +/* + * Length of header by opcode, 0 --> not supported + */ +const u8 hfi2_hdr_len_by_opcode[256] = { + /* RC */ + [IB_OPCODE_RC_SEND_FIRST] = 12 + 8, + [IB_OPCODE_RC_SEND_MIDDLE] = 12 + 8, + [IB_OPCODE_RC_SEND_LAST] = 12 + 8, + [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, + [IB_OPCODE_RC_SEND_ONLY] = 12 + 8, + [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = 12 + 8 + 4, + [IB_OPCODE_RC_RDMA_WRITE_FIRST] = 12 + 8 + 16, + [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = 12 + 8, + [IB_OPCODE_RC_RDMA_WRITE_LAST] = 12 + 8, + [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, + [IB_OPCODE_RC_RDMA_WRITE_ONLY] = 12 + 8 + 16, + [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = 12 + 8 + 20, + [IB_OPCODE_RC_RDMA_READ_REQUEST] = 12 + 8 + 16, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = 12 + 8 + 4, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = 12 + 8, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = 12 + 8 + 4, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = 12 + 8 + 4, + [IB_OPCODE_RC_ACKNOWLEDGE] = 12 + 8 + 4, + [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = 12 + 8 + 4 + 8, + [IB_OPCODE_RC_COMPARE_SWAP] = 12 + 8 + 28, + [IB_OPCODE_RC_FETCH_ADD] = 12 + 8 + 28, + [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = 12 + 8 + 4, + [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = 12 + 8 + 4, + [IB_OPCODE_TID_RDMA_READ_REQ] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_READ_RESP] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_WRITE_REQ] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_WRITE_RESP] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_WRITE_DATA] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_WRITE_DATA_LAST] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_ACK] = 12 + 8 + 36, + [IB_OPCODE_TID_RDMA_RESYNC] = 12 + 8 + 36, + /* UC */ + [IB_OPCODE_UC_SEND_FIRST] = 12 + 8, + [IB_OPCODE_UC_SEND_MIDDLE] = 12 + 8, + [IB_OPCODE_UC_SEND_LAST] = 12 + 8, + [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, + [IB_OPCODE_UC_SEND_ONLY] = 12 + 8, + [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = 12 + 8 + 4, + [IB_OPCODE_UC_RDMA_WRITE_FIRST] = 12 + 8 + 16, + [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = 12 + 8, + [IB_OPCODE_UC_RDMA_WRITE_LAST] = 12 + 8, + [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = 12 + 8 + 4, + [IB_OPCODE_UC_RDMA_WRITE_ONLY] = 12 + 8 + 16, + [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = 12 + 8 + 20, + /* UD */ + [IB_OPCODE_UD_SEND_ONLY] = 12 + 8 + 8, + [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = 12 + 8 + 12 +}; + +static const opcode_handler opcode_handler_tbl[256] = { + /* RC */ + [IB_OPCODE_RC_SEND_FIRST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_MIDDLE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_LAST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_ONLY] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_WRITE_FIRST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_WRITE_LAST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_WRITE_ONLY] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_READ_REQUEST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = &hfi2_rc_rcv, + [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = &hfi2_rc_rcv, + [IB_OPCODE_RC_ACKNOWLEDGE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_COMPARE_SWAP] = &hfi2_rc_rcv, + [IB_OPCODE_RC_FETCH_ADD] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = &hfi2_rc_rcv, + [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = &hfi2_rc_rcv, + + /* TID RDMA has separate handlers for different opcodes.*/ + [IB_OPCODE_TID_RDMA_WRITE_REQ] = &hfi2_rc_rcv_tid_rdma_write_req, + [IB_OPCODE_TID_RDMA_WRITE_RESP] = &hfi2_rc_rcv_tid_rdma_write_resp, + [IB_OPCODE_TID_RDMA_WRITE_DATA] = &hfi2_rc_rcv_tid_rdma_write_data, + [IB_OPCODE_TID_RDMA_WRITE_DATA_LAST] = &hfi2_rc_rcv_tid_rdma_write_data, + [IB_OPCODE_TID_RDMA_READ_REQ] = &hfi2_rc_rcv_tid_rdma_read_req, + [IB_OPCODE_TID_RDMA_READ_RESP] = &hfi2_rc_rcv_tid_rdma_read_resp, + [IB_OPCODE_TID_RDMA_RESYNC] = &hfi2_rc_rcv_tid_rdma_resync, + [IB_OPCODE_TID_RDMA_ACK] = &hfi2_rc_rcv_tid_rdma_ack, + + /* UC */ + [IB_OPCODE_UC_SEND_FIRST] = &hfi2_uc_rcv, + [IB_OPCODE_UC_SEND_MIDDLE] = &hfi2_uc_rcv, + [IB_OPCODE_UC_SEND_LAST] = &hfi2_uc_rcv, + [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = &hfi2_uc_rcv, + [IB_OPCODE_UC_SEND_ONLY] = &hfi2_uc_rcv, + [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = &hfi2_uc_rcv, + [IB_OPCODE_UC_RDMA_WRITE_FIRST] = &hfi2_uc_rcv, + [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = &hfi2_uc_rcv, + [IB_OPCODE_UC_RDMA_WRITE_LAST] = &hfi2_uc_rcv, + [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = &hfi2_uc_rcv, + [IB_OPCODE_UC_RDMA_WRITE_ONLY] = &hfi2_uc_rcv, + [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = &hfi2_uc_rcv, + /* UD */ + [IB_OPCODE_UD_SEND_ONLY] = &hfi2_ud_rcv, + [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = &hfi2_ud_rcv, + /* CNP */ + [IB_OPCODE_CNP] = &hfi2_cnp_rcv +}; + +#define OPMASK 0x1f + +static const u32 pio_opmask[BIT(3)] = { + /* RC */ + [IB_OPCODE_RC >> 5] = + BIT(RC_OP(SEND_ONLY) & OPMASK) | + BIT(RC_OP(SEND_ONLY_WITH_IMMEDIATE) & OPMASK) | + BIT(RC_OP(RDMA_WRITE_ONLY) & OPMASK) | + BIT(RC_OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE) & OPMASK) | + BIT(RC_OP(RDMA_READ_REQUEST) & OPMASK) | + BIT(RC_OP(ACKNOWLEDGE) & OPMASK) | + BIT(RC_OP(ATOMIC_ACKNOWLEDGE) & OPMASK) | + BIT(RC_OP(COMPARE_SWAP) & OPMASK) | + BIT(RC_OP(FETCH_ADD) & OPMASK), + /* UC */ + [IB_OPCODE_UC >> 5] = + BIT(UC_OP(SEND_ONLY) & OPMASK) | + BIT(UC_OP(SEND_ONLY_WITH_IMMEDIATE) & OPMASK) | + BIT(UC_OP(RDMA_WRITE_ONLY) & OPMASK) | + BIT(UC_OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE) & OPMASK), +}; + +/* + * System image GUID. + */ +__be64 hfi2_ib_hfi2_sys_image_guid; + +/* + * Make sure the QP is ready and able to accept the given opcode. + */ +static inline opcode_handler qp_ok(struct hfi2_packet *packet) +{ + if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK)) + return NULL; + if (((packet->opcode & RVT_OPCODE_QP_MASK) == + packet->qp->allowed_ops) || + (packet->opcode == IB_OPCODE_CNP)) + return opcode_handler_tbl[packet->opcode]; + + return NULL; +} + +static u64 hfi2_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc) +{ +#ifdef CONFIG_FAULT_INJECTION + if ((opcode & IB_OPCODE_MSP) == IB_OPCODE_MSP) { + /* + * In order to drop non-IB traffic we + * set PbcInsertHrc to NONE (0x2). + * The packet will still be delivered + * to the receiving node but a + * KHdrHCRCErr (KDETH packet with a bad + * HCRC) will be triggered and the + * packet will not be delivered to the + * correct context. + */ + pbc &= ~PBC_INSERT_HCRC_SMASK; + pbc |= (u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT; + } else { + /* + * In order to drop regular verbs + * traffic we set the PbcTestEbp + * flag. The packet will still be + * delivered to the receiving node but + * a 'late ebp error' will be + * triggered and will be dropped. + */ + pbc |= PBC_TEST_EBP; + } +#endif + return pbc; +} + +static opcode_handler tid_qp_ok(int opcode, struct hfi2_packet *packet) +{ + if (packet->qp->ibqp.qp_type != IB_QPT_RC || + !(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK)) + return NULL; + if ((opcode & RVT_OPCODE_QP_MASK) == IB_OPCODE_TID_RDMA) + return opcode_handler_tbl[opcode]; + return NULL; +} + +void hfi2_kdeth_eager_rcv(struct hfi2_packet *packet) +{ + struct hfi2_ctxtdata *rcd = packet->rcd; + struct ib_header *hdr = packet->hdr; + u32 tlen = packet->tlen; + struct hfi2_pportdata *ppd = rcd->ppd; + struct hfi2_ibport *ibp = &ppd->ibport_data; + struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi; + opcode_handler opcode_handler; + unsigned long flags; + u32 qp_num; + int lnh; + u8 opcode; + + /* DW == LRH (2) + BTH (3) + KDETH (9) + CRC (1) */ + if (unlikely(tlen < 15 * sizeof(u32))) + goto drop; + + lnh = be16_to_cpu(hdr->lrh[0]) & 3; + if (lnh != HFI2_LRH_BTH) + goto drop; + + packet->ohdr = &hdr->u.oth; + trace_input_ibhdr(rcd->dd, packet, packet->sc4); + + opcode = (be32_to_cpu(packet->ohdr->bth[0]) >> 24); + inc_opstats(tlen, &rcd->opstats->stats[opcode]); + + /* verbs_qp can be picked up from any tid_rdma header struct */ + qp_num = be32_to_cpu(packet->ohdr->u.tid_rdma.r_req.verbs_qp) & + RVT_QPN_MASK; + + rcu_read_lock(); + packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); + if (!packet->qp) + goto drop_rcu; + spin_lock_irqsave(&packet->qp->r_lock, flags); + opcode_handler = tid_qp_ok(opcode, packet); + if (likely(opcode_handler)) + opcode_handler(packet); + else + goto drop_unlock; + spin_unlock_irqrestore(&packet->qp->r_lock, flags); + rcu_read_unlock(); + + return; +drop_unlock: + spin_unlock_irqrestore(&packet->qp->r_lock, flags); +drop_rcu: + rcu_read_unlock(); +drop: + ibp->rvp.n_pkt_drops++; +} + +void hfi2_kdeth_expected_rcv(struct hfi2_packet *packet) +{ + struct hfi2_ctxtdata *rcd = packet->rcd; + struct ib_header *hdr = packet->hdr; + u32 tlen = packet->tlen; + struct hfi2_pportdata *ppd = rcd->ppd; + struct hfi2_ibport *ibp = &ppd->ibport_data; + struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi; + opcode_handler opcode_handler; + unsigned long flags; + u32 qp_num; + int lnh; + u8 opcode; + + /* DW == LRH (2) + BTH (3) + KDETH (9) + CRC (1) */ + if (unlikely(tlen < 15 * sizeof(u32))) + goto drop; + + lnh = be16_to_cpu(hdr->lrh[0]) & 3; + if (lnh != HFI2_LRH_BTH) + goto drop; + + packet->ohdr = &hdr->u.oth; + trace_input_ibhdr(rcd->dd, packet, packet->sc4); + + opcode = (be32_to_cpu(packet->ohdr->bth[0]) >> 24); + inc_opstats(tlen, &rcd->opstats->stats[opcode]); + + /* verbs_qp can be picked up from any tid_rdma header struct */ + qp_num = be32_to_cpu(packet->ohdr->u.tid_rdma.r_rsp.verbs_qp) & + RVT_QPN_MASK; + + rcu_read_lock(); + packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); + if (!packet->qp) + goto drop_rcu; + spin_lock_irqsave(&packet->qp->r_lock, flags); + opcode_handler = tid_qp_ok(opcode, packet); + if (likely(opcode_handler)) + opcode_handler(packet); + else + goto drop_unlock; + spin_unlock_irqrestore(&packet->qp->r_lock, flags); + rcu_read_unlock(); + + return; +drop_unlock: + spin_unlock_irqrestore(&packet->qp->r_lock, flags); +drop_rcu: + rcu_read_unlock(); +drop: + ibp->rvp.n_pkt_drops++; +} + +static int hfi2_do_pkey_check(struct hfi2_packet *packet) +{ + struct hfi2_ctxtdata *rcd = packet->rcd; + struct hfi2_pportdata *ppd = rcd->ppd; + struct hfi2_16b_header *hdr = packet->hdr; + u16 pkey; + + /* Pkey check needed only for bypass packets */ + if (packet->etype != RHF_RCV_TYPE_BYPASS) + return 0; + + /* Perform pkey check */ + pkey = hfi2_16B_get_pkey(hdr); + return ingress_pkey_check(ppd, pkey, packet->sc, + packet->qp->s_pkey_index, packet->slid, true); +} + +static inline void hfi2_handle_packet(struct hfi2_packet *packet, bool is_mcast) +{ + u32 qp_num; + struct hfi2_ctxtdata *rcd = packet->rcd; + struct hfi2_pportdata *ppd = rcd->ppd; + struct hfi2_ibport *ibp = rcd_to_iport(rcd); + struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi; + opcode_handler packet_handler; + unsigned long flags; + + inc_opstats(packet->tlen, &rcd->opstats->stats[packet->opcode]); + + if (unlikely(is_mcast)) { + struct rvt_mcast *mcast; + struct rvt_mcast_qp *p; + + if (!packet->grh) + goto drop; + mcast = rvt_mcast_find(&ibp->rvp, &packet->grh->dgid, + opa_get_lid(packet->dlid, 9B)); + if (!mcast) + goto drop; + rcu_read_lock(); + list_for_each_entry_rcu(p, &mcast->qp_list, list) { + packet->qp = p->qp; + if (hfi2_do_pkey_check(packet)) + goto unlock_drop; + spin_lock_irqsave(&packet->qp->r_lock, flags); + packet_handler = qp_ok(packet); + if (likely(packet_handler)) + packet_handler(packet); + else + ibp->rvp.n_pkt_drops++; + spin_unlock_irqrestore(&packet->qp->r_lock, flags); + } + rcu_read_unlock(); + /* + * Notify rvt_multicast_detach() if it is waiting for us + * to finish. + */ + if (atomic_dec_return(&mcast->refcount) <= 1) + wake_up(&mcast->wait); + } else { + /* Get the destination QP number. */ + if (packet->etype == RHF_RCV_TYPE_BYPASS && + hfi2_16B_get_l4(packet->hdr) == OPA_16B_L4_FM) + qp_num = hfi2_16B_get_dest_qpn(packet->mgmt); + else + qp_num = ib_bth_get_qpn(packet->ohdr); + + rcu_read_lock(); + packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); + if (!packet->qp) + goto unlock_drop; + + if (hfi2_do_pkey_check(packet)) + goto unlock_drop; + + spin_lock_irqsave(&packet->qp->r_lock, flags); + packet_handler = qp_ok(packet); + if (likely(packet_handler)) + packet_handler(packet); + else + ibp->rvp.n_pkt_drops++; + spin_unlock_irqrestore(&packet->qp->r_lock, flags); + rcu_read_unlock(); + } + return; +unlock_drop: + rcu_read_unlock(); +drop: + ibp->rvp.n_pkt_drops++; +} + +/** + * hfi2_ib_rcv - process an incoming packet + * @packet: data packet information + * + * This is called to process an incoming packet at interrupt level. + */ +void hfi2_ib_rcv(struct hfi2_packet *packet) +{ + struct hfi2_ctxtdata *rcd = packet->rcd; + + trace_input_ibhdr(rcd->dd, packet, packet->sc4); + hfi2_handle_packet(packet, hfi2_check_mcast(packet->dlid)); +} + +void hfi2_16B_rcv(struct hfi2_packet *packet) +{ + struct hfi2_ctxtdata *rcd = packet->rcd; + + trace_input_ibhdr(rcd->dd, packet, false); + hfi2_handle_packet(packet, hfi2_check_mcast(packet->dlid)); +} + +/* + * This is called from a timer to check for QPs + * which need kernel memory in order to send a packet. + */ +static void mem_timer(struct timer_list *t) +{ + struct hfi2_ibdev *dev = timer_container_of(dev, t, mem_timer); + struct list_head *list = &dev->memwait; + struct rvt_qp *qp = NULL; + struct iowait *wait; + unsigned long flags; + struct hfi2_qp_priv *priv; + + write_seqlock_irqsave(&dev->iowait_lock, flags); + if (!list_empty(list)) { + wait = list_first_entry(list, struct iowait, list); + qp = iowait_to_qp(wait); + priv = qp->priv; + list_del_init(&priv->s_iowait.list); + priv->s_iowait.lock = NULL; + /* refcount held until actual wake up */ + if (!list_empty(list)) + mod_timer(&dev->mem_timer, jiffies + 1); + } + write_sequnlock_irqrestore(&dev->iowait_lock, flags); + + if (qp) + hfi2_qp_wakeup(qp, RVT_S_WAIT_KMEM); +} + +/* + * This is called with progress side lock held. + */ +/* New API */ +static void verbs_sdma_complete(struct sdma_txreq *cookie, int status) +{ + struct verbs_txreq *tx = + container_of(cookie, struct verbs_txreq, txreq); + struct rvt_qp *qp = tx->qp; + enum ib_wc_status ib_status = + status == SDMA_TXREQ_S_OK ? IB_WC_SUCCESS : IB_WC_GENERAL_ERR; + unsigned long flags; + + spin_lock_irqsave(&qp->s_lock, flags); + if (tx->wqe) { + rvt_send_complete(qp, tx->wqe, ib_status); + } else if (qp->ibqp.qp_type == IB_QPT_RC) { + struct hfi2_opa_header *hdr; + + hdr = &tx->phdr.hdr; + if (unlikely(status != SDMA_TXREQ_S_OK)) + hfi2_rc_verbs_aborted(qp, hdr); + hfi2_rc_send_complete(qp, hdr); + } + spin_unlock_irqrestore(&qp->s_lock, flags); + + hfi2_put_txreq(tx); +} + +void hfi2_wait_kmem(struct rvt_qp *qp) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct ib_qp *ibqp = &qp->ibqp; + struct ib_device *ibdev = ibqp->device; + struct hfi2_ibdev *dev = to_idev(ibdev); + + if (list_empty(&priv->s_iowait.list)) { + if (list_empty(&dev->memwait)) + mod_timer(&dev->mem_timer, jiffies + 1); + qp->s_flags |= RVT_S_WAIT_KMEM; + list_add_tail(&priv->s_iowait.list, &dev->memwait); + priv->s_iowait.lock = &dev->iowait_lock; + trace_hfi2_qpsleep(qp, RVT_S_WAIT_KMEM); + rvt_get_qp(qp); + } +} + +/* + * This routine calls txadds for each sg entry. + * + * Add failures will revert the sge cursor + */ +static noinline int build_verbs_ulp_payload(struct sdma_engine *sde, u32 length, + struct verbs_txreq *tx) +{ + struct rvt_sge_state *ss = tx->ss; + struct rvt_sge *sg_list = ss->sg_list; + struct rvt_sge sge = ss->sge; + u8 num_sge = ss->num_sge; + u32 len; + int ret = 0; + + while (length) { + len = rvt_get_sge_length(&ss->sge, length); + WARN_ON_ONCE(len == 0); + ret = sdma_txadd_kvaddr(sde->dd, &tx->txreq, ss->sge.vaddr, + len); + if (ret) + goto bail_txadd; + rvt_update_sge(ss, len, false); + length -= len; + } + return ret; +bail_txadd: + /* unwind cursor */ + ss->sge = sge; + ss->num_sge = num_sge; + ss->sg_list = sg_list; + return ret; +} + +/** + * update_tx_opstats - record stats by opcode + * @qp: the qp + * @ps: transmit packet state + * @plen: the plen in dwords + * + * This is a routine to record the tx opstats after a + * packet has been presented to the egress mechanism. + */ +static void update_tx_opstats(struct rvt_qp *qp, struct hfi2_pkt_state *ps, + u32 plen) +{ +#ifdef CONFIG_DEBUG_FS + struct hfi2_devdata *dd = dd_from_ibdev(qp->ibqp.device); + struct hfi2_opcode_stats_perctx *s = get_cpu_ptr(dd->tx_opstats); + + inc_opstats(plen * 4, &s->stats[ps->opcode]); + put_cpu_ptr(s); +#endif +} + +/* + * Build the number of DMA descriptors needed to send length bytes of data. + * + * NOTE: DMA mapping is held in the tx until completed in the ring or + * the tx desc is freed without having been submitted to the ring + * + * This routine ensures all the helper routine calls succeed. + */ +/* New API */ +static int build_verbs_tx_desc(struct sdma_engine *sde, u32 length, + struct verbs_txreq *tx, + struct hfi2_ahg_info *ahg_info, u64 pbc) +{ + int ret = 0; + struct hfi2_sdma_header *phdr = &tx->phdr; + u16 hdrbytes = (tx->hdr_dwords + sizeof(pbc) / 4) << 2; + u8 extra_bytes = 0; + + if (tx->phdr.hdr.hdr_type) { + /* + * hdrbytes accounts for PBC. Need to subtract 8 bytes + * before calculating padding. + */ + if (sde->dd->params->chip_type == CHIP_WFR) { + extra_bytes = + hfi2_get_16b_padding(hdrbytes - 8, length) + + (SIZE_OF_CRC << 2) + SIZE_OF_LT; + } else { + /* add ICRC QW */ + extra_bytes = hfi2_pad8(hdrbytes - 8 + length) + 8; + } + } + if (!ahg_info->ahgcount) { + ret = sdma_txinit_ahg(sde->dd, &tx->txreq, ahg_info->tx_flags, + hdrbytes + length + extra_bytes, + ahg_info->ahgidx, 0, NULL, 0, + verbs_sdma_complete); + if (ret) + goto bail_txadd; + phdr->pbc = cpu_to_le64(pbc); + ret = sdma_txadd_kvaddr(sde->dd, &tx->txreq, phdr, hdrbytes); + if (ret) + goto bail_txadd; + } else { + ret = sdma_txinit_ahg(sde->dd, &tx->txreq, ahg_info->tx_flags, + length + extra_bytes, ahg_info->ahgidx, + ahg_info->ahgcount, ahg_info->ahgdesc, + hdrbytes, verbs_sdma_complete); + if (ret) + goto bail_txadd; + } + /* add the ulp payload - if any. tx->ss can be NULL for acks */ + if (tx->ss) { + ret = build_verbs_ulp_payload(sde, length, tx); + if (ret) + goto bail_txadd; + } + + /* add icrc, lt byte, and padding to flit */ + if (extra_bytes) + ret = sdma_txadd_daddr(sde->dd, &tx->txreq, + sde->dd->sdma_pad_phys, extra_bytes); + +bail_txadd: + return ret; +} + +static u64 update_hcrc(u8 opcode, u64 pbc) +{ + if ((opcode & IB_OPCODE_TID_RDMA) == IB_OPCODE_TID_RDMA) { + pbc &= ~PBC_INSERT_HCRC_SMASK; + pbc |= (u64)PBC_IHCRC_LKDETH << PBC_INSERT_HCRC_SHIFT; + } + return pbc; +} + +int hfi2_verbs_send_dma(struct rvt_qp *qp, struct hfi2_pkt_state *ps, u64 pbc) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_ahg_info *ahg_info = priv->s_ahg; + u32 hdrwords = ps->s_txreq->hdr_dwords; + u32 len = ps->s_txreq->s_cur_size; + u32 plen; + struct hfi2_pportdata *ppd = ps->ppd; + struct hfi2_devdata *dd = ppd->dd; + struct verbs_txreq *tx; + u8 sc5 = priv->s_sc; + int ret; + u32 dwords; + + if (ps->s_txreq->phdr.hdr.hdr_type) { + u8 extra_bytes; + + if (dd->params->chip_type == CHIP_WFR) { + extra_bytes = + hfi2_get_16b_padding((hdrwords << 2), len); + dwords = (len + extra_bytes + (SIZE_OF_CRC << 2) + + SIZE_OF_LT) >> + 2; + } else { + /* round up to multiple of 8 */ + extra_bytes = hfi2_pad8((hdrwords << 2) + len); + /* add ICRC QW */ + dwords = (len + extra_bytes + 8) >> 2; + } + } else { + dwords = (len + 3) >> 2; + } + plen = hdrwords + dwords + sizeof(pbc) / 4; + + tx = ps->s_txreq; + if (!sdma_txreq_built(&tx->txreq)) { + if (likely(pbc == 0)) { + u32 vl = sc_to_vlt(ppd, sc5); + u32 l2; + u32 dlid; + + /* No vl15 here */ + /* set PBC_DC_INFO bit (aka SC[4]) in pbc */ + if (ps->s_txreq->phdr.hdr.hdr_type) { + l2 = PBC_L2_16B; + dlid = hfi2_16B_get_dlid(&tx->phdr.hdr.opah); + } else { + pbc |= pbc_sc4_flag(sc5); + l2 = PBC_L2_9B; + dlid = ib_get_dlid(&tx->phdr.hdr.ibh); + } + + pbc = dd->params->create_pbc( + ppd, ps->hfi2_loopback, pbc, qp->srate_mbps, vl, + plen, l2, dlid, + priv->s_sendcontext->hw_context); + + if (unlikely(hfi2_dbg_should_fault_tx(qp, ps->opcode))) + pbc = hfi2_fault_tx(qp, ps->opcode, pbc); + else + /* Update HCRC based on packet opcode */ + pbc = update_hcrc(ps->opcode, pbc); + } + tx->wqe = qp->s_wqe; + ret = build_verbs_tx_desc(tx->sde, len, tx, ahg_info, pbc); + if (unlikely(ret)) { + /* + * Drop this transmit with an error. Expect that + * whatever caused the build error will occur again, + * so do not attempt to retry. If a retry is attempted, + * then tx would need to be cleaned first - tx may not + * be fully constructed. tx will be cleaned when + * deallocated. + */ + verbs_sdma_complete(&tx->txreq, SDMA_TXREQ_S_SENDERROR); + /* verbs_sdma_complete already freed tx via hfi2_put_txreq */ + ps->s_txreq = NULL; + /* return 0 so the next transmit is attempted */ + ret = 0; + goto bail_build; + } + } + /* + * hfi2_sdma_send_txreq() does not know about verbs_txreq and may transfer + * ownership of tx's ref. Take an extra reference here so that tx + * remains valid for the trace that follows the hfi2_sdma_send_txreq() call. + * + * Reference ownership will have been transferred if the return value + * is 0, -ECOMM, or -EIOCBQUEUED. + */ + hfi2_get_txreq(tx); + ret = hfi2_sdma_send_txreq(tx->sde, ps->wait, &tx->txreq, + ps->pkts_sent); + trace_sdma_output_ibhdr(dd, &ps->s_txreq->phdr.hdr, ib_is_sc5(sc5), + ret); + if (unlikely(ret < 0)) + goto bail_ecomm; + /* ret == 0 => the extra reference has been transferred */ + + update_tx_opstats(qp, ps, plen); + /* put ps's reference */ + hfi2_put_txreq(tx); + ps->s_txreq = NULL; + return 0; + +bail_ecomm: + if (ret == -EIOCBQUEUED) { + /* + * The SDMA engine descriptor queue is full. The txreq (and + * its reference) has been added to the wait list. + * + * Return an error so pending transmit attempts cease until + * the engine is ready. + */ + } else if (ret == -ECOMM) { + /* + * The SDMA engine has stopped. The txreq (and its reference) + * has been transferred to a flush list. + * + * Return 0 so all pending transmits also drop like this one. + */ + ret = 0; + } else { + /* + * An unexpected error has occurred. Complete the tx with an + * error. The complete will remove the reference. A call to + * sdma_txclean() will occur when the structure is deallocated. + * + * Return 0 so the next transmit is attempted. + */ + tx->txreq.complete(&tx->txreq, SDMA_TXREQ_S_SENDERROR); + ret = 0; + } +bail_build: + /* put ps's reference if not already freed */ + if (ps->s_txreq) { + hfi2_put_txreq(ps->s_txreq); + ps->s_txreq = NULL; + } + return ret; +} + +/* + * If we are now in the error state, return zero to flush the + * send work request. + */ +static int pio_wait(struct rvt_qp *qp, struct send_context *sc, + struct hfi2_pkt_state *ps, u32 flag) +{ + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_devdata *dd = sc->dd; + unsigned long flags; + int ret = 0; + + /* + * Note that as soon as want_buffer() is called and + * possibly before it returns, sc_piobufavail() + * could be called. Therefore, put QP on the I/O wait list before + * enabling the PIO avail interrupt. + */ + spin_lock_irqsave(&qp->s_lock, flags); + if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { + /* take a new txreq reference */ + hfi2_get_txreq(ps->s_txreq); + write_seqlock(&sc->waitlock); + list_add_tail(&ps->s_txreq->txreq.list, &ps->wait->tx_head); + if (list_empty(&priv->s_iowait.list)) { + struct hfi2_ibdev *dev = &dd->verbs_dev; + int was_empty; + + dev->n_piowait += !!(flag & RVT_S_WAIT_PIO); + dev->n_piodrain += !!(flag & HFI2_S_WAIT_PIO_DRAIN); + qp->s_flags |= flag; + was_empty = list_empty(&sc->piowait); + iowait_get_priority(&priv->s_iowait); + iowait_queue(ps->pkts_sent, &priv->s_iowait, + &sc->piowait); + priv->s_iowait.lock = &sc->waitlock; + trace_hfi2_qpsleep(qp, RVT_S_WAIT_PIO); + rvt_get_qp(qp); + /* counting: only call wantpiobuf_intr if first user */ + if (was_empty) + hfi2_sc_wantpiobuf_intr(sc, 1); + } + write_sequnlock(&sc->waitlock); + hfi2_qp_unbusy(qp, ps->wait); + ret = -EBUSY; + } + spin_unlock_irqrestore(&qp->s_lock, flags); + return ret; +} + +static void verbs_pio_complete(void *arg, int code) +{ + struct rvt_qp *qp = (struct rvt_qp *)arg; + struct hfi2_qp_priv *priv = qp->priv; + + if (iowait_pio_dec(&priv->s_iowait)) + iowait_drain_wakeup(&priv->s_iowait); +} + +static void hfi2_verbs_complete_tx(struct rvt_qp *qp, struct hfi2_pkt_state *ps, + int wc_status) +{ + unsigned long flags; + + spin_lock_irqsave(&qp->s_lock, flags); + if (qp->s_wqe) { + rvt_send_complete(qp, qp->s_wqe, wc_status); + } else if (qp->ibqp.qp_type == IB_QPT_RC) { + if (unlikely(wc_status != IB_WC_SUCCESS)) + hfi2_rc_verbs_aborted(qp, &ps->s_txreq->phdr.hdr); + hfi2_rc_send_complete(qp, &ps->s_txreq->phdr.hdr); + } + spin_unlock_irqrestore(&qp->s_lock, flags); +} + +int hfi2_verbs_send_pio(struct rvt_qp *qp, struct hfi2_pkt_state *ps, u64 pbc) +{ + struct hfi2_qp_priv *priv = qp->priv; + u32 hdrwords = ps->s_txreq->hdr_dwords; + struct rvt_sge_state *ss = ps->s_txreq->ss; + u32 len = ps->s_txreq->s_cur_size; + u32 dwords; + u32 plen; + struct hfi2_pportdata *ppd = ps->ppd; + struct hfi2_devdata *dd = ppd->dd; + u32 *hdr; + u8 sc5; + struct send_context *sc; + struct pio_buf *pbuf; + int wc_status = IB_WC_SUCCESS; + int ret = 0; + pio_release_cb cb = NULL; + u8 extra_bytes = 0; + + if (ps->s_txreq->phdr.hdr.hdr_type) { + if (dd->params->chip_type == CHIP_WFR) { + u8 pad_size = + hfi2_get_16b_padding((hdrwords << 2), len); + + extra_bytes = + pad_size + (SIZE_OF_CRC << 2) + SIZE_OF_LT; + } else { + /* round up to next multiple of 8, add ICRC */ + extra_bytes = hfi2_pad8((hdrwords << 2) + len) + 8; + } + dwords = (len + extra_bytes) >> 2; + hdr = (u32 *)&ps->s_txreq->phdr.hdr.opah; + } else { + dwords = (len + 3) >> 2; + hdr = (u32 *)&ps->s_txreq->phdr.hdr.ibh; + } + plen = hdrwords + dwords + sizeof(pbc) / 4; + + /* only RC/UC use complete */ + switch (qp->ibqp.qp_type) { + case IB_QPT_RC: + case IB_QPT_UC: + cb = verbs_pio_complete; + break; + default: + break; + } + + /* vl15 special case taken care of in ud.c */ + sc5 = priv->s_sc; + sc = ps->s_txreq->psc; + + if (likely(pbc == 0)) { + u8 vl = sc_to_vlt(ppd, sc5); + u32 l2; + u32 dlid; + + /* set PBC_DC_INFO bit (aka SC[4]) in pbc */ + if (ps->s_txreq->phdr.hdr.hdr_type) { + l2 = PBC_L2_16B; + dlid = hfi2_16B_get_dlid(&ps->s_txreq->phdr.hdr.opah); + } else { + pbc |= pbc_sc4_flag(sc5); + l2 = PBC_L2_9B; + dlid = ib_get_dlid(&ps->s_txreq->phdr.hdr.ibh); + } + + pbc = dd->params->create_pbc(ppd, ps->hfi2_loopback, pbc, + qp->srate_mbps, vl, plen, l2, dlid, + priv->s_sendcontext->hw_context); + if (unlikely(hfi2_dbg_should_fault_tx(qp, ps->opcode))) + pbc = hfi2_fault_tx(qp, ps->opcode, pbc); + else + /* Update HCRC based on packet opcode */ + pbc = update_hcrc(ps->opcode, pbc); + } + if (cb) + iowait_pio_inc(&priv->s_iowait); + pbuf = hfi2_sc_buffer_alloc(sc, plen, cb, qp); + if (IS_ERR_OR_NULL(pbuf)) { + if (cb) + verbs_pio_complete(qp, 0); + if (IS_ERR(pbuf)) { + /* + * If we have filled the PIO buffers to capacity and are + * not in an active state this request is not going to + * go out to so just complete it with an error or else a + * ULP or the core may be stuck waiting. + */ + hfi2_cdbg(PIO, "alloc failed with error, completing"); + wc_status = IB_WC_GENERAL_ERR; + goto pio_bail; + } + + /* + * This is a normal occurrence. The PIO buffers are all busy. + * Queue the tx request. + */ + ret = pio_wait(qp, sc, ps, RVT_S_WAIT_PIO); + if (ret == 0) { + /* not able to queue - complete tx with an error */ + wc_status = IB_WC_GENERAL_ERR; + hfi2_cdbg(PIO, + "alloc failed, unable to queue, completing"); + goto pio_bail; + } + /* tx is queued - return an error to stop processing */ + hfi2_cdbg(PIO, "alloc failed, queued"); + goto bail; + } + + if (dwords == 0) { + hfi2_pio_copy(dd, pbuf, pbc, hdr, hdrwords); + } else { + hfi2_seg_pio_copy_start(pbuf, pbc, hdr, hdrwords * 4); + if (ss) { + while (len) { + void *addr = ss->sge.vaddr; + u32 slen = rvt_get_sge_length(&ss->sge, len); + + rvt_update_sge(ss, slen, false); + hfi2_seg_pio_copy_mid(pbuf, addr, slen); + len -= slen; + } + } + /* add icrc, lt byte, and padding to flit */ + if (extra_bytes) + hfi2_seg_pio_copy_mid(pbuf, dd->sdma_pad_dma, + extra_bytes); + + hfi2_seg_pio_copy_end(pbuf); + } + + update_tx_opstats(qp, ps, plen); + trace_pio_output_ibhdr(dd, &ps->s_txreq->phdr.hdr, ib_is_sc5(sc5), 0); + +pio_bail: + hfi2_verbs_complete_tx(qp, ps, wc_status); + ret = 0; + +bail: + /* release ps hold on s_txreq */ + hfi2_put_txreq(ps->s_txreq); + ps->s_txreq = NULL; + return ret; +} + +/* + * egress_pkey_matches_entry - return 1 if the pkey matches ent (ent + * being an entry from the partition key table), return 0 + * otherwise. Use the matching criteria for egress partition keys + * specified in the OPAv1 spec., section 9.1l.7. + */ +static inline int egress_pkey_matches_entry(u16 pkey, u16 ent) +{ + u16 mkey = pkey & PKEY_LOW_15_MASK; + u16 mentry = ent & PKEY_LOW_15_MASK; + + if (mkey == mentry) { + /* + * If pkey[15] is set (full partition member), + * is bit 15 in the corresponding table element + * clear (limited member)? + */ + if (pkey & PKEY_MEMBER_MASK) + return !!(ent & PKEY_MEMBER_MASK); + return 1; + } + return 0; +} + +/** + * hfi2_egress_pkey_check - check P_KEY of a packet + * @ppd: Physical IB port data + * @slid: SLID for packet + * @pkey: PKEY for header + * @sc5: SC for packet + * @s_pkey_index: It will be used for look up optimization for kernel contexts + * only. If it is negative value, then it means user contexts is calling this + * function. + * + * It checks if hdr's pkey is valid. + * + * Return: 0 on success, otherwise, 1 + */ +int hfi2_egress_pkey_check(struct hfi2_pportdata *ppd, u32 slid, u16 pkey, + u8 sc5, int8_t s_pkey_index) +{ + struct hfi2_devdata *dd = ppd->dd; + int i; + int is_user_ctxt_mechanism = (s_pkey_index < 0); + + if (!(ppd->part_enforce & HFI2_PART_ENFORCE_OUT)) + return 0; + + /* If SC15, pkey[0:14] must be 0x7fff */ + if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK)) + goto bad; + + /* Is the pkey = 0x0, or 0x8000? */ + if ((pkey & PKEY_LOW_15_MASK) == 0) + goto bad; + + /* + * For the kernel contexts only, if a qp is passed into the function, + * the most likely matching pkey has index qp->s_pkey_index + */ + if (!is_user_ctxt_mechanism && + egress_pkey_matches_entry(pkey, ppd->pkeys[s_pkey_index])) { + return 0; + } + + for (i = 0; i < dd->params->pkey_table_size; i++) { + if (egress_pkey_matches_entry(pkey, ppd->pkeys[i])) + return 0; + } +bad: + /* + * For the user-context mechanism, the P_KEY check would only happen + * once per SDMA request, not once per packet. Therefore, there's no + * need to increment the counter for the user-context mechanism. + */ + if (!is_user_ctxt_mechanism) { + incr_cntr64(&ppd->port_xmit_constraint_errors); + if (!(dd->err_info_xmit_constraint.status & + OPA_EI_STATUS_SMASK)) { + dd->err_info_xmit_constraint.status |= + OPA_EI_STATUS_SMASK; + dd->err_info_xmit_constraint.slid = slid; + dd->err_info_xmit_constraint.pkey = pkey; + } + } + return 1; +} + +/* + * get_send_routine - choose an egress routine + * + * Choose an egress routine based on QP type + * and size + */ +static inline send_routine get_send_routine(struct rvt_qp *qp, + struct hfi2_pkt_state *ps) +{ + struct hfi2_devdata *dd = dd_from_ibdev(qp->ibqp.device); + struct hfi2_qp_priv *priv = qp->priv; + struct verbs_txreq *tx = ps->s_txreq; + + if (unlikely(!(dd->flags & HFI2_HAS_SEND_DMA))) + return dd->process_pio_send; + switch (qp->ibqp.qp_type) { + case IB_QPT_SMI: + return dd->process_pio_send; + case IB_QPT_GSI: + case IB_QPT_UD: + break; + case IB_QPT_UC: + case IB_QPT_RC: + priv->s_running_pkt_size = + (tx->s_cur_size + priv->s_running_pkt_size) / 2; + if (hfi2_piothreshold && + priv->s_running_pkt_size <= + min(hfi2_piothreshold, qp->pmtu) && + (BIT(ps->opcode & OPMASK) & pio_opmask[ps->opcode >> 5]) && + iowait_sdma_pending(&priv->s_iowait) == 0 && + !sdma_txreq_built(&tx->txreq)) + return dd->process_pio_send; + break; + default: + break; + } + return dd->process_dma_send; +} + +/** + * hfi2_verbs_send - send a packet + * @qp: the QP to send on + * @ps: the state of the packet to send + * + * Return zero if packet is sent or queued OK. + * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise. + */ +int hfi2_verbs_send(struct rvt_qp *qp, struct hfi2_pkt_state *ps) +{ + struct hfi2_devdata *dd = dd_from_ibdev(qp->ibqp.device); + struct hfi2_qp_priv *priv = qp->priv; + struct hfi2_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct ib_other_headers *ohdr = NULL; + send_routine sr; + int ret; + u16 pkey; + u32 slid; + u8 l4 = 0; + + /* locate the pkey within the headers */ + if (ps->s_txreq->phdr.hdr.hdr_type) { + struct hfi2_16b_header *hdr = &ps->s_txreq->phdr.hdr.opah; + + l4 = hfi2_16B_get_l4(hdr); + if (l4 == OPA_16B_L4_IB_LOCAL) + ohdr = &hdr->u.oth; + else if (l4 == OPA_16B_L4_IB_GLOBAL) + ohdr = &hdr->u.l.oth; + + slid = hfi2_16B_get_slid(hdr); + pkey = hfi2_16B_get_pkey(hdr); + } else { + struct ib_header *hdr = &ps->s_txreq->phdr.hdr.ibh; + u8 lnh = ib_get_lnh(hdr); + + if (lnh == HFI2_LRH_GRH) + ohdr = &hdr->u.l.oth; + else + ohdr = &hdr->u.oth; + slid = ib_get_slid(hdr); + pkey = ib_bth_get_pkey(ohdr); + } + + if (likely(l4 != OPA_16B_L4_FM)) + ps->opcode = ib_bth_get_opcode(ohdr); + else + ps->opcode = IB_OPCODE_UD_SEND_ONLY; + + sr = get_send_routine(qp, ps); + ret = hfi2_egress_pkey_check(ppd, slid, pkey, priv->s_sc, + qp->s_pkey_index); + if (unlikely(ret)) { + /* + * The value we are returning here does not get propagated to + * the verbs caller. Thus we need to complete the request with + * error otherwise the caller could be sitting waiting on the + * completion event. Only do this for PIO. SDMA has its own + * mechanism for handling the errors. So for SDMA we can just + * return. + */ + if (sr == dd->process_pio_send) { + unsigned long flags; + + hfi2_cdbg(PIO, "%s() Failed. Completing with err", + __func__); + spin_lock_irqsave(&qp->s_lock, flags); + rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); + spin_unlock_irqrestore(&qp->s_lock, flags); + } else { + hfi2_put_txreq(ps->s_txreq); + ps->s_txreq = NULL; + } + return -EINVAL; + } + if (sr == dd->process_dma_send && iowait_pio_pending(&priv->s_iowait)) { + ret = pio_wait(qp, ps->s_txreq->psc, ps, HFI2_S_WAIT_PIO_DRAIN); + if (ret == 0) { + /* not able to queue - complete tx with an error */ + hfi2_verbs_complete_tx(qp, ps, IB_WC_GENERAL_ERR); + } + /* queued or not, ps is done with s_txreq */ + hfi2_put_txreq(ps->s_txreq); + ps->s_txreq = NULL; + return ret; + } + return sr(qp, ps, 0); +} + +/** + * hfi2_fill_device_attr - Fill in rvt dev info device attributes. + * @dd: the device data structure + */ +static void hfi2_fill_device_attr(struct hfi2_devdata *dd) +{ + struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; + u32 ver = dd->dc8051_ver; + + memset(&rdi->dparms.props, 0, sizeof(rdi->dparms.props)); + + if (!dd->cport) { + rdi->dparms.props.fw_ver = ((u64)(dc8051_ver_maj(ver)) << 32) | + ((u64)(dc8051_ver_min(ver)) << 16) | + (u64)dc8051_ver_patch(ver); + } else { + rdi->dparms.props.fw_ver = dd->cport_ver; + } + rdi->dparms.props.device_cap_flags = + IB_DEVICE_BAD_PKEY_CNTR | IB_DEVICE_BAD_QKEY_CNTR | + IB_DEVICE_SHUTDOWN_PORT | IB_DEVICE_SYS_IMAGE_GUID | + IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_PORT_ACTIVE_EVENT | + IB_DEVICE_SRQ_RESIZE | IB_DEVICE_MEM_MGT_EXTENSIONS; + rdi->dparms.props.kernel_cap_flags = IBK_RDMA_NETDEV_OPA; + rdi->dparms.props.page_size_cap = PAGE_SIZE; + rdi->dparms.props.vendor_id = dd->oui1 << 16 | dd->oui2 << 8 | dd->oui3; + rdi->dparms.props.vendor_part_id = dd->pcidev->device; + rdi->dparms.props.hw_ver = dd->minrev; + rdi->dparms.props.sys_image_guid = hfi2_ib_hfi2_sys_image_guid; + rdi->dparms.props.max_mr_size = U64_MAX; + rdi->dparms.props.max_fast_reg_page_list_len = UINT_MAX; + rdi->dparms.props.max_qp = hfi2_max_qps; + rdi->dparms.props.max_qp_wr = (hfi2_max_qp_wrs >= HFI2_QP_WQE_INVALID ? + HFI2_QP_WQE_INVALID - 1 : + hfi2_max_qp_wrs); + rdi->dparms.props.max_send_sge = hfi2_max_sges; + rdi->dparms.props.max_recv_sge = hfi2_max_sges; + rdi->dparms.props.max_sge_rd = hfi2_max_sges; + rdi->dparms.props.max_cq = hfi2_max_cqs; + rdi->dparms.props.max_ah = hfi2_max_ahs; + rdi->dparms.props.max_cqe = hfi2_max_cqes; + rdi->dparms.props.max_pd = hfi2_max_pds; + rdi->dparms.props.max_qp_rd_atom = HFI2_MAX_RDMA_ATOMIC; + rdi->dparms.props.max_qp_init_rd_atom = 255; + rdi->dparms.props.max_srq = hfi2_max_srqs; + rdi->dparms.props.max_srq_wr = hfi2_max_srq_wrs; + rdi->dparms.props.max_srq_sge = hfi2_max_srq_sges; + rdi->dparms.props.atomic_cap = IB_ATOMIC_GLOB; + rdi->dparms.props.max_pkeys = hfi2_get_npkeys(dd); + rdi->dparms.props.max_mcast_grp = hfi2_max_mcast_grps; + rdi->dparms.props.max_mcast_qp_attach = hfi2_max_mcast_qp_attached; + rdi->dparms.props.max_total_mcast_qp_attach = + rdi->dparms.props.max_mcast_qp_attach * + rdi->dparms.props.max_mcast_grp; +} + +static inline u16 opa_speed_to_ib(u16 in) +{ + u16 out = 0; + + if (in & OPA_LINK_SPEED_100G) + out |= IB_SPEED_NDR; + if (in & OPA_LINK_SPEED_50G) + out |= IB_SPEED_HDR; + if (in & OPA_LINK_SPEED_25G) + out |= IB_SPEED_EDR; + if (in & OPA_LINK_SPEED_12_5G) + out |= IB_SPEED_FDR; + + return out; +} + +/* + * Convert a single OPA link width (no multiple flags) to an IB value. + * A zero OPA link width means link down, which means the IB width value + * is a don't care. + */ +static inline u16 opa_width_to_ib(u16 in) +{ + switch (in) { + case OPA_LINK_WIDTH_1X: + /* map 2x and 3x to 1x as they don't exist in IB */ + case OPA_LINK_WIDTH_2X: + case OPA_LINK_WIDTH_3X: + return IB_WIDTH_1X; + default: /* link down or unknown, return our largest width */ + case OPA_LINK_WIDTH_4X: + return IB_WIDTH_4X; + } +} + +static int query_port(struct rvt_dev_info *rdi, u32 port_num, + struct ib_port_attr *props) +{ + struct hfi2_ibdev *verbs_dev = dev_from_rdi(rdi); + struct hfi2_devdata *dd = dd_from_dev(verbs_dev); + struct hfi2_pportdata *ppd = &dd->pport[port_num - 1]; + u32 lid = ppd->lid; + + /* props being zeroed by the caller, avoid zeroing it here */ + props->lid = lid ? lid : 0; + props->lmc = ppd->lmc; + /* OPA logical states match IB logical states */ + props->state = hfi2_driver_lstate(ppd); + props->phys_state = hfi2_driver_pstate(ppd); + props->gid_tbl_len = HFI2_GUIDS_PER_PORT; + props->active_width = (u8)opa_width_to_ib(ppd->link_width_active); + /* see rate_show() in ib core/sysfs.c */ + props->active_speed = opa_speed_to_ib(ppd->link_speed_active); + props->max_vl_num = ppd->vls_supported; + + /* Once we are a "first class" citizen and have added the OPA MTUs to + * the core we can advertise the larger MTU enum to the ULPs, for now + * advertise only 4K. + * + * Those applications which are either OPA aware or pass the MTU enum + * from the Path Records to us will get the new 8k MTU. Those that + * attempt to process the MTU enum may fail in various ways. + */ + props->max_mtu = hfi2_mtu_to_enum( + (!valid_ib_mtu(hfi2_max_mtu) ? 4096 : hfi2_max_mtu), + IB_MTU_4096); + props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? + props->max_mtu : + hfi2_mtu_to_enum(ppd->ibmtu, IB_MTU_4096); + props->phys_mtu = hfi2_max_mtu; + + return 0; +} + +static int modify_device(struct ib_device *device, int device_modify_mask, + struct ib_device_modify *device_modify) +{ + struct hfi2_devdata *dd = dd_from_ibdev(device); + unsigned int i; + int ret; + + if (device_modify_mask & + ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID | IB_DEVICE_MODIFY_NODE_DESC)) { + ret = -EOPNOTSUPP; + goto bail; + } + + if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) { + memcpy(device->node_desc, device_modify->node_desc, + IB_DEVICE_NODE_DESC_MAX); + for (i = 0; i < dd->num_pports; i++) { + struct hfi2_ibport *ibp = &dd->pport[i].ibport_data; + + hfi2_node_desc_chg(ibp); + } + } + + if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) { + hfi2_ib_hfi2_sys_image_guid = + cpu_to_be64(device_modify->sys_image_guid); + for (i = 0; i < dd->num_pports; i++) { + struct hfi2_ibport *ibp = &dd->pport[i].ibport_data; + + hfi2_sys_guid_chg(ibp); + } + } + + ret = 0; + +bail: + return ret; +} + +static int shut_down_port(struct rvt_dev_info *rdi, u32 port_num) +{ + struct hfi2_ibdev *verbs_dev = dev_from_rdi(rdi); + struct hfi2_devdata *dd = dd_from_dev(verbs_dev); + struct hfi2_pportdata *ppd = &dd->pport[port_num - 1]; + + hfi2_set_link_down_reason(ppd, OPA_LINKDOWN_REASON_UNKNOWN, 0, + OPA_LINKDOWN_REASON_UNKNOWN); + return hfi2_set_link_state(ppd, HLS_DN_DOWNDEF); +} + +static int hfi2_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp, + int guid_index, __be64 *guid) +{ + struct hfi2_ibport *ibp = container_of(rvp, struct hfi2_ibport, rvp); + + if (guid_index >= HFI2_GUIDS_PER_PORT) + return -EINVAL; + + *guid = get_sguid(ibp, guid_index); + return 0; +} + +/* + * convert ah port,sl to sc + */ +u8 hfi2_ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah) +{ + struct hfi2_ibport *ibp = to_iport(ibdev, rdma_ah_get_port_num(ah)); + + return ibp->sl_to_sc[rdma_ah_get_sl(ah)]; +} + +static int hfi2_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr) +{ + struct hfi2_ibport *ibp; + struct hfi2_pportdata *ppd; + u8 sc5; + u8 sl; + u8 vl; + + if (hfi2_check_mcast(rdma_ah_get_dlid(ah_attr)) && + !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) + return -EINVAL; + + /* test the mapping for validity */ + ibp = to_iport(ibdev, rdma_ah_get_port_num(ah_attr)); + ppd = ppd_from_ibp(ibp); + + sl = rdma_ah_get_sl(ah_attr); + if (sl >= ARRAY_SIZE(ibp->sl_to_sc)) + return -EINVAL; + sl = array_index_nospec(sl, ARRAY_SIZE(ibp->sl_to_sc)); + + sc5 = ibp->sl_to_sc[sl]; + vl = sc_to_vlt(ppd, sc5); + if (vl > hfi2_num_vls && vl != 0xf) + return -EINVAL; + return 0; +} + +static void hfi2_notify_new_ah(struct ib_device *ibdev, + struct rdma_ah_attr *ah_attr, struct rvt_ah *ah) +{ + struct hfi2_ibport *ibp; + struct hfi2_pportdata *ppd; + u8 sc5; + struct rdma_ah_attr *attr = &ah->attr; + + /* + * Do not trust reading anything from rvt_ah at this point as it is not + * done being setup. We can however modify things which we need to set. + */ + + ibp = to_iport(ibdev, rdma_ah_get_port_num(ah_attr)); + ppd = ppd_from_ibp(ibp); + sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)]; + hfi2_update_ah_attr(ibdev, attr); + hfi2_make_opa_lid(attr); + ah->vl = sc_to_vlt(ppd, sc5); + if (ah->vl < hfi2_num_vls || ah->vl == 15) + ah->log_pmtu = ilog2(ppd->vld[ah->vl].mtu); +} + +/** + * hfi2_get_npkeys - return the size of the PKEY table for context 0 + * @dd: the hfi2_ib device + */ +unsigned int hfi2_get_npkeys(struct hfi2_devdata *dd) +{ + return dd->params->pkey_table_size; +} + +static void init_ibport(struct hfi2_pportdata *ppd) +{ + struct hfi2_ibport *ibp = &ppd->ibport_data; + size_t sz = ARRAY_SIZE(ibp->sl_to_sc); + int i; + + for (i = 0; i < sz; i++) { + ibp->sl_to_sc[i] = i; + ibp->sc_to_sl[i] = i; + } + + for (i = 0; i < RVT_MAX_TRAP_LISTS; i++) + INIT_LIST_HEAD(&ibp->rvp.trap_lists[i].list); + timer_setup(&ibp->rvp.trap_timer, hfi2_handle_trap_timer, 0); + + spin_lock_init(&ibp->rvp.lock); + /* Set the prefix to the default value (see ch. 4.1.1) */ + ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX; + ibp->rvp.sm_lid = 0; + /* + * Below should only set bits defined in OPA PortInfo.CapabilityMask + * and PortInfo.CapabilityMask3 + */ + ibp->rvp.port_cap_flags = IB_PORT_AUTO_MIGR_SUP | + IB_PORT_CAP_MASK_NOTICE_SUP; + ibp->rvp.port_cap3_flags = OPA_CAP_MASK3_IsSharedSpaceSupported; + ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA; + ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA; + ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS; + ibp->rvp.pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS; + ibp->rvp.pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT; + + RCU_INIT_POINTER(ibp->rvp.qp[0], NULL); + RCU_INIT_POINTER(ibp->rvp.qp[1], NULL); +} + +/* ib_device_ops->get_dev_fw_str for wfr */ +static void dc8051_get_dev_fw_str(struct ib_device *ibdev, char *str) +{ + struct rvt_dev_info *rdi = ib_to_rvt(ibdev); + struct hfi2_ibdev *dev = dev_from_rdi(rdi); + u32 ver = dd_from_dev(dev)->dc8051_ver; + + snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u", dc8051_ver_maj(ver), + dc8051_ver_min(ver), dc8051_ver_patch(ver)); +} + +/* + * CPORT FW has defined the following mapping for WHO->VERSION_PATCH[4:7] + * aka quality field + */ +static const char *const cport_ver_qlt_to_str_map[] = { + "P0", "P1", "P2", /* 0-2: PowerOn[0-2] */ + "A0", "A1", "A2", /* 3-5: Alpha[0-2] */ + "B0", "B1", "B2", /* 6-8: Beta[0-2] */ + "RC0", "RC1", "RC2", + "RC3", "RC4" /* 9-13: RC[0-4] */ + /* 14: use patch field */ + /* 15: Reserved */ +}; + +/* ib_device_ops->get_dev_fw_str for jkr and beyond */ +void hfi2_cport_get_dev_fw_str(struct ib_device *ibdev, char *str) +{ + struct rvt_dev_info *rdi = ib_to_rvt(ibdev); + struct hfi2_ibdev *dev = dev_from_rdi(rdi); + u64 ver = dd_from_dev(dev)->cport_ver; + u32 qlt = CPORT_FW_VER_QLT(ver); + + /* Does quality map to a special substring or do we just use patch */ + if (qlt < ARRAY_SIZE(cport_ver_qlt_to_str_map) && ver != 0) + snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u.%s.%u", + CPORT_FW_VER_MAJ(ver), CPORT_FW_VER_MIN(ver), + CPORT_FW_VER_MNT(ver), cport_ver_qlt_to_str_map[qlt], + CPORT_FW_VER_BLD(ver)); + else + snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u.%u.%u", + CPORT_FW_VER_MAJ(ver), CPORT_FW_VER_MIN(ver), + CPORT_FW_VER_MNT(ver), CPORT_FW_VER_PAT(ver), + CPORT_FW_VER_BLD(ver)); +} + +static const char *const driver_cntr_names[] = { + /* must be element 0*/ + "DRIVER_KernIntr", "DRIVER_ErrorIntr", "DRIVER_Tx_Errs", + "DRIVER_Rcv_Errs", "DRIVER_HW_Errs", "DRIVER_NoPIOBufs", + "DRIVER_CtxtsOpen", "DRIVER_RcvLen_Errs", "DRIVER_EgrBufFull", + "DRIVER_EgrHdrFull" +}; + +int hfi2_num_driver_cntrs = ARRAY_SIZE(driver_cntr_names); + +/* + * Convert a list of names separated by '\n' into an array of NULL terminated + * strings. Optionally some entries can be reserved in the array to hold extra + * external strings. + */ +static int init_cntr_names(const char *names_in, const size_t names_len, + int num_extra_names, int *num_cntrs, + struct rdma_stat_desc **cntr_descs) +{ + struct rdma_stat_desc *names_out; + char *p; + int i, n; + + n = 0; + for (i = 0; i < names_len; i++) + if (names_in[i] == '\n') + n++; + + names_out = + kzalloc((n + num_extra_names) * sizeof(*names_out) + names_len, + GFP_KERNEL); + if (!names_out) { + *num_cntrs = 0; + *cntr_descs = NULL; + return -ENOMEM; + } + + p = (char *)&names_out[n + num_extra_names]; + memcpy(p, names_in, names_len); + + for (i = 0; i < n; i++) { + names_out[i].name = p; + p = strchr(p, '\n'); + *p++ = '\0'; + } + + *num_cntrs = n; + *cntr_descs = names_out; + return 0; +} + +static struct rdma_hw_stats *hfi2_alloc_hw_device_stats(struct ib_device *ibdev) +{ + struct hfi2_devdata *dd = dd_from_ibdev(ibdev); + + return rdma_alloc_hw_stats_struct( + dd->dev_cntr_descs, dd->num_dev_cntrs + hfi2_num_driver_cntrs, + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} + +static struct rdma_hw_stats *hfi_alloc_hw_port_stats(struct ib_device *ibdev, + u32 port_num) +{ + struct hfi2_devdata *dd = dd_from_ibdev(ibdev); + + return rdma_alloc_hw_stats_struct(dd->port_cntr_descs, + dd->num_port_cntrs, + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} + +static u64 hfi2_sps_ints(void) +{ + unsigned long index, flags; + struct hfi2_devdata *dd; + u64 sps_ints = 0; + + xa_lock_irqsave(&hfi2_dev_table, flags); + xa_for_each(&hfi2_dev_table, index, dd) { + sps_ints += hfi2_get_all_cpu_total(dd->int_counter); + } + xa_unlock_irqrestore(&hfi2_dev_table, flags); + return sps_ints; +} + +static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, + u32 port, int index) +{ + struct hfi2_devdata *dd = dd_from_ibdev(ibdev); + u64 *values; + int count; + + if (!port) { + u64 *stats = (u64 *)&hfi2_stats; + int i; + + hfi2_read_cntrs(dd, NULL, &values); + values[dd->num_dev_cntrs] = hfi2_sps_ints(); + for (i = 1; i < hfi2_num_driver_cntrs; i++) + values[dd->num_dev_cntrs + i] = stats[i]; + count = dd->num_dev_cntrs + hfi2_num_driver_cntrs; + } else { + struct hfi2_ibport *ibp = to_iport(ibdev, port); + + hfi2_read_portcntrs(ppd_from_ibp(ibp), NULL, &values); + count = dd->num_port_cntrs; + } + + memcpy(stats->value, values, count * sizeof(u64)); + return count; +} + +static const struct ib_device_ops hfi2_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HFI2, + + .alloc_hw_device_stats = hfi2_alloc_hw_device_stats, + .alloc_hw_port_stats = hfi_alloc_hw_port_stats, + .device_group = &hfi2_ib_hfi2_attr_group, + .get_dev_fw_str = dc8051_get_dev_fw_str, + .get_hw_stats = get_hw_stats, + .modify_device = modify_device, + .port_groups = hfi2_wfr_attr_port_groups, + /* keep process mad in the driver */ + .process_mad = hfi2_process_mad, + .rdma_netdev_get_params = hfi2_ipoib_rn_get_params, + .write_iter = hfi2_uverbs_write_iter, +}; + +static const struct ib_device_ops cport_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HFI2, + + .alloc_hw_device_stats = hfi2_alloc_hw_device_stats, + .alloc_hw_port_stats = hfi_alloc_hw_port_stats, + .device_group = &hfi2_ib_hfi2_attr_group, + .get_dev_fw_str = hfi2_cport_get_dev_fw_str, + .get_hw_stats = get_hw_stats, + .modify_device = modify_device, + .port_groups = hfi2_cport_attr_port_groups, + /* keep process mad in the driver */ + .process_mad = hfi2_cport_process_mad, + .rdma_netdev_get_params = hfi2_ipoib_rn_get_params, + .write_iter = hfi2_uverbs_write_iter, +}; + +static const struct ib_device_ops vf_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HFI2, + + .alloc_hw_device_stats = hfi2_alloc_hw_device_stats, + .alloc_hw_port_stats = hfi_alloc_hw_port_stats, + .device_group = &hfi2_ib_hfi2_attr_group, + .get_dev_fw_str = hfi2_cport_get_dev_fw_str, + .get_hw_stats = get_hw_stats, + .modify_device = modify_device, + .port_groups = hfi2_cport_attr_port_groups, + /* keep process mad in the driver */ + .process_mad = hfi2_vf_process_mad, + .rdma_netdev_get_params = hfi2_ipoib_rn_get_params, +}; + +/** + * hfi2_register_ib_device - register our device with the infiniband core + * @dd: the device data structure + * Return 0 if successful, errno if unsuccessful. + */ +int hfi2_register_ib_device(struct hfi2_devdata *dd) +{ + struct hfi2_ibdev *dev = &dd->verbs_dev; + struct ib_device *ibdev = &dev->rdi.ibdev; + struct hfi2_pportdata *ppd; + unsigned int i; + int ret; + u8 max_qos_shift; + + ret = init_cntr_names(dd->cntrnames, dd->cntrnameslen, + hfi2_num_driver_cntrs, &dd->num_dev_cntrs, + &dd->dev_cntr_descs); + if (ret) + goto err_cntr_descs; + for (i = 0; i < hfi2_num_driver_cntrs; i++) + dd->dev_cntr_descs[dd->num_dev_cntrs + i].name = + driver_cntr_names[i]; + ret = init_cntr_names(dd->portcntrnames, dd->portcntrnameslen, 0, + &dd->num_port_cntrs, &dd->port_cntr_descs); + if (ret) + goto err_cntr_descs; + + /* rdmavt has only a single QPN space - use the largest QOS shift */ + max_qos_shift = 1; /* always shift by at least 1 */ + for (i = 0, ppd = dd->pport; i < dd->num_pports; i++, ppd++) { + init_ibport(ppd); + max_qos_shift = max_t(u8, max_qos_shift, ppd->qos_shift); + } + + /* Only need to initialize non-zero fields. */ + + timer_setup(&dev->mem_timer, mem_timer, 0); + + seqlock_init(&dev->iowait_lock); + seqlock_init(&dev->txwait_lock); + INIT_LIST_HEAD(&dev->txwait); + INIT_LIST_HEAD(&dev->memwait); + + ret = hfi2_verbs_txreq_init(dev); + if (ret) + goto err_verbs_txreq; + + /* Use first-port GUID as node guid */ + ibdev->node_guid = get_sguid(&dd->pport[HFI2_PORT_IDX].ibport_data, + HFI2_PORT_GUID_INDEX); + + /* + * The system image GUID is supposed to be the same for all + * HFIs in a single system but since there can be other + * device types in the system, we can't be sure this is unique. + */ + if (!hfi2_ib_hfi2_sys_image_guid) + hfi2_ib_hfi2_sys_image_guid = ibdev->node_guid; + ibdev->phys_port_cnt = dd->num_pports; + ibdev->dev.parent = &dd->pcidev->dev; + + if (dd->cport) + ib_set_device_ops(ibdev, &cport_dev_ops); + else if (dd->is_vf) + ib_set_device_ops(ibdev, &vf_dev_ops); + else + ib_set_device_ops(ibdev, &hfi2_dev_ops); + + strscpy(ibdev->node_desc, init_utsname()->nodename, + sizeof(ibdev->node_desc)); + + if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) + ibdev->driver_def = hfi2_ib_defs; + + /* + * Fill in rvt info object. + */ + dd->verbs_dev.rdi.driver_f.get_pci_dev = hfi2_get_pci_dev; + dd->verbs_dev.rdi.driver_f.check_ah = hfi2_check_ah; + dd->verbs_dev.rdi.driver_f.notify_new_ah = hfi2_notify_new_ah; + dd->verbs_dev.rdi.driver_f.get_guid_be = hfi2_get_guid_be; + dd->verbs_dev.rdi.driver_f.query_port_state = query_port; + dd->verbs_dev.rdi.driver_f.shut_down_port = shut_down_port; + dd->verbs_dev.rdi.driver_f.cap_mask_chg = hfi2_cap_mask_chg; + /* + * Fill in rvt info device attributes. + */ + hfi2_fill_device_attr(dd); + + /* queue pair */ + dd->verbs_dev.rdi.dparms.qp_table_size = hfi2_qp_table_size; + dd->verbs_dev.rdi.dparms.qpn_start = 0; + dd->verbs_dev.rdi.dparms.qpn_inc = 1; + dd->verbs_dev.rdi.dparms.qos_shift = max_qos_shift; + dd->verbs_dev.rdi.dparms.qpn_res_start = RVT_KDETH_QP_BASE; + dd->verbs_dev.rdi.dparms.qpn_res_end = RVT_AIP_QP_MAX; + dd->verbs_dev.rdi.dparms.max_rdma_atomic = HFI2_MAX_RDMA_ATOMIC; + dd->verbs_dev.rdi.dparms.psn_mask = PSN_MASK; + dd->verbs_dev.rdi.dparms.psn_shift = PSN_SHIFT; + dd->verbs_dev.rdi.dparms.psn_modify_mask = PSN_MODIFY_MASK; + dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_INTEL_OPA | + RDMA_CORE_CAP_OPA_AH; + dd->verbs_dev.rdi.dparms.max_mad_size = OPA_MGMT_MAD_SIZE; + + if (dd->is_sriov) { + dd->verbs_dev.rdi.driver_f.alloc_qpn = hfi2_sriov_alloc_qpn; + dd->verbs_dev.rdi.dparms.qpn_start = + (dd->rsrcs.c.first_rcv_context << 1) - + (1 << max_qos_shift); + } + dd->verbs_dev.rdi.driver_f.qp_priv_alloc = qp_priv_alloc; + dd->verbs_dev.rdi.driver_f.qp_priv_init = hfi2_qp_priv_init; + dd->verbs_dev.rdi.driver_f.qp_priv_free = qp_priv_free; + dd->verbs_dev.rdi.driver_f.free_all_qps = hfi2_free_all_qps; + dd->verbs_dev.rdi.driver_f.notify_qp_reset = hfi2_notify_qp_reset; + dd->verbs_dev.rdi.driver_f.do_send = hfi2_do_send_from_rvt; + dd->verbs_dev.rdi.driver_f.schedule_send = hfi2_schedule_send; + dd->verbs_dev.rdi.driver_f.schedule_send_no_lock = _hfi2_schedule_send; + dd->verbs_dev.rdi.driver_f.get_pmtu_from_attr = hfi2_get_pmtu_from_attr; + dd->verbs_dev.rdi.driver_f.notify_error_qp = hfi2_notify_error_qp; + dd->verbs_dev.rdi.driver_f.flush_qp_waiters = hfi2_flush_qp_waiters; + dd->verbs_dev.rdi.driver_f.stop_send_queue = hfi2_stop_send_queue; + dd->verbs_dev.rdi.driver_f.quiesce_qp = hfi2_quiesce_qp; + dd->verbs_dev.rdi.driver_f.notify_error_qp = hfi2_notify_error_qp; + dd->verbs_dev.rdi.driver_f.mtu_from_qp = hfi2_mtu_from_qp; + dd->verbs_dev.rdi.driver_f.mtu_to_path_mtu = hfi2_mtu_to_path_mtu; + dd->verbs_dev.rdi.driver_f.check_modify_qp = hfi2_check_modify_qp; + dd->verbs_dev.rdi.driver_f.modify_qp = hfi2_modify_qp; + dd->verbs_dev.rdi.driver_f.notify_restart_rc = hfi2_restart_rc; + dd->verbs_dev.rdi.driver_f.setup_wqe = hfi2_setup_wqe; + dd->verbs_dev.rdi.driver_f.comp_vect_cpu_lookup = + hfi2_comp_vect_mappings_lookup; + dd->verbs_dev.rdi.driver_f.alloc_ucontext = hfi2_alloc_ucontext; + dd->verbs_dev.rdi.driver_f.dealloc_ucontext = hfi2_dealloc_ucontext; + dd->verbs_dev.rdi.driver_f.mmap = hfi2_rdma_mmap; + + /* completeion queue */ + dd->verbs_dev.rdi.ibdev.num_comp_vectors = dd->comp_vect_possible_cpus; + dd->verbs_dev.rdi.dparms.node = dd->node; + + /* misc settings */ + dd->verbs_dev.rdi.flags = 0; /* Let rdmavt handle it all */ + dd->verbs_dev.rdi.dparms.lkey_table_size = hfi2_lkey_table_size; + dd->verbs_dev.rdi.dparms.nports = dd->num_pports; + dd->verbs_dev.rdi.dparms.npkeys = hfi2_get_npkeys(dd); + dd->verbs_dev.rdi.dparms.sge_copy_mode = sge_copy_mode; + dd->verbs_dev.rdi.dparms.wss_threshold = wss_threshold; + dd->verbs_dev.rdi.dparms.wss_clean_period = wss_clean_period; + dd->verbs_dev.rdi.dparms.reserved_operations = 1; + dd->verbs_dev.rdi.dparms.extra_rdma_atomic = HFI2_TID_RDMA_WRITE_CNT; + + /* post send table */ + dd->verbs_dev.rdi.post_parms = hfi2_post_parms; + + /* opcode translation table */ + dd->verbs_dev.rdi.wc_opcode = hfi2_ib_hfi2_wc_opcode; + + for (i = 0, ppd = dd->pport; i < dd->num_pports; i++, ppd++) + rvt_init_port(&dd->verbs_dev.rdi, &ppd->ibport_data.rvp, i, + ppd->pkeys); + + ret = rvt_register_device(&dd->verbs_dev.rdi); + if (ret) + goto err_verbs_txreq; + + ret = hfi2_verbs_register_sysfs(dd); + if (ret) + goto err_class; + + return ret; + +err_class: + rvt_unregister_device(&dd->verbs_dev.rdi); +err_verbs_txreq: + hfi2_verbs_txreq_exit(dev); +err_cntr_descs: + kfree(dd->port_cntr_descs); + kfree(dd->dev_cntr_descs); + dd_dev_err(dd, "cannot register verbs: %d!\n", -ret); + return ret; +} + +void hfi2_unregister_ib_device(struct hfi2_devdata *dd) +{ + struct hfi2_ibdev *dev = &dd->verbs_dev; + + hfi2_verbs_unregister_sysfs(dd); + + rvt_unregister_device(&dd->verbs_dev.rdi); + + if (!list_empty(&dev->txwait)) + dd_dev_err(dd, "txwait list not empty!\n"); + if (!list_empty(&dev->memwait)) + dd_dev_err(dd, "memwait list not empty!\n"); + + timer_shutdown_sync(&dev->mem_timer); + hfi2_verbs_txreq_exit(dev); + + kfree(dd->dev_cntr_descs); + kfree(dd->port_cntr_descs); +} + +void hfi2_cnp_rcv(struct hfi2_packet *packet) +{ + struct hfi2_ibport *ibp = rcd_to_iport(packet->rcd); + struct hfi2_pportdata *ppd = ppd_from_ibp(ibp); + struct rvt_qp *qp = packet->qp; + u32 lqpn, rqpn = 0; + u16 rlid = 0; + u8 sl, svc_type; + + switch (packet->qp->ibqp.qp_type) { + case IB_QPT_UC: + rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); + rqpn = qp->remote_qpn; + svc_type = IB_CC_SVCTYPE_UC; + break; + case IB_QPT_RC: + rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); + rqpn = qp->remote_qpn; + svc_type = IB_CC_SVCTYPE_RC; + break; + case IB_QPT_SMI: + case IB_QPT_GSI: + case IB_QPT_UD: + svc_type = IB_CC_SVCTYPE_UD; + break; + default: + ibp->rvp.n_pkt_drops++; + return; + } + + sl = ibp->sc_to_sl[packet->sc]; + lqpn = qp->ibqp.qp_num; + + hfi2_process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); +} diff --git a/drivers/infiniband/hw/hfi2/verbs.h b/drivers/infiniband/hw/hfi2/verbs.h index 5b61a951e1d6..18ec034de681 100644 --- a/drivers/infiniband/hw/hfi2/verbs.h +++ b/drivers/infiniband/hw/hfi2/verbs.h @@ -35,12 +35,6 @@ struct hfi2_packet; #define HFI2_MAX_RDMA_ATOMIC 16 -/* - * Increment this value if any changes that break userspace ABI - * compatibility are made. - */ -#define HFI2_UVERBS_ABI_VERSION 2 - /* IB Performance Manager status values */ #define IB_PMA_SAMPLE_STATUS_DONE 0x00 #define IB_PMA_SAMPLE_STATUS_STARTED 0x01 diff --git a/drivers/infiniband/hw/hfi2/verbs_txreq.c b/drivers/infiniband/hw/hfi2/verbs_txreq.c new file mode 100644 index 000000000000..73cd0ced63a0 --- /dev/null +++ b/drivers/infiniband/hw/hfi2/verbs_txreq.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright(c) 2016 - 2018 Intel Corporation. + * Copyright(c) 2025-2026 Cornelis Networks, Inc. + */ + +#include "hfi2.h" +#include "verbs_txreq.h" +#include "qp.h" +#include "trace.h" + +#define TXREQ_LEN 24 + +void hfi2_dealloc_txreq(struct kref *ref) +{ + struct verbs_txreq *tx = container_of(ref, struct verbs_txreq, ref); + struct hfi2_ibdev *dev; + struct rvt_qp *qp; + unsigned long flags; + unsigned int seq; + struct hfi2_qp_priv *priv; + + qp = tx->qp; + dev = to_idev(qp->ibqp.device); + + if (tx->mr) + rvt_put_mr(tx->mr); + + sdma_txclean(dd_from_dev(dev), &tx->txreq); + + /* Free verbs_txreq and return to slab cache */ + kmem_cache_free(dev->verbs_txreq_cache, tx); + + do { + seq = read_seqbegin(&dev->txwait_lock); + if (!list_empty(&dev->txwait)) { + struct iowait *wait; + + write_seqlock_irqsave(&dev->txwait_lock, flags); + wait = list_first_entry(&dev->txwait, struct iowait, + list); + qp = iowait_to_qp(wait); + priv = qp->priv; + list_del_init(&priv->s_iowait.list); + /* refcount held until actual wake up */ + write_sequnlock_irqrestore(&dev->txwait_lock, flags); + hfi2_qp_wakeup(qp, RVT_S_WAIT_TX); + break; + } + } while (read_seqretry(&dev->txwait_lock, seq)); +} + +struct verbs_txreq *__hfi2_get_txreq(struct hfi2_ibdev *dev, + struct rvt_qp *qp) + __must_hold(&qp->s_lock) +{ + struct verbs_txreq *tx = NULL; + + write_seqlock(&dev->txwait_lock); + if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { + struct hfi2_qp_priv *priv; + + tx = kmem_cache_alloc(dev->verbs_txreq_cache, VERBS_TXREQ_GFP); + if (tx) + goto out; + priv = qp->priv; + if (list_empty(&priv->s_iowait.list)) { + dev->n_txwait++; + qp->s_flags |= RVT_S_WAIT_TX; + list_add_tail(&priv->s_iowait.list, &dev->txwait); + priv->s_iowait.lock = &dev->txwait_lock; + trace_hfi2_qpsleep(qp, RVT_S_WAIT_TX); + rvt_get_qp(qp); + } + qp->s_flags &= ~RVT_S_BUSY; + } +out: + write_sequnlock(&dev->txwait_lock); + return tx; +} + +int hfi2_verbs_txreq_init(struct hfi2_ibdev *dev) +{ + char buf[TXREQ_LEN]; + struct hfi2_devdata *dd = dd_from_dev(dev); + + snprintf(buf, sizeof(buf), "hfi2_%u_vtxreq_cache", dd->unit); + dev->verbs_txreq_cache = kmem_cache_create(buf, + sizeof(struct verbs_txreq), + 0, SLAB_HWCACHE_ALIGN, + NULL); + if (!dev->verbs_txreq_cache) + return -ENOMEM; + return 0; +} + +void hfi2_verbs_txreq_exit(struct hfi2_ibdev *dev) +{ + kmem_cache_destroy(dev->verbs_txreq_cache); + dev->verbs_txreq_cache = NULL; +}