Gbkr local

drivers/infiniband/hw/bnxt_re/bnxt_re.h

@@ -177,6 +177,8 @@ struct bnxt_re_dev {
 	atomic_t			srq_count;
 	atomic_t			mr_count;
 	atomic_t			mw_count;
+	atomic_t			ah_count;
+	atomic_t			pd_count;
 	/* Max of 2 lossless traffic class supported per port */
 	u16				cosq[2];
 

drivers/infiniband/hw/bnxt_re/hw_counters.c

@@ -58,6 +58,8 @@
 #include "hw_counters.h"
 
 static const char * const bnxt_re_stat_name[] = {
+	[BNXT_RE_ACTIVE_PD]		=  "active_pds",
+	[BNXT_RE_ACTIVE_AH]		=  "active_ahs",
 	[BNXT_RE_ACTIVE_QP]		=  "active_qps",
 	[BNXT_RE_ACTIVE_SRQ]		=  "active_srqs",
 	[BNXT_RE_ACTIVE_CQ]		=  "active_cqs",
@@ -128,6 +130,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
 	stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->cq_count);
 	stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->mr_count);
 	stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->mw_count);
+	stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&rdev->pd_count);
+	stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&rdev->ah_count);
 	if (bnxt_re_stats) {
 		stats->value[BNXT_RE_RECOVERABLE_ERRORS] =
 			le64_to_cpu(bnxt_re_stats->tx_bcast_pkts);

drivers/infiniband/hw/bnxt_re/hw_counters.h

@@ -41,6 +41,8 @@
 #define __BNXT_RE_HW_STATS_H__
 
 enum bnxt_re_hw_stats {
+	BNXT_RE_ACTIVE_PD,
+	BNXT_RE_ACTIVE_AH,
 	BNXT_RE_ACTIVE_QP,
 	BNXT_RE_ACTIVE_SRQ,
 	BNXT_RE_ACTIVE_CQ,

drivers/infiniband/hw/bnxt_re/ib_verbs.c

@@ -541,9 +541,12 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 
 	bnxt_re_destroy_fence_mr(pd);
 
-	if (pd->qplib_pd.id)
-		bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
-				      &pd->qplib_pd);
+	if (pd->qplib_pd.id) {
+		if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res,
+					   &rdev->qplib_res.pd_tbl,
+					   &pd->qplib_pd))
+			atomic_dec(&rdev->pd_count);
+	}
 	return 0;
 }
 
@@ -595,6 +598,8 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 		if (bnxt_re_create_fence_mr(pd))
 			ibdev_warn(&rdev->ibdev,
 				   "Failed to create Fence-MR\n");
+	atomic_inc(&rdev->pd_count);
+
 	return 0;
 dbfail:
 	bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
@@ -611,6 +616,8 @@ int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 
 	bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
 			      !(flags & RDMA_DESTROY_AH_SLEEPABLE));
+	atomic_dec(&rdev->ah_count);
+
 	return 0;
 }
 
@@ -695,6 +702,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 		wmb(); /* make sure cache is updated. */
 		spin_unlock_irqrestore(&uctx->sh_lock, flag);
 	}
+	atomic_inc(&rdev->ah_count);
 
 	return 0;
 }
@@ -760,6 +768,7 @@ static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
 	bnxt_qplib_destroy_ah(&rdev->qplib_res,
 			      &gsi_sah->qplib_ah,
 			      true);
+	atomic_dec(&rdev->ah_count);
 	bnxt_qplib_clean_qp(&qp->qplib_qp);
 
 	ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
@@ -1009,6 +1018,7 @@ static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
 			  "Failed to allocate HW AH for Shadow QP");
 		goto fail;
 	}
+	atomic_inc(&rdev->ah_count);
 
 	return ah;
 
@@ -1519,42 +1529,6 @@ static enum ib_qp_state __to_ib_qp_state(u8 state)
 	}
 }
 
-static u32 __from_ib_mtu(enum ib_mtu mtu)
-{
-	switch (mtu) {
-	case IB_MTU_256:
-		return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
-	case IB_MTU_512:
-		return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
-	case IB_MTU_1024:
-		return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
-	case IB_MTU_2048:
-		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
-	case IB_MTU_4096:
-		return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
-	default:
-		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
-	}
-}
-
-static enum ib_mtu __to_ib_mtu(u32 mtu)
-{
-	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
-	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
-		return IB_MTU_256;
-	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
-		return IB_MTU_512;
-	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
-		return IB_MTU_1024;
-	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
-		return IB_MTU_2048;
-	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
-		return IB_MTU_4096;
-	default:
-		return IB_MTU_2048;
-	}
-}
-
 /* Shared Receive Queues */
 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
 {
@@ -1933,18 +1907,19 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		}
 	}
 
-	if (qp_attr_mask & IB_QP_PATH_MTU) {
-		qp->qplib_qp.modify_flags |=
-				CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
-		qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
-		qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
-	} else if (qp_attr->qp_state == IB_QPS_RTR) {
-		qp->qplib_qp.modify_flags |=
-			CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
-		qp->qplib_qp.path_mtu =
-			__from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
-		qp->qplib_qp.mtu =
-			ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
+	/* MTU settings allowed only during INIT -> RTR */
+	if (qp_attr->qp_state == IB_QPS_RTR) {
+		rc = bnxt_re_init_qpmtu(qp, rdev->netdev->mtu, qp_attr_mask,
+					qp_attr);
+		if (rc) {
+			ibdev_err(&rdev->ibdev, "qp %#x invalid mtu",
+				  qp->qplib_qp.id);
+			/* TODO: Remove below line when driver has a way to
+			 * update user QP about trimmed mtu. Failure is non-
+			 * compliance to IB-Spec and is temporarily here.
+			 */
+			return -EINVAL;
+		}
 	}
 
 	if (qp_attr_mask & IB_QP_TIMEOUT) {

drivers/infiniband/hw/bnxt_re/ib_verbs.h

@@ -146,6 +146,75 @@ static inline u16 bnxt_re_get_rwqe_size(int nsge)
 	return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
 }
 
+static inline u32 __from_ib_mtu(enum ib_mtu mtu)
+{
+	switch (mtu) {
+	case IB_MTU_256:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
+	case IB_MTU_512:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
+	case IB_MTU_1024:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
+	case IB_MTU_2048:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+	case IB_MTU_4096:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
+	default:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+	}
+}
+
+static inline enum ib_mtu __to_ib_mtu(u32 mtu)
+{
+	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
+		return IB_MTU_256;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
+		return IB_MTU_512;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
+		return IB_MTU_1024;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
+		return IB_MTU_2048;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
+		return IB_MTU_4096;
+	default:
+		return IB_MTU_2048;
+	}
+}
+
+static inline int bnxt_re_init_qpmtu(struct bnxt_re_qp *qp, int mtu,
+				     int mask, struct ib_qp_attr *qp_attr)
+{
+	int qpmtu, qpmtu_int;
+	int ifmtu, ifmtu_int;
+	int rc = 0;
+
+	ifmtu = iboe_get_mtu(mtu);
+	ifmtu_int = ib_mtu_enum_to_int(ifmtu);
+	qpmtu = ifmtu;
+	qpmtu_int = ifmtu_int;
+	if (mask & IB_QP_PATH_MTU) {
+		qpmtu = qp_attr->path_mtu;
+		qpmtu_int = ib_mtu_enum_to_int(qpmtu);
+		if (qpmtu_int > ifmtu_int) {
+			qpmtu = ifmtu;
+			qpmtu_int = ifmtu_int;
+			/* However, the code appears to be trimming mtu here
+			 * but, driver should fail this request due to inabi-
+			 * lity to cummunicate new mtu to user qp. Silent
+			 * trim to mtu would break retranmission psn calcul-
+			 * ations.
+			 */
+			rc = -EINVAL;
+		}
+	}
+	qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu);
+	qp->qplib_qp.mtu = qpmtu_int;
+	qp->qplib_qp.modify_flags |=
+		CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+	return rc;
+}
+
 int bnxt_re_query_device(struct ib_device *ibdev,
 			 struct ib_device_attr *ib_attr,
 			 struct ib_udata *udata);

drivers/infiniband/hw/bnxt_re/main.c

@@ -774,6 +774,8 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
 	atomic_set(&rdev->srq_count, 0);
 	atomic_set(&rdev->mr_count, 0);
 	atomic_set(&rdev->mw_count, 0);
+	atomic_set(&rdev->ah_count, 0);
+	atomic_set(&rdev->pd_count, 0);
 	rdev->cosq[0] = 0xFFFF;
 	rdev->cosq[1] = 0xFFFF;
 

drivers/infiniband/hw/bnxt_re/qplib_sp.c

@@ -42,6 +42,7 @@
 #include <linux/spinlock.h>
 #include <linux/sched.h>
 #include <linux/pci.h>
+#include <rdma/bnxt_re-abi.h>
 
 #include "roce_hsi.h"
 
@@ -92,6 +93,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 	struct creq_query_func_resp resp;
 	struct bnxt_qplib_rcfw_sbuf *sbuf;
 	struct creq_query_func_resp_sb *sb;
+	struct bnxt_qplib_chip_ctx *cctx;
 	u16 cmd_flags = 0;
 	u32 temp;
 	u8 *tqm_alloc;
@@ -106,6 +108,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 		return -ENOMEM;
 	}
 
+	cctx = rcfw->res->cctx;
 	sb = sbuf->sb;
 	req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
 	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
@@ -130,8 +133,18 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 	 * reporting the max number
 	 */
 	attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS + 1;
-	attr->max_qp_sges = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx) ?
-			    6 : sb->max_sge;
+
+	if ((bnxt_qplib_is_chip_gen_p5(cctx)) &&
+	    (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)) {
+		u16 data_var_wqe;
+		attr->max_qp_sges = sb->max_sge_var_wqe;
+		data_var_wqe = le16_to_cpu(sb->max_inline_data_var_wqe);
+		attr->max_inline_data =	data_var_wqe;
+	} else {
+		attr->max_qp_sges = sb->max_sge;
+		attr->max_inline_data = le32_to_cpu(sb->max_inline_data);
+	}
+
 	attr->max_cq = le32_to_cpu(sb->max_cq);
 	attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
 	attr->max_cq_sges = attr->max_qp_sges;
@@ -157,7 +170,6 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 		attr->max_pkey = 0xFFFF;
 	}
 
-	attr->max_inline_data = le32_to_cpu(sb->max_inline_data);
 	attr->l2_db_size = (sb->l2_db_space_size + 1) *
 			    (0x01 << RCFW_DBR_BASE_PAGE_SHIFT);
 	attr->max_sgid = BNXT_QPLIB_NUM_GIDS_SUPPORTED;

drivers/infiniband/hw/bnxt_re/roce_hsi.h

@@ -2869,7 +2869,9 @@ struct creq_query_func_resp_sb {
 	__le32 max_gid;
 	__le32 tqm_alloc_reqs[12];
 	__le32 max_dpi;
-	__le32 reserved_32;
+	u8 max_sge_var_wqe;
+	u8 reserved_8;
+	__le16 max_inline_data_var_wqe;
 };
 
 /* Set resources command response (16 bytes) */