[infiniband] Allow external QPN to differ from real QPN

Most IB hardware seems not to allow allocation of the genuine QPNs 0
and 1, so allow for the externally-visible QPN (as constructed and
parsed by ib_packet, where used) to differ from the real
hardware-allocated QPN.

3 files changed, 32 insertions, 8 deletions

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 9c1f57a0..dacb13e4 100644
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -96,8 +96,15 @@ struct ib_queue_pair {
 	struct ib_device *ibdev;
 	/** List of queue pairs on this Infiniband device */
 	struct list_head list;
-	/** Queue Pair Number */
+	/** Queue pair number */
 	unsigned long qpn;
+	/** Externally-visible queue pair number
+	 *
+	 * This may differ from the real queue pair number (e.g. when
+	 * the HCA cannot use the management QPNs 0 and 1 as hardware
+	 * QPNs and needs to remap them).
+	 */
+	unsigned long ext_qpn;
 	/** Queue pair type */
 	enum ib_queue_pair_type type;
 	/** Queue key */
diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 1444a12c..b15dcc61 100644
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -196,7 +196,6 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
 		       "%s\n", ibdev, strerror ( rc ) );
 		goto err_dev_create_qp;
 	}
-
 	DBGC ( ibdev, "IBDEV %p created queue pair %p (%p) with QPN %#lx\n",
 	       ibdev, qp, ib_qp_get_drvdata ( qp ), qp->qpn );
 	DBGC ( ibdev, "IBDEV %p QPN %#lx has %d send entries at [%p,%p)\n",
@@ -205,6 +204,24 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
 	DBGC ( ibdev, "IBDEV %p QPN %#lx has %d receive entries at [%p,%p)\n",
 	       ibdev, qp->qpn, num_recv_wqes, qp->recv.iobufs,
 	       ( ( ( void * ) qp ) + total_size ) );
+
+	/* Calculate externally-visible QPN */
+	switch ( type ) {
+	case IB_QPT_SMA:
+		qp->ext_qpn = IB_QPN_SMA;
+		break;
+	case IB_QPT_GMA:
+		qp->ext_qpn = IB_QPN_GMA;
+		break;
+	default:
+		qp->ext_qpn = qp->qpn;
+		break;
+	}
+	if ( qp->ext_qpn != qp->qpn ) {
+		DBGC ( ibdev, "IBDEV %p QPN %#lx has external QPN %#lx\n",
+		       ibdev, qp->qpn, qp->ext_qpn );
+	}
+
 	return qp;
 
 	ibdev->op->destroy_qp ( ibdev, qp );
@@ -295,7 +312,7 @@ struct ib_queue_pair * ib_find_qp_qpn ( struct ib_device *ibdev,
 	struct ib_queue_pair *qp;
 
 	list_for_each_entry ( qp, &ibdev->qps, list ) {
-		if ( qp->qpn == qpn )
+		if ( ( qpn == qp->qpn ) || ( qpn == qp->ext_qpn ) )
 			return qp;
 	}
 	return NULL;
diff --git a/src/net/infiniband/ib_packet.c b/src/net/infiniband/ib_packet.c
index 409ef2f8..689e00dc 100644
--- a/src/net/infiniband/ib_packet.c
+++ b/src/net/infiniband/ib_packet.c
@@ -58,7 +58,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
 	unsigned int lnh;
 
 	DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
-		ibdev, ibdev->lid, qp->qpn, av->lid, av->qpn, av->qkey );
+		ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey );
 
 	/* Calculate packet length */
 	pad_len = ( (-payload_len) & 0x3 );
@@ -76,7 +76,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
 	lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
 
 	/* Construct LRH */
-	vl = ( ( av->qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT );
+	vl = ( ( qp->ext_qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT );
 	lrh->vl__lver = ( vl << 4 );
 	lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
 	lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
@@ -104,7 +104,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
 
 	/* Construct DETH */
 	deth->qkey = htonl ( av->qkey );
-	deth->src_qp = htonl ( qp->qpn );
+	deth->src_qp = htonl ( qp->ext_qpn );
 
 	DBGCP_HDA ( ibdev, 0, iobuf->data,
 		    ( iob_len ( iobuf ) - orig_iob_len ) );
@@ -233,8 +233,8 @@ int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
 	}
 
 	DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n",
-		ibdev, lid,
-		( IB_LID_MULTICAST( lid ) ? ( qp ? (*qp)->qpn : -1UL ) : qpn ),
+		ibdev, lid, ( IB_LID_MULTICAST( lid ) ?
+			      ( qp ? (*qp)->ext_qpn : -1UL ) : qpn ),
 		av->lid, av->qpn, ntohl ( deth->qkey ) );
 	DBGCP_HDA ( ibdev, 0,
 		    ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),