[arbel] Allow for multiple calls to ib_modify_qp()

Signed-off-by: Michael Brown <mcb30@ipxe.org>

2 files changed, 44 insertions, 22 deletions

diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index 35ad88b36..6a7816206 100644
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -953,7 +953,7 @@ static int arbel_create_qp ( struct ib_device *ibdev,
 		     res, ARBEL_UAR_RES_RQ,
 		     qp_number, qp->qpn );
 
-	/* Hand queue over to hardware */
+	/* Transition queue to INIT state */
 	memset ( &qpctx, 0, sizeof ( qpctx ) );
 	MLX_FILL_3 ( &qpctx, 2,
 		     qpc_eec_data.de, 1,
@@ -961,9 +961,7 @@ static int arbel_create_qp ( struct ib_device *ibdev,
 		     qpc_eec_data.st,
 		     ( ( qp->type == IB_QPT_UD ) ?
 		       ARBEL_ST_UD : ARBEL_ST_MLX ) );
-	MLX_FILL_6 ( &qpctx, 4,
-		     qpc_eec_data.mtu, ARBEL_MTU_2048,
-		     qpc_eec_data.msg_max, 11 /* 2^11 = 2048 */,
+	MLX_FILL_4 ( &qpctx, 4,
 		     qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
 		     qpc_eec_data.log_rq_stride,
 		     ( fls ( sizeof ( arbel_qp->recv.wqe[0] ) - 1 ) - 4 ),
@@ -993,21 +991,7 @@ static int arbel_create_qp ( struct ib_device *ibdev,
 		       arbel, qp->qpn, strerror ( rc ) );
 		goto err_rst2init_qpee;
 	}
-	memset ( &qpctx, 0, sizeof ( qpctx ) );
-	MLX_FILL_2 ( &qpctx, 4,
-		     qpc_eec_data.mtu, ARBEL_MTU_2048,
-		     qpc_eec_data.msg_max, 11 /* 2^11 = 2048 */ );
-	if ( ( rc = arbel_cmd_init2rtr_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
-		DBGC ( arbel, "Arbel %p QPN %#lx INIT2RTR_QPEE failed: %s\n",
-		       arbel, qp->qpn, strerror ( rc ) );
-		goto err_init2rtr_qpee;
-	}
-	memset ( &qpctx, 0, sizeof ( qpctx ) );
-	if ( ( rc = arbel_cmd_rtr2rts_qpee ( arbel, qp->qpn, &qpctx ) ) != 0 ){
-		DBGC ( arbel, "Arbel %p QPN %#lx RTR2RTS_QPEE failed: %s\n",
-		       arbel, qp->qpn, strerror ( rc ) );
-		goto err_rtr2rts_qpee;
-	}
+	arbel_qp->state = ARBEL_QP_ST_INIT;
 
 	DBGC ( arbel, "Arbel %p QPN %#lx send ring [%08lx,%08lx), doorbell "
 	       "%08lx\n", arbel, qp->qpn, virt_to_phys ( arbel_qp->send.wqe ),
@@ -1024,8 +1008,6 @@ static int arbel_create_qp ( struct ib_device *ibdev,
 	ib_qp_set_drvdata ( qp, arbel_qp );
 	return 0;
 
- err_rtr2rts_qpee:
- err_init2rtr_qpee:
 	arbel_cmd_2rst_qpee ( arbel, qp->qpn );
  err_rst2init_qpee:
 	MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
@@ -1051,10 +1033,40 @@ static int arbel_create_qp ( struct ib_device *ibdev,
 static int arbel_modify_qp ( struct ib_device *ibdev,
 			     struct ib_queue_pair *qp ) {
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
+	struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
 	struct arbelprm_qp_ee_state_transitions qpctx;
 	int rc;
 
-	/* Issue RTS2RTS_QPEE */
+	/* Transition queue to RTR state, if applicable */
+	if ( arbel_qp->state < ARBEL_QP_ST_RTR ) {
+		memset ( &qpctx, 0, sizeof ( qpctx ) );
+		MLX_FILL_2 ( &qpctx, 4,
+			     qpc_eec_data.mtu, ARBEL_MTU_2048,
+			     qpc_eec_data.msg_max, 11 /* 2^11 = 2048 */ );
+		if ( ( rc = arbel_cmd_init2rtr_qpee ( arbel, qp->qpn,
+						      &qpctx ) ) != 0 ) {
+			DBGC ( arbel, "Arbel %p QPN %#lx INIT2RTR_QPEE failed:"
+			       " %s\n", arbel, qp->qpn, strerror ( rc ) );
+			return rc;
+		}
+		arbel_qp->state = ARBEL_QP_ST_RTR;
+	}
+
+	/* Transition queue to RTS state, if applicable */
+	if ( arbel_qp->state < ARBEL_QP_ST_RTS ) {
+		memset ( &qpctx, 0, sizeof ( qpctx ) );
+		MLX_FILL_1 ( &qpctx, 32,
+			     qpc_eec_data.next_send_psn, qp->send.psn );
+		if ( ( rc = arbel_cmd_rtr2rts_qpee ( arbel, qp->qpn,
+						     &qpctx ) ) != 0 ) {
+			DBGC ( arbel, "Arbel %p QPN %#lx RTR2RTS_QPEE failed: "
+			       "%s\n", arbel, qp->qpn, strerror ( rc ) );
+			return rc;
+		}
+		arbel_qp->state = ARBEL_QP_ST_RTS;
+	}
+
+	/* Update parameters in RTS state */
 	memset ( &qpctx, 0, sizeof ( qpctx ) );
 	MLX_FILL_1 ( &qpctx, 0, opt_param_mask, ARBEL_QPEE_OPT_PARAM_QKEY );
 	MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
diff --git a/src/drivers/infiniband/arbel.h b/src/drivers/infiniband/arbel.h
index 3fea17a5f..dd64afb56 100644
--- a/src/drivers/infiniband/arbel.h
+++ b/src/drivers/infiniband/arbel.h
@@ -365,12 +365,22 @@ struct arbel_recv_work_queue {
 /** Queue pair number randomisation mask */
 #define ARBEL_QPN_RANDOM_MASK 0xfff000
 
+/** Arbel queue pair state */
+enum arbel_queue_pair_state {
+	ARBEL_QP_ST_RST = 0,
+	ARBEL_QP_ST_INIT,
+	ARBEL_QP_ST_RTR,
+	ARBEL_QP_ST_RTS,
+};
+
 /** An Arbel queue pair */
 struct arbel_queue_pair {
 	/** Send work queue */
 	struct arbel_send_work_queue send;
 	/** Receive work queue */
 	struct arbel_recv_work_queue recv;
+	/** Queue state */
+	enum arbel_queue_pair_state state;
 };
 
 /** Maximum number of allocatable completion queues