[Infiniband] Add preliminary multiple port support for Hermon cards

Infiniband devices no longer block waiting for link-up in
register_ibdev().

Hermon driver needs to create an event queue and poll for link-up events.

Infiniband core needs to reread MAD parameters when link state changes.

IPoIB needs to cope with Infiniband link parameters being only partially
available at probe and open time.

1 files changed, 52 insertions, 41 deletions

diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 39d11285..e5c79e96 100644
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -153,14 +153,40 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
 }
 
 /**
+ * Modify queue pair
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v mod_list		Modification list
+ * @v qkey		New queue key, if applicable
+ * @ret rc		Return status code
+ */
+int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
+		   unsigned long mod_list, unsigned long qkey ) {
+	int rc;
+
+	DBGC ( ibdev, "IBDEV %p modifying QPN %#lx\n", ibdev, qp->qpn );
+
+	if ( mod_list & IB_MODIFY_QKEY )
+		qp->qkey = qkey;
+
+	if ( ( rc = ibdev->op->modify_qp ( ibdev, qp, mod_list ) ) != 0 ) {
+		DBGC ( ibdev, "IBDEV %p could not modify QPN %#lx: %s\n",
+		       ibdev, qp->qpn, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
  * Destroy queue pair
  *
  * @v ibdev		Infiniband device
  * @v qp		Queue pair
  */
-void ib_destroy_qp ( struct ib_device *ibdev,
-		     struct ib_queue_pair *qp ) {
-	DBGC ( ibdev, "IBDEV %p destroying queue pair %#lx\n",
+void ib_destroy_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
+	DBGC ( ibdev, "IBDEV %p destroying QPN %#lx\n",
 	       ibdev, qp->qpn );
 	ibdev->op->destroy_qp ( ibdev, qp );
 	list_del ( &qp->send.list );
@@ -280,38 +306,6 @@ static int ib_get_pkey_table ( struct ib_device *ibdev,
 }
 
 /**
- * Wait for link up
- *
- * @v ibdev		Infiniband device
- * @ret rc		Return status code
- *
- * This function shouldn't really exist.  Unfortunately, IB links take
- * a long time to come up, and we can't get various key parameters
- * e.g. our own IPoIB MAC address without information from the subnet
- * manager).  We should eventually make link-up an asynchronous event.
- */
-static int ib_wait_for_link ( struct ib_device *ibdev ) {
-	struct ib_mad_port_info port_info;
-	unsigned int retries;
-	int rc;
-
-	printf ( "Waiting for Infiniband link-up..." );
-	for ( retries = 20 ; retries ; retries-- ) {
-		if ( ( rc = ib_get_port_info ( ibdev, &port_info ) ) != 0 )
-			continue;
-		if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
-		     == 4 ) {
-			printf ( "ok\n" );
-			return 0;
-		}
-		printf ( "." );
-		sleep ( 1 );
-	}
-	printf ( "failed\n" );
-	return -ENODEV;
-};
-
-/**
  * Get MAD parameters
  *
  * @v ibdev		Infiniband device
@@ -326,9 +320,13 @@ static int ib_get_mad_params ( struct ib_device *ibdev ) {
 	} u;
 	int rc;
 
-	/* Port info gives us the first half of the port GID and the SM LID */
+	/* Port info gives us the link state, the first half of the
+	 * port GID and the SM LID.
+	 */
 	if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 )
 		return rc;
+	ibdev->link_up = ( ( u.port_info.port_state__link_speed_supported
+			     & 0xf ) == 4 );
 	memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 );
 	ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid );
 
@@ -391,10 +389,6 @@ int register_ibdev ( struct ib_device *ibdev ) {
 	if ( ( rc = ib_open ( ibdev ) ) != 0 )
 		goto err_open;
 
-	/* Wait for link */
-	if ( ( rc = ib_wait_for_link ( ibdev ) ) != 0 )
-		goto err_wait_for_link;
-
 	/* Get MAD parameters */
 	if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 )
 		goto err_get_mad_params;
@@ -410,7 +404,6 @@ int register_ibdev ( struct ib_device *ibdev ) {
 
  err_ipoib_probe:
  err_get_mad_params:
- err_wait_for_link:
 	ib_close ( ibdev );
  err_open:
 	return rc;
@@ -435,3 +428,21 @@ void free_ibdev ( struct ib_device *ibdev ) {
 	free ( ibdev );
 }
 
+/**
+ * Handle Infiniband link state change
+ *
+ * @v ibdev		Infiniband device
+ */
+void ib_link_state_changed ( struct ib_device *ibdev ) {
+	int rc;
+
+	/* Update MAD parameters */
+	if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) {
+		DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n",
+		       ibdev, strerror ( rc ) );
+		return;
+	}
+
+	/* Notify IPoIB of link state change */
+	ipoib_link_state_changed ( ibdev );
+}