[infiniband] Add the concept of an Infiniband upper-layer driver

Replace the explicit calls from the Infiniband core to the IPoIB layer
with the general concept of an Infiniband upper-layer driver
(analogous to a PCI driver) which can create arbitrary devices on top
of Infiniband devices.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

4 files changed, 169 insertions, 80 deletions

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index baa1c644..82d7ca99 100644
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -598,6 +598,42 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
 }
 
 /**
+ * Handle link status change
+ *
+ * @v ibdev		Infiniband device
+ */
+static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
+	struct net_device *netdev = ib_get_ownerdata ( ibdev );
+	struct ipoib_device *ipoib = netdev->priv;
+	struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
+	int rc;
+
+	/* Leave existing broadcast group */
+	ipoib_leave_broadcast_group ( ipoib );
+
+	/* Update MAC address based on potentially-new GID prefix */
+	memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
+		 sizeof ( mac->gid.u.half[0] ) );
+
+	/* Update broadcast GID based on potentially-new partition key */
+	ipoib->broadcast.gid.u.words[2] =
+		htons ( ibdev->pkey | IB_PKEY_FULL );
+
+	/* Set net device link state to reflect Infiniband link state */
+	rc = ib_link_rc ( ibdev );
+	netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
+
+	/* Join new broadcast group */
+	if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
+	     ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
+		DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
+		       "%s\n", ipoib, strerror ( rc ) );
+		netdev_link_err ( netdev, rc );
+		return;
+	}
+}
+
+/**
  * Open IPoIB network device
  *
  * @v netdev		Network device
@@ -691,48 +727,12 @@ static struct net_device_operations ipoib_operations = {
 };
 
 /**
- * Handle link status change
- *
- * @v ibdev		Infiniband device
- */
-void ipoib_link_state_changed ( struct ib_device *ibdev ) {
-	struct net_device *netdev = ib_get_ownerdata ( ibdev );
-	struct ipoib_device *ipoib = netdev->priv;
-	struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
-	int rc;
-
-	/* Leave existing broadcast group */
-	ipoib_leave_broadcast_group ( ipoib );
-
-	/* Update MAC address based on potentially-new GID prefix */
-	memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
-		 sizeof ( mac->gid.u.half[0] ) );
-
-	/* Update broadcast GID based on potentially-new partition key */
-	ipoib->broadcast.gid.u.words[2] =
-		htons ( ibdev->pkey | IB_PKEY_FULL );
-
-	/* Set net device link state to reflect Infiniband link state */
-	rc = ib_link_rc ( ibdev );
-	netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
-
-	/* Join new broadcast group */
-	if ( ib_link_ok ( ibdev ) &&
-	     ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
-		DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
-		       "%s\n", ipoib, strerror ( rc ) );
-		netdev_link_err ( netdev, rc );
-		return;
-	}
-}
-
-/**
  * Probe IPoIB device
  *
  * @v ibdev		Infiniband device
  * @ret rc		Return status code
  */
-int ipoib_probe ( struct ib_device *ibdev ) {
+static int ipoib_probe ( struct ib_device *ibdev ) {
 	struct net_device *netdev;
 	struct ipoib_device *ipoib;
 	int rc;
@@ -775,10 +775,18 @@ int ipoib_probe ( struct ib_device *ibdev ) {
  *
  * @v ibdev		Infiniband device
  */
-void ipoib_remove ( struct ib_device *ibdev ) {
+static void ipoib_remove ( struct ib_device *ibdev ) {
 	struct net_device *netdev = ib_get_ownerdata ( ibdev );
 
 	unregister_netdev ( netdev );
 	netdev_nullify ( netdev );
 	netdev_put ( netdev );
 }
+
+/** IPoIB driver */
+struct ib_driver ipoib_driver __ib_driver = {
+	.name = "IPoIB",
+	.probe = ipoib_probe,
+	.notify = ipoib_link_state_changed,
+	.remove = ipoib_remove,
+};
diff --git a/src/include/ipxe/infiniband.h b/src/include/ipxe/infiniband.h
index f2eb57cd..edcce371 100644
--- a/src/include/ipxe/infiniband.h
+++ b/src/include/ipxe/infiniband.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <stdint.h>
 #include <ipxe/refcnt.h>
 #include <ipxe/device.h>
+#include <ipxe/tables.h>
 #include <ipxe/ib_packet.h>
 #include <ipxe/ib_mad.h>
 
@@ -432,6 +433,34 @@ struct ib_device {
 	void *owner_priv;
 };
 
+/** An Infiniband upper-layer driver */
+struct ib_driver {
+	/** Name */
+	const char *name;
+	/** Probe device
+	 *
+	 * @v ibdev		Infiniband device
+	 * @ret rc		Return status code
+	 */
+	int ( * probe ) ( struct ib_device *ibdev );
+	/** Notify of device or link state change
+	 *
+	 * @v ibdev		Infiniband device
+	 */
+	void ( * notify ) ( struct ib_device *ibdev );
+	/** Remove device
+	 *
+	 * @v ibdev		Infiniband device
+	 */
+	void ( * remove ) ( struct ib_device *ibdev );
+};
+
+/** Infiniband driver table */
+#define IB_DRIVERS __table ( struct ib_driver, "ib_drivers" )
+
+/** Declare an Infiniband driver */
+#define __ib_driver __table_entry ( IB_DRIVERS, 01 )
+
 extern struct ib_completion_queue *
 ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
 	       struct ib_completion_queue_operations *op );
@@ -492,7 +521,7 @@ extern struct list_head ib_devices;
 	list_for_each_entry ( (ibdev), &ib_devices, list )
 
 /**
- * Check link state
+ * Check link state of Infiniband device
  *
  * @v ibdev		Infiniband device
  * @ret link_up		Link is up
@@ -503,6 +532,17 @@ ib_link_ok ( struct ib_device *ibdev ) {
 }
 
 /**
+ * Check whether or not Infiniband device is open
+ *
+ * @v ibdev		Infiniband device
+ * @v is_open		Infiniband device is open
+ */
+static inline __attribute__ (( always_inline )) int
+ib_is_open ( struct ib_device *ibdev ) {
+	return ( ibdev->open_count > 0 );
+}
+
+/**
  * Get reference to Infiniband device
  *
  * @v ibdev		Infiniband device
diff --git a/src/include/ipxe/ipoib.h b/src/include/ipxe/ipoib.h
index 31b0c1b6..6a3fd607 100644
--- a/src/include/ipxe/ipoib.h
+++ b/src/include/ipxe/ipoib.h
@@ -53,9 +53,6 @@ struct ipoib_hdr {
 } __attribute__ (( packed ));
 
 extern const char * ipoib_ntoa ( const void *ll_addr );
-extern void ipoib_link_state_changed ( struct ib_device *ibdev );
-extern int ipoib_probe ( struct ib_device *ibdev );
-extern void ipoib_remove ( struct ib_device *ibdev );
 extern struct net_device * alloc_ipoibdev ( size_t priv_size );
 
 #endif /* _IPXE_IPOIB_H */
diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 5daecd52..76fce3bd 100644
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/if_arp.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/iobuf.h>
-#include <ipxe/ipoib.h>
 #include <ipxe/process.h>
 #include <ipxe/infiniband.h>
 #include <ipxe/ib_mi.h>
@@ -539,6 +538,64 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
  */
 
 /**
+ * Get link state
+ *
+ * @v ibdev		Infiniband device
+ * @ret rc		Link status code
+ */
+int ib_link_rc ( struct ib_device *ibdev ) {
+	switch ( ibdev->port_state ) {
+	case IB_PORT_STATE_DOWN:	return -ENOTCONN;
+	case IB_PORT_STATE_INIT:	return -EINPROGRESS_INIT;
+	case IB_PORT_STATE_ARMED:	return -EINPROGRESS_ARMED;
+	case IB_PORT_STATE_ACTIVE:	return 0;
+	default:			return -EINVAL;
+	}
+}
+
+/**
+ * Textual representation of Infiniband link state
+ *
+ * @v ibdev		Infiniband device
+ * @ret link_text	Link state text
+ */
+static const char * ib_link_state_text ( struct ib_device *ibdev ) {
+	switch ( ibdev->port_state ) {
+	case IB_PORT_STATE_DOWN:	return "DOWN";
+	case IB_PORT_STATE_INIT:	return "INIT";
+	case IB_PORT_STATE_ARMED:	return "ARMED";
+	case IB_PORT_STATE_ACTIVE:	return "ACTIVE";
+	default:			return "UNKNOWN";
+	}
+}
+
+/**
+ * Notify drivers of Infiniband device or link state change
+ *
+ * @v ibdev		Infiniband device
+ */
+static void ib_notify ( struct ib_device *ibdev ) {
+	struct ib_driver *driver;
+
+	for_each_table_entry ( driver, IB_DRIVERS )
+		driver->notify ( ibdev );
+}
+
+/**
+ * Notify of Infiniband link state change
+ *
+ * @v ibdev		Infiniband device
+ */
+void ib_link_state_changed ( struct ib_device *ibdev ) {
+
+	DBGC ( ibdev, "IBDEV %p link state is %s\n",
+	       ibdev, ib_link_state_text ( ibdev ) );
+
+	/* Notify drivers of link state change */
+	ib_notify ( ibdev );
+}
+
+/**
  * Open port
  *
  * @v ibdev		Infiniband device
@@ -586,6 +643,9 @@ int ib_open ( struct ib_device *ibdev ) {
 	/* Add to head of open devices list */
 	list_add ( &ibdev->open_list, &open_ib_devices );
 
+	/* Notify drivers of device state change */
+	ib_notify ( ibdev );
+
 	assert ( ibdev->open_count == 1 );
 	return 0;
 
@@ -614,6 +674,7 @@ void ib_close ( struct ib_device *ibdev ) {
 
 	/* Close device if this was the last remaining requested opening */
 	if ( ibdev->open_count == 0 ) {
+		ib_notify ( ibdev );
 		list_del ( &ibdev->open_list );
 		ib_destroy_mi ( ibdev, ibdev->gsi );
 		ib_destroy_sma ( ibdev, ibdev->smi );
@@ -622,22 +683,6 @@ void ib_close ( struct ib_device *ibdev ) {
 	}
 }
 
-/**
- * Get link state
- *
- * @v ibdev		Infiniband device
- * @ret rc		Link status code
- */
-int ib_link_rc ( struct ib_device *ibdev ) {
-	switch ( ibdev->port_state ) {
-	case IB_PORT_STATE_DOWN:	return -ENOTCONN;
-	case IB_PORT_STATE_INIT:	return -EINPROGRESS_INIT;
-	case IB_PORT_STATE_ARMED:	return -EINPROGRESS_ARMED;
-	case IB_PORT_STATE_ACTIVE:	return 0;
-	default:			return -EINVAL;
-	}
-}
-
 /***************************************************************************
  *
  * Multicast
@@ -800,17 +845,6 @@ int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad ) {
  */
 
 /**
- * Handle Infiniband link state change
- *
- * @v ibdev		Infiniband device
- */
-void ib_link_state_changed ( struct ib_device *ibdev ) {
-
-	/* Notify IPoIB of link state change */
-	ipoib_link_state_changed ( ibdev );
-}
-
-/**
  * Poll event queue
  *
  * @v ibdev		Infiniband device
@@ -883,24 +917,29 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
  * @ret rc		Return status code
  */
 int register_ibdev ( struct ib_device *ibdev ) {
+	struct ib_driver *driver;
 	int rc;
 
 	/* Add to device list */
 	ibdev_get ( ibdev );
 	list_add_tail ( &ibdev->list, &ib_devices );
+	DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
+	       ibdev->dev->name );
 
-	/* Add IPoIB device */
-	if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
-		DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
-		       ibdev, strerror ( rc ) );
-		goto err_ipoib_probe;
+	/* Probe device */
+	for_each_table_entry ( driver, IB_DRIVERS ) {
+		if ( ( rc = driver->probe ( ibdev ) ) != 0 ) {
+			DBGC ( ibdev, "IBDEV %p could not add %s device: %s\n",
+			       ibdev, driver->name, strerror ( rc ) );
+			goto err_probe;
+		}
 	}
 
-	DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
-	       ibdev->dev->name );
 	return 0;
 
- err_ipoib_probe:
+ err_probe:
+	for_each_table_entry_continue_reverse ( driver, IB_DRIVERS )
+		driver->remove ( ibdev );
 	list_del ( &ibdev->list );
 	ibdev_put ( ibdev );
 	return rc;
@@ -912,9 +951,11 @@ int register_ibdev ( struct ib_device *ibdev ) {
  * @v ibdev		Infiniband device
  */
 void unregister_ibdev ( struct ib_device *ibdev ) {
+	struct ib_driver *driver;
 
-	/* Close device */
-	ipoib_remove ( ibdev );
+	/* Remove device */
+	for_each_table_entry_reverse ( driver, IB_DRIVERS )
+		driver->remove ( ibdev );
 
 	/* Remove from device list */
 	list_del ( &ibdev->list );
@@ -953,3 +994,6 @@ struct ib_device * last_opened_ibdev ( void ) {
 
 	return NULL;
 }
+
+/* Drag in IPoIB */
+REQUIRE_OBJECT ( ipoib );