[infiniband] Poll completion queues automatically

Currently, all Infiniband users must create a process for polling
their completion queues (or rely on a regular hook such as
netdev_poll() in ipoib.c).

Move instead to a model whereby the Infiniband core maintains a single
process calling ib_poll_eq(), and polling the event queue triggers
polls of the applicable completion queues.  (At present, the
Infiniband core simply polls all of the device's completion queues.)
Polling a completion queue will now implicitly refill all attached
receive work queues; this is analogous to the way that netdev_poll()
implicitly refills the RX ring.

Infiniband users no longer need to create a process just to poll their
completion queues and refill their receive rings.

7 files changed, 124 insertions, 113 deletions

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 03dfa53a..a559d015 100644
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -743,10 +743,7 @@ static void ipoib_poll ( struct net_device *netdev ) {
 	struct ipoib_device *ipoib = netdev->priv;
 	struct ib_device *ibdev = ipoib->ibdev;
 
-	ib_poll_cq ( ibdev, ipoib->meta.cq );
-	ib_poll_cq ( ibdev, ipoib->data.cq );
-	ib_qset_refill_recv ( ibdev, &ipoib->meta );
-	ib_qset_refill_recv ( ibdev, &ipoib->data );
+	ib_poll_eq ( ibdev );
 }
 
 /**
@@ -861,8 +858,8 @@ static int ipoib_open ( struct net_device *netdev ) {
 	mac->qpn = htonl ( ipoib->data.qp->qpn );
 
 	/* Fill receive rings */
-	ib_qset_refill_recv ( ibdev, &ipoib->meta );
-	ib_qset_refill_recv ( ibdev, &ipoib->data );
+	ib_refill_recv ( ibdev, ipoib->meta.qp );
+	ib_refill_recv ( ibdev, ipoib->data.qp );
 
 	/* Join broadcast group */
 	if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) {
diff --git a/src/include/gpxe/ib_qset.h b/src/include/gpxe/ib_qset.h
index d7a76178..a1116978 100644
--- a/src/include/gpxe/ib_qset.h
+++ b/src/include/gpxe/ib_qset.h
@@ -18,8 +18,6 @@ struct ib_queue_set {
 	struct ib_completion_queue *cq;
 	/** Queue pair */
 	struct ib_queue_pair *qp;
-	/** Receive work queue maximum fill level */
-	unsigned int recv_max_fill;
 };
 
 extern int ib_create_qset ( struct ib_device *ibdev,
@@ -27,8 +25,6 @@ extern int ib_create_qset ( struct ib_device *ibdev,
 			    struct ib_completion_queue_operations *cq_op,
 			    unsigned int num_send_wqes,
 			    unsigned int num_recv_wqes, unsigned long qkey );
-extern void ib_qset_refill_recv ( struct ib_device *ibdev,
-				  struct ib_queue_set *qset );
 extern void ib_destroy_qset ( struct ib_device *ibdev,
 			      struct ib_queue_set *qset );
 
diff --git a/src/include/gpxe/ib_sma.h b/src/include/gpxe/ib_sma.h
index 6d984804..dadcdff2 100644
--- a/src/include/gpxe/ib_sma.h
+++ b/src/include/gpxe/ib_sma.h
@@ -10,7 +10,6 @@
 FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <gpxe/infiniband.h>
-#include <gpxe/process.h>
 
 /** Infiniband Subnet Management Agent operations */
 struct ib_sma_operations {
@@ -33,8 +32,6 @@ struct ib_sma {
 	struct ib_completion_queue *cq;
 	/** SMA queue pair */
 	struct ib_queue_pair *qp;
-	/** Poll process */
-	struct process poll;
 };
 
 /** SMA number of send WQEs
diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 6cfceda6..41c55ac5 100644
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -154,6 +154,10 @@ struct ib_completion_queue_operations {
 
 /** An Infiniband Completion Queue */
 struct ib_completion_queue {
+	/** Containing Infiniband device */
+	struct ib_device *ibdev;
+	/** List of completion queues on this Infiniband device */
+	struct list_head list;
 	/** Completion queue number */
 	unsigned long cqn;
 	/** Number of completion queue entries */
@@ -310,6 +314,8 @@ struct ib_device {
 	struct list_head list;
 	/** Underlying device */
 	struct device *dev;
+	/** List of completion queues */
+	struct list_head cqs;
 	/** List of queue pairs */
 	struct list_head qps;
 	/** Infiniband operations */
@@ -350,6 +356,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
 	       struct ib_completion_queue_operations *op );
 extern void ib_destroy_cq ( struct ib_device *ibdev,
 			    struct ib_completion_queue *cq );
+extern void ib_poll_cq ( struct ib_device *ibdev,
+			 struct ib_completion_queue *cq );
 extern struct ib_queue_pair *
 ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes,
 	       struct ib_completion_queue *send_cq, unsigned int num_recv_wqes,
@@ -376,6 +384,8 @@ extern void ib_complete_recv ( struct ib_device *ibdev,
 			       struct ib_queue_pair *qp,
 			       struct ib_address_vector *av,
 			       struct io_buffer *iobuf, int rc );
+extern void ib_refill_recv ( struct ib_device *ibdev,
+			     struct ib_queue_pair *qp );
 extern int ib_open ( struct ib_device *ibdev );
 extern void ib_close ( struct ib_device *ibdev );
 extern int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
@@ -388,6 +398,7 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size );
 extern int register_ibdev ( struct ib_device *ibdev );
 extern void unregister_ibdev ( struct ib_device *ibdev );
 extern void ib_link_state_changed ( struct ib_device *ibdev );
+extern void ib_poll_eq ( struct ib_device *ibdev );
 extern struct list_head ib_devices;
 
 /** Iterate over all network devices */
@@ -395,17 +406,6 @@ extern struct list_head ib_devices;
 	list_for_each_entry ( (ibdev), &ib_devices, list )
 
 /**
- * Poll completion queue
- *
- * @v ibdev		Infiniband device
- * @v cq		Completion queue
- */
-static inline __always_inline void
-ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
-	ibdev->op->poll_cq ( ibdev, cq );
-}
-
-/**
  * Check link state
  *
  * @v ibdev		Infiniband device
diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 4784f404..48572e03 100644
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -43,6 +43,13 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /** List of Infiniband devices */
 struct list_head ib_devices = LIST_HEAD_INIT ( ib_devices );
 
+/***************************************************************************
+ *
+ * Completion queues
+ *
+ ***************************************************************************
+ */
+
 /**
  * Create completion queue
  *
@@ -63,6 +70,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
 	cq = zalloc ( sizeof ( *cq ) );
 	if ( ! cq )
 		goto err_alloc_cq;
+	cq->ibdev = ibdev;
+	list_add ( &cq->list, &ibdev->cqs );
 	cq->num_cqes = num_cqes;
 	INIT_LIST_HEAD ( &cq->work_queues );
 	cq->op = op;
@@ -81,6 +90,7 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
 
 	ibdev->op->destroy_cq ( ibdev, cq );
  err_dev_create_cq:
+	list_del ( &cq->list );
 	free ( cq );
  err_alloc_cq:
 	return NULL;
@@ -98,10 +108,38 @@ void ib_destroy_cq ( struct ib_device *ibdev,
 	       ibdev, cq->cqn );
 	assert ( list_empty ( &cq->work_queues ) );
 	ibdev->op->destroy_cq ( ibdev, cq );
+	list_del ( &cq->list );
 	free ( cq );
 }
 
 /**
+ * Poll completion queue
+ *
+ * @v ibdev		Infiniband device
+ * @v cq		Completion queue
+ */
+void ib_poll_cq ( struct ib_device *ibdev,
+		  struct ib_completion_queue *cq ) {
+	struct ib_work_queue *wq;
+
+	/* Poll completion queue */
+	ibdev->op->poll_cq ( ibdev, cq );
+
+	/* Refill receive work queues */
+	list_for_each_entry ( wq, &cq->work_queues, list ) {
+		if ( ! wq->is_send )
+			ib_refill_recv ( ibdev, wq->qp );
+	}
+}
+
+/***************************************************************************
+ *
+ * Work queues
+ *
+ ***************************************************************************
+ */
+
+/**
  * Create queue pair
  *
  * @v ibdev		Infiniband device
@@ -401,6 +439,44 @@ void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 }
 
 /**
+ * Refill receive work queue
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ */
+void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
+	struct io_buffer *iobuf;
+	int rc;
+
+	/* Keep filling while unfilled entries remain */
+	while ( qp->recv.fill < qp->recv.num_wqes ) {
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE );
+		if ( ! iobuf ) {
+			/* Non-fatal; we will refill on next attempt */
+			return;
+		}
+
+		/* Post I/O buffer */
+		if ( ( rc = ib_post_recv ( ibdev, qp, iobuf ) ) != 0 ) {
+			DBGC ( ibdev, "IBDEV %p could not refill: %s\n",
+			       ibdev, strerror ( rc ) );
+			free_iob ( iobuf );
+			/* Give up */
+			return;
+		}
+	}
+}
+
+/***************************************************************************
+ *
+ * Link control
+ *
+ ***************************************************************************
+ */
+
+/**
  * Open port
  *
  * @v ibdev		Infiniband device
@@ -436,6 +512,13 @@ void ib_close ( struct ib_device *ibdev ) {
 		ibdev->op->close ( ibdev );
 }
 
+/***************************************************************************
+ *
+ * Multicast
+ *
+ ***************************************************************************
+ */
+
 /**
  * Attach to multicast group
  *
@@ -495,6 +578,13 @@ void ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	}
 }
 
+/***************************************************************************
+ *
+ * Miscellaneous
+ *
+ ***************************************************************************
+ */
+
 /**
  * Get Infiniband HCA information
  *
@@ -541,6 +631,22 @@ void ib_link_state_changed ( struct ib_device *ibdev ) {
 }
 
 /**
+ * Poll event queue
+ *
+ * @v ibdev		Infiniband device
+ */
+void ib_poll_eq ( struct ib_device *ibdev ) {
+	struct ib_completion_queue *cq;
+
+	/* Poll device's event queue */
+	ibdev->op->poll_eq ( ibdev );
+
+	/* Poll all completion queues */
+	list_for_each_entry ( cq, &ibdev->cqs, list )
+		ib_poll_cq ( ibdev, cq );
+}
+
+/**
  * Single-step the Infiniband event queue
  *
  * @v process		Infiniband event queue process
@@ -548,9 +654,8 @@ void ib_link_state_changed ( struct ib_device *ibdev ) {
 static void ib_step ( struct process *process __unused ) {
 	struct ib_device *ibdev;
 
-	list_for_each_entry ( ibdev, &ib_devices, list ) {
-		ibdev->op->poll_eq ( ibdev );
-	}
+	for_each_ibdev ( ibdev )
+		ib_poll_eq ( ibdev );
 }
 
 /** Infiniband event queue process */
@@ -581,6 +686,7 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
 	if ( ibdev ) {
 		drv_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) );
 		ib_set_drvdata ( ibdev, drv_priv );
+		INIT_LIST_HEAD ( &ibdev->cqs );
 		INIT_LIST_HEAD ( &ibdev->qps );
 		ibdev->lid = IB_LID_NONE;
 		ibdev->pkey = IB_PKEY_NONE;
diff --git a/src/net/infiniband/ib_qset.c b/src/net/infiniband/ib_qset.c
index 799489f9..0a1e1f9d 100644
--- a/src/net/infiniband/ib_qset.c
+++ b/src/net/infiniband/ib_qset.c
@@ -54,9 +54,6 @@ int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset,
 	assert ( qset->cq == NULL );
 	assert ( qset->qp == NULL );
 
-	/* Store queue parameters */
-	qset->recv_max_fill = num_recv_wqes;
-
 	/* Allocate completion queue */
 	qset->cq = ib_create_cq ( ibdev, num_cqes, cq_op );
 	if ( ! qset->cq ) {
@@ -84,37 +81,6 @@ int ib_create_qset ( struct ib_device *ibdev, struct ib_queue_set *qset,
 }
 
 /**
- * Refill IPoIB receive ring
- *
- * @v ibdev		Infiniband device
- * @v qset		Queue set
- */
-void ib_qset_refill_recv ( struct ib_device *ibdev,
-			   struct ib_queue_set *qset ) {
-	struct io_buffer *iobuf;
-	int rc;
-
-	while ( qset->qp->recv.fill < qset->recv_max_fill ) {
-
-		/* Allocate I/O buffer */
-		iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE );
-		if ( ! iobuf ) {
-			/* Non-fatal; we will refill on next attempt */
-			return;
-		}
-
-		/* Post I/O buffer */
-		if ( ( rc = ib_post_recv ( ibdev, qset->qp, iobuf ) ) != 0 ) {
-			DBGC ( ibdev, "IBDEV %p could not refill: %s\n",
-			       ibdev, strerror ( rc ) );
-			free_iob ( iobuf );
-			/* Give up */
-			return;
-		}
-	}
-}
-
-/**
  * Destroy queue set
  *
  * @v ibdev		Infiniband device
diff --git a/src/net/infiniband/ib_sma.c b/src/net/infiniband/ib_sma.c
index b83d20ea..5fd1319c 100644
--- a/src/net/infiniband/ib_sma.c
+++ b/src/net/infiniband/ib_sma.c
@@ -27,7 +27,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <byteswap.h>
 #include <gpxe/infiniband.h>
 #include <gpxe/iobuf.h>
-#include <gpxe/process.h>
 #include <gpxe/ib_sma.h>
 
 /**
@@ -349,36 +348,6 @@ static int ib_sma_mad ( struct ib_sma *sma, union ib_mad *mad ) {
 }
 
 /**
- * Refill SMA receive ring
- *
- * @v sma		Subnet management agent
- */
-static void ib_sma_refill_recv ( struct ib_sma *sma ) {
-	struct ib_device *ibdev = sma->ibdev;
-	struct io_buffer *iobuf;
-	int rc;
-
-	while ( sma->qp->recv.fill < IB_SMA_NUM_RECV_WQES ) {
-
-		/* Allocate I/O buffer */
-		iobuf = alloc_iob ( IB_MAX_PAYLOAD_SIZE );
-		if ( ! iobuf ) {
-			/* Non-fatal; we will refill on next attempt */
-			return;
-		}
-
-		/* Post I/O buffer */
-		if ( ( rc = ib_post_recv ( ibdev, sma->qp, iobuf ) ) != 0 ) {
-			DBGC ( sma, "SMA %p could not refill: %s\n",
-			       sma, strerror ( rc ) );
-			free_iob ( iobuf );
-			/* Give up */
-			return;
-		}
-	}
-}
-
-/**
  * Complete SMA send
  *
  *
@@ -457,23 +426,6 @@ static struct ib_completion_queue_operations ib_sma_completion_ops = {
 };
 
 /**
- * Poll SMA
- *
- * @v process		Process
- */
-static void ib_sma_step ( struct process *process ) {
-	struct ib_sma *sma =
-		container_of ( process, struct ib_sma, poll );
-	struct ib_device *ibdev = sma->ibdev;
-
-	/* Poll the kernel completion queue */
-	ib_poll_cq ( ibdev, sma->cq );
-
-	/* Refill the receive ring */
-	ib_sma_refill_recv ( sma );
-}
-
-/**
  * Create SMA
  *
  * @v sma		Subnet management agent
@@ -489,7 +441,6 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev,
 	memset ( sma, 0, sizeof ( *sma ) );
 	sma->ibdev = ibdev;
 	sma->op = op;
-	process_init ( &sma->poll, ib_sma_step, &ibdev->refcnt );
 
 	/* Create completion queue */
 	sma->cq = ib_create_cq ( ibdev, IB_SMA_NUM_CQES,
@@ -517,7 +468,7 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev,
 	}
 
 	/* Fill receive ring */
-	ib_sma_refill_recv ( sma );
+	ib_refill_recv ( ibdev, sma->qp );
 	return 0;
 
  err_not_qp0:
@@ -525,7 +476,6 @@ int ib_create_sma ( struct ib_sma *sma, struct ib_device *ibdev,
  err_create_qp:
 	ib_destroy_cq ( ibdev, sma->cq );
  err_create_cq:
-	process_del ( &sma->poll );
 	return rc;
 }
 
@@ -539,5 +489,4 @@ void ib_destroy_sma ( struct ib_sma *sma ) {
 
 	ib_destroy_qp ( ibdev, sma->qp );
 	ib_destroy_cq ( ibdev, sma->cq );
-	process_del ( &sma->poll );
 }