[infiniband] Poll completion queues automatically

Currently, all Infiniband users must create a process for polling
their completion queues (or rely on a regular hook such as
netdev_poll() in ipoib.c).

Move instead to a model whereby the Infiniband core maintains a single
process calling ib_poll_eq(), and polling the event queue triggers
polls of the applicable completion queues.  (At present, the
Infiniband core simply polls all of the device's completion queues.)
Polling a completion queue will now implicitly refill all attached
receive work queues; this is analogous to the way that netdev_poll()
implicitly refills the RX ring.

Infiniband users no longer need to create a process just to poll their
completion queues and refill their receive rings.

3 files changed, 11 insertions, 18 deletions

diff --git a/src/include/gpxe/ib_qset.h b/src/include/gpxe/ib_qset.h
index d7a76178e..a11169785 100644
--- a/src/include/gpxe/ib_qset.h
+++ b/src/include/gpxe/ib_qset.h
@@ -18,8 +18,6 @@ struct ib_queue_set {
 	struct ib_completion_queue *cq;
 	/** Queue pair */
 	struct ib_queue_pair *qp;
-	/** Receive work queue maximum fill level */
-	unsigned int recv_max_fill;
 };
 
 extern int ib_create_qset ( struct ib_device *ibdev,
@@ -27,8 +25,6 @@ extern int ib_create_qset ( struct ib_device *ibdev,
 			    struct ib_completion_queue_operations *cq_op,
 			    unsigned int num_send_wqes,
 			    unsigned int num_recv_wqes, unsigned long qkey );
-extern void ib_qset_refill_recv ( struct ib_device *ibdev,
-				  struct ib_queue_set *qset );
 extern void ib_destroy_qset ( struct ib_device *ibdev,
 			      struct ib_queue_set *qset );
 
diff --git a/src/include/gpxe/ib_sma.h b/src/include/gpxe/ib_sma.h
index 6d984804a..dadcdff2e 100644
--- a/src/include/gpxe/ib_sma.h
+++ b/src/include/gpxe/ib_sma.h
@@ -10,7 +10,6 @@
 FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <gpxe/infiniband.h>
-#include <gpxe/process.h>
 
 /** Infiniband Subnet Management Agent operations */
 struct ib_sma_operations {
@@ -33,8 +32,6 @@ struct ib_sma {
 	struct ib_completion_queue *cq;
 	/** SMA queue pair */
 	struct ib_queue_pair *qp;
-	/** Poll process */
-	struct process poll;
 };
 
 /** SMA number of send WQEs
diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 6cfceda66..41c55ac55 100644
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -154,6 +154,10 @@ struct ib_completion_queue_operations {
 
 /** An Infiniband Completion Queue */
 struct ib_completion_queue {
+	/** Containing Infiniband device */
+	struct ib_device *ibdev;
+	/** List of completion queues on this Infiniband device */
+	struct list_head list;
 	/** Completion queue number */
 	unsigned long cqn;
 	/** Number of completion queue entries */
@@ -310,6 +314,8 @@ struct ib_device {
 	struct list_head list;
 	/** Underlying device */
 	struct device *dev;
+	/** List of completion queues */
+	struct list_head cqs;
 	/** List of queue pairs */
 	struct list_head qps;
 	/** Infiniband operations */
@@ -350,6 +356,8 @@ ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
 	       struct ib_completion_queue_operations *op );
 extern void ib_destroy_cq ( struct ib_device *ibdev,
 			    struct ib_completion_queue *cq );
+extern void ib_poll_cq ( struct ib_device *ibdev,
+			 struct ib_completion_queue *cq );
 extern struct ib_queue_pair *
 ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes,
 	       struct ib_completion_queue *send_cq, unsigned int num_recv_wqes,
@@ -376,6 +384,8 @@ extern void ib_complete_recv ( struct ib_device *ibdev,
 			       struct ib_queue_pair *qp,
 			       struct ib_address_vector *av,
 			       struct io_buffer *iobuf, int rc );
+extern void ib_refill_recv ( struct ib_device *ibdev,
+			     struct ib_queue_pair *qp );
 extern int ib_open ( struct ib_device *ibdev );
 extern void ib_close ( struct ib_device *ibdev );
 extern int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
@@ -388,6 +398,7 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size );
 extern int register_ibdev ( struct ib_device *ibdev );
 extern void unregister_ibdev ( struct ib_device *ibdev );
 extern void ib_link_state_changed ( struct ib_device *ibdev );
+extern void ib_poll_eq ( struct ib_device *ibdev );
 extern struct list_head ib_devices;
 
 /** Iterate over all network devices */
@@ -395,17 +406,6 @@ extern struct list_head ib_devices;
 	list_for_each_entry ( (ibdev), &ib_devices, list )
 
 /**
- * Poll completion queue
- *
- * @v ibdev		Infiniband device
- * @v cq		Completion queue
- */
-static inline __always_inline void
-ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
-	ibdev->op->poll_cq ( ibdev, cq );
-}
-
-/**
  * Check link state
  *
  * @v ibdev		Infiniband device