[infiniband] Flush uncompleted work queue entries at QP teardown

Avoid leaking I/O buffers in ib_destroy_qp() by completing any
outstanding work queue entries with a generic error code.  This
requires the completion handlers to be available to ib_destroy_qp(),
which is done by making them static configuration parameters of the CQ
(set by ib_create_cq()) rather than being provided on each call to
ib_poll_cq().

This mimics the functionality of netdev_{tx,rx}_flush().  The netdev
flush functions would previously have been catching any I/O buffers
leaked by the IPoIB data queue (though not by the IPoIB metadata
queue).

5 files changed, 131 insertions, 70 deletions

diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index 1b55131b..2649a4ab 100644
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -1106,15 +1106,11 @@ static int arbel_post_recv ( struct ib_device *ibdev,
  * @v ibdev		Infiniband device
  * @v cq		Completion queue
  * @v cqe		Hardware completion queue entry
- * @v complete_send	Send completion handler
- * @v complete_recv	Receive completion handler
  * @ret rc		Return status code
  */
 static int arbel_complete ( struct ib_device *ibdev,
 			    struct ib_completion_queue *cq,
-			    union arbelprm_completion_entry *cqe,
-			    ib_completer_t complete_send,
-			    ib_completer_t complete_recv ) {
+			    union arbelprm_completion_entry *cqe ) {
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
 	struct ib_completion completion;
 	struct ib_work_queue *wq;
@@ -1124,7 +1120,6 @@ static int arbel_complete ( struct ib_device *ibdev,
 	struct arbel_recv_work_queue *arbel_recv_wq;
 	struct arbelprm_recv_wqe *recv_wqe;
 	struct io_buffer *iobuf;
-	ib_completer_t complete;
 	unsigned int opcode;
 	unsigned long qpn;
 	int is_send;
@@ -1201,8 +1196,11 @@ static int arbel_complete ( struct ib_device *ibdev,
 	}
 
 	/* Pass off to caller's completion handler */
-	complete = ( is_send ? complete_send : complete_recv );
-	complete ( ibdev, qp, &completion, iobuf );
+	if ( is_send ) {
+		ib_complete_send ( ibdev, qp, &completion, iobuf );
+	} else {
+		ib_complete_recv ( ibdev, qp, &completion, iobuf );
+	}
 
 	return rc;
 }			     
@@ -1212,13 +1210,9 @@ static int arbel_complete ( struct ib_device *ibdev,
  *
  * @v ibdev		Infiniband device
  * @v cq		Completion queue
- * @v complete_send	Send completion handler
- * @v complete_recv	Receive completion handler
  */
 static void arbel_poll_cq ( struct ib_device *ibdev,
-			    struct ib_completion_queue *cq,
-			    ib_completer_t complete_send,
-			    ib_completer_t complete_recv ) {
+			    struct ib_completion_queue *cq ) {
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
 	struct arbel_completion_queue *arbel_cq = ib_cq_get_drvdata ( cq );
 	struct arbelprm_cq_ci_db_record *ci_db_rec;
@@ -1236,8 +1230,7 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
 		}
 
 		/* Handle completion */
-		if ( ( rc = arbel_complete ( ibdev, cq, cqe, complete_send,
-					     complete_recv ) ) != 0 ) {
+		if ( ( rc = arbel_complete ( ibdev, cq, cqe ) ) != 0 ) {
 			DBGC ( arbel, "Arbel %p failed to complete: %s\n",
 			       arbel, strerror ( rc ) );
 			DBGC_HD ( arbel, cqe, sizeof ( *cqe ) );
diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index 3ca60033..a8907634 100644
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -1101,22 +1101,17 @@ static int hermon_post_recv ( struct ib_device *ibdev,
  * @v ibdev		Infiniband device
  * @v cq		Completion queue
  * @v cqe		Hardware completion queue entry
- * @v complete_send	Send completion handler
- * @v complete_recv	Receive completion handler
  * @ret rc		Return status code
  */
 static int hermon_complete ( struct ib_device *ibdev,
 			     struct ib_completion_queue *cq,
-			     union hermonprm_completion_entry *cqe,
-			     ib_completer_t complete_send,
-			     ib_completer_t complete_recv ) {
+			     union hermonprm_completion_entry *cqe ) {
 	struct hermon *hermon = ib_get_drvdata ( ibdev );
 	struct ib_completion completion;
 	struct ib_work_queue *wq;
 	struct ib_queue_pair *qp;
 	struct hermon_queue_pair *hermon_qp;
 	struct io_buffer *iobuf;
-	ib_completer_t complete;
 	unsigned int opcode;
 	unsigned long qpn;
 	int is_send;
@@ -1172,8 +1167,11 @@ static int hermon_complete ( struct ib_device *ibdev,
 	}
 
 	/* Pass off to caller's completion handler */
-	complete = ( is_send ? complete_send : complete_recv );
-	complete ( ibdev, qp, &completion, iobuf );
+	if ( is_send ) {
+		ib_complete_send ( ibdev, qp, &completion, iobuf );
+	} else {
+		ib_complete_recv ( ibdev, qp, &completion, iobuf );
+	}
 
 	return rc;
 }
@@ -1183,13 +1181,9 @@ static int hermon_complete ( struct ib_device *ibdev,
  *
  * @v ibdev		Infiniband device
  * @v cq		Completion queue
- * @v complete_send	Send completion handler
- * @v complete_recv	Receive completion handler
  */
 static void hermon_poll_cq ( struct ib_device *ibdev,
-			     struct ib_completion_queue *cq,
-			     ib_completer_t complete_send,
-			     ib_completer_t complete_recv ) {
+			     struct ib_completion_queue *cq ) {
 	struct hermon *hermon = ib_get_drvdata ( ibdev );
 	struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
 	union hermonprm_completion_entry *cqe;
@@ -1209,8 +1203,7 @@ static void hermon_poll_cq ( struct ib_device *ibdev,
 		DBGCP_HD ( hermon, cqe, sizeof ( *cqe ) );
 
 		/* Handle completion */
-		if ( ( rc = hermon_complete ( ibdev, cq, cqe, complete_send,
-					      complete_recv ) ) != 0 ) {
+		if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) {
 			DBGC ( hermon, "Hermon %p failed to complete: %s\n",
 			       hermon, strerror ( rc ) );
 			DBGC_HD ( hermon, cqe, sizeof ( *cqe ) );
diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 812c11c9..a56ceb89 100644
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -278,13 +278,21 @@ static void ipoib_destroy_qset ( struct ipoib_device *ipoib,
  *
  * @v ipoib		IPoIB device
  * @v qset		Queue set
+ * @v num_cqes		Number of completion queue entries
+ * @v num_send_wqes	Number of send work queue entries
+ * @v complete_send	Send completion handler
+ * @v num_recv_wqes	Number of receive work queue entries
+ * @v complete_recv	Receive completion handler
+ * @v qkey		Queue key
  * @ret rc		Return status code
  */
 static int ipoib_create_qset ( struct ipoib_device *ipoib,
 			       struct ipoib_queue_set *qset,
 			       unsigned int num_cqes,
 			       unsigned int num_send_wqes,
+			       ib_completer_t complete_send,
 			       unsigned int num_recv_wqes,
+			       ib_completer_t complete_recv,
 			       unsigned long qkey ) {
 	struct ib_device *ibdev = ipoib->ibdev;
 	int rc;
@@ -297,7 +305,8 @@ static int ipoib_create_qset ( struct ipoib_device *ipoib,
 	qset->recv_max_fill = num_recv_wqes;
 
 	/* Allocate completion queue */
-	qset->cq = ib_create_cq ( ibdev, num_cqes );
+	qset->cq = ib_create_cq ( ibdev, num_cqes, complete_send,
+				  complete_recv );
 	if ( ! qset->cq ) {
 		DBGC ( ipoib, "IPoIB %p could not allocate completion queue\n",
 		       ipoib );
@@ -759,10 +768,8 @@ static void ipoib_poll ( struct net_device *netdev ) {
 	struct ipoib_device *ipoib = netdev->priv;
 	struct ib_device *ibdev = ipoib->ibdev;
 
-	ib_poll_cq ( ibdev, ipoib->meta.cq, ipoib_meta_complete_send,
-		     ipoib_meta_complete_recv );
-	ib_poll_cq ( ibdev, ipoib->data.cq, ipoib_data_complete_send,
-		     ipoib_data_complete_recv );
+	ib_poll_cq ( ibdev, ipoib->meta.cq );
+	ib_poll_cq ( ibdev, ipoib->data.cq );
 	ipoib_refill_recv ( ipoib, &ipoib->meta );
 	ipoib_refill_recv ( ipoib, &ipoib->data );
 }
@@ -847,7 +854,9 @@ static int ipoib_open ( struct net_device *netdev ) {
 	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
 					IPOIB_META_NUM_CQES,
 					IPOIB_META_NUM_SEND_WQES,
+					ipoib_meta_complete_send,
 					IPOIB_META_NUM_RECV_WQES,
+					ipoib_meta_complete_recv,
 					IB_GLOBAL_QKEY ) ) != 0 ) {
 		DBGC ( ipoib, "IPoIB %p could not allocate metadata QP: %s\n",
 		       ipoib, strerror ( rc ) );
@@ -858,7 +867,9 @@ static int ipoib_open ( struct net_device *netdev ) {
 	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
 					IPOIB_DATA_NUM_CQES,
 					IPOIB_DATA_NUM_SEND_WQES,
+					ipoib_data_complete_send,
 					IPOIB_DATA_NUM_RECV_WQES,
+					ipoib_data_complete_recv,
 					IB_GLOBAL_QKEY ) ) != 0 ) {
 		DBGC ( ipoib, "IPoIB %p could not allocate data QP: %s\n",
 		       ipoib, strerror ( rc ) );
diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index f9e65348..82b7a0a8 100644
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -101,26 +101,6 @@ enum ib_queue_pair_mods {
 	IB_MODIFY_QKEY = 0x0001,
 };
 
-/** An Infiniband Completion Queue */
-struct ib_completion_queue {
-	/** Completion queue number */
-	unsigned long cqn;
-	/** Number of completion queue entries */
-	unsigned int num_cqes;
-	/** Next completion queue entry index
-	 *
-	 * This is the index of the next entry to be filled (i.e. the
-	 * first empty entry).  This value is not bounded by num_wqes;
-	 * users must logical-AND with (num_wqes-1) to generate an
-	 * array index.
-	 */
-	unsigned long next_idx;
-	/** List of work queues completing to this queue */
-	struct list_head work_queues;
-	/** Driver private data */
-	void *drv_priv;
-};
-
 /** An Infiniband completion */
 struct ib_completion {
 	/** Syndrome
@@ -132,6 +112,14 @@ struct ib_completion {
 	size_t len;
 };
 
+/** Infiniband completion syndromes */
+enum ib_syndrome {
+	IB_SYN_NONE = 0,
+	IB_SYN_LOCAL_LENGTH = 1,
+	IB_SYN_LOCAL_QP = 2,
+	IB_SYN_LOCAL_PROT = 4,
+};
+
 /** An Infiniband completion handler
  *
  * @v ibdev		Infiniband device
@@ -144,6 +132,30 @@ typedef void ( * ib_completer_t ) ( struct ib_device *ibdev,
 				    struct ib_completion *completion,
 				    struct io_buffer *iobuf );
 
+/** An Infiniband Completion Queue */
+struct ib_completion_queue {
+	/** Completion queue number */
+	unsigned long cqn;
+	/** Number of completion queue entries */
+	unsigned int num_cqes;
+	/** Next completion queue entry index
+	 *
+	 * This is the index of the next entry to be filled (i.e. the
+	 * first empty entry).  This value is not bounded by num_wqes;
+	 * users must logical-AND with (num_wqes-1) to generate an
+	 * array index.
+	 */
+	unsigned long next_idx;
+	/** List of work queues completing to this queue */
+	struct list_head work_queues;
+	/** Send completion handler */
+	ib_completer_t complete_send;
+	/** Receive completion handler */
+	ib_completer_t complete_recv;
+	/** Driver private data */
+	void *drv_priv;
+};
+
 /** An Infiniband Address Vector */
 struct ib_address_vector {
 	/** Destination Queue Pair */
@@ -246,15 +258,12 @@ struct ib_device_operations {
 	 *
 	 * @v ibdev		Infiniband device
 	 * @v cq		Completion queue
-	 * @v complete_send	Send completion handler
-	 * @v complete_recv	Receive completion handler
 	 *
-	 * The completion handler takes ownership of the I/O buffer.
+	 * The relevant completion handler (specified at completion
+	 * queue creation time) takes ownership of the I/O buffer.
 	 */
 	void ( * poll_cq ) ( struct ib_device *ibdev,
-			     struct ib_completion_queue *cq,
-			     ib_completer_t complete_send,
-			     ib_completer_t complete_recv );
+			     struct ib_completion_queue *cq );
 	/**
 	 * Poll event queue
 	 *
@@ -331,8 +340,9 @@ struct ib_device {
 	void *owner_priv;
 };
 
-extern struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
-						   unsigned int num_cqes );
+extern struct ib_completion_queue *
+ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
+	       ib_completer_t complete_send, ib_completer_t complete_recv );
 extern void ib_destroy_cq ( struct ib_device *ibdev,
 			    struct ib_completion_queue *cq );
 extern struct ib_queue_pair *
@@ -380,17 +390,44 @@ ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 }
 
 /**
+ * Complete send work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v completion	Completion
+ * @v iobuf		I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+ib_complete_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
+		   struct ib_completion *completion,
+		   struct io_buffer *iobuf ) {
+	return qp->send.cq->complete_send ( ibdev, qp, completion, iobuf );
+}
+
+/**
+ * Complete receive work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v qp		Queue pair
+ * @v completion	Completion
+ * @v iobuf		I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
+		   struct ib_completion *completion,
+		   struct io_buffer *iobuf ) {
+	return qp->recv.cq->complete_recv ( ibdev, qp, completion, iobuf );
+}
+
+/**
  * Poll completion queue
  *
  * @v ibdev		Infiniband device
  * @v cq		Completion queue
- * @v complete_send	Send completion handler
- * @v complete_recv	Receive completion handler
  */
 static inline __attribute__ (( always_inline )) void
-ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq,
-	     ib_completer_t complete_send, ib_completer_t complete_recv ) {
-	ibdev->op->poll_cq ( ibdev, cq, complete_send, complete_recv );
+ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
+	ibdev->op->poll_cq ( ibdev, cq );
 }
 
 /**
diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index ab76742e..8437b4ce 100644
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -46,10 +46,13 @@ struct list_head ib_devices = LIST_HEAD_INIT ( ib_devices );
  *
  * @v ibdev		Infiniband device
  * @v num_cqes		Number of completion queue entries
+ * @v complete_send	Send completion handler
+ * @v complete_recv	Receive completion handler
  * @ret cq		New completion queue
  */
-struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
-					    unsigned int num_cqes ) {
+struct ib_completion_queue *
+ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
+	       ib_completer_t complete_send, ib_completer_t complete_recv ) {
 	struct ib_completion_queue *cq;
 	int rc;
 
@@ -61,6 +64,8 @@ struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
 		return NULL;
 	cq->num_cqes = num_cqes;
 	INIT_LIST_HEAD ( &cq->work_queues );
+	cq->complete_send = complete_send;
+	cq->complete_recv = complete_recv;
 
 	/* Perform device-specific initialisation and get CQN */
 	if ( ( rc = ibdev->op->create_cq ( ibdev, cq ) ) != 0 ) {
@@ -190,11 +195,33 @@ int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
  * @v qp		Queue pair
  */
 void ib_destroy_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
+	struct ib_completion completion = {
+		.syndrome = IB_SYN_LOCAL_QP,
+	};
+	struct io_buffer *iobuf;
+	unsigned int i;
+
 	DBGC ( ibdev, "IBDEV %p destroying QPN %#lx\n",
 	       ibdev, qp->qpn );
+
+	/* Perform device-specific destruction */
 	ibdev->op->destroy_qp ( ibdev, qp );
+
+	/* Complete any remaining I/O buffers with errors */
+	for ( i = 0 ; i < qp->send.num_wqes ; i++ ) {
+		if ( ( iobuf = qp->send.iobufs[i] ) != NULL )
+			ib_complete_send ( ibdev, qp, &completion, iobuf );
+	}
+	for ( i = 0 ; i < qp->recv.num_wqes ; i++ ) {
+		if ( ( iobuf = qp->recv.iobufs[i] ) != NULL )
+			ib_complete_recv ( ibdev, qp, &completion, iobuf );
+	}
+
+	/* Remove work queues from completion queue */
 	list_del ( &qp->send.list );
 	list_del ( &qp->recv.list );
+
+	/* Free QP */
 	free ( qp );
 }