[hermon] Allow software GMA to receive packets destined for QP1

The Linux IB Communication Manager will always send MADs to QP1,
rather than back to the originating QP.  On Hermon, QP1 is by default
handled by the embedded firmware.  We can change this, but the cost is
that we have to handle both QP0 and QP1 (i.e. we have to provide SMA
as well as GMA service in software), and we have to use MLX queues
rather than standard UD queues (i.e. we have to construct the UD
datagrams by hand).

There doesn't seem to be any viable way around this situation, ugly
though it is.

1 files changed, 30 insertions, 1 deletions

diff --git a/src/drivers/infiniband/hermon.h b/src/drivers/infiniband/hermon.h
index 5d92efdf8..cd805fd0f 100644
--- a/src/drivers/infiniband/hermon.h
+++ b/src/drivers/infiniband/hermon.h
@@ -11,6 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <stdint.h>
 #include <gpxe/uaccess.h>
+#include <gpxe/ib_packet.h>
 #include "mlx_bitops.h"
 #include "MT25408_PRM.h"
 
@@ -53,6 +54,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define HERMON_HCR_RTR2RTS_QP		0x001b
 #define HERMON_HCR_RTS2RTS_QP		0x001c
 #define HERMON_HCR_2RST_QP		0x0021
+#define HERMON_HCR_CONF_SPECIAL_QP	0x0023
 #define HERMON_HCR_MAD_IFC		0x0024
 #define HERMON_HCR_READ_MCG		0x0025
 #define HERMON_HCR_WRITE_MCG		0x0026
@@ -70,6 +72,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 /* Service types */
 #define HERMON_ST_UD			0x03
+#define HERMON_ST_MLX			0x07
 
 /* MTUs */
 #define HERMON_MTU_2048			0x04
@@ -89,6 +92,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #define HERMON_EV_PORT_STATE_CHANGE	0x09
 
+#define HERMON_SCHED_QP0		0x3f
+#define HERMON_SCHED_DEFAULT		0x83
+
 /*
  * Datatypes that seem to be missing from the autogenerated documentation
  *
@@ -178,6 +184,7 @@ struct MLX_DECLARE_STRUCT ( hermonprm_scalar_parameter );
 struct MLX_DECLARE_STRUCT ( hermonprm_send_db_register );
 struct MLX_DECLARE_STRUCT ( hermonprm_ud_address_vector );
 struct MLX_DECLARE_STRUCT ( hermonprm_virtual_physical_mapping );
+struct MLX_DECLARE_STRUCT ( hermonprm_wqe_segment_ctrl_mlx );
 struct MLX_DECLARE_STRUCT ( hermonprm_wqe_segment_ctrl_send );
 struct MLX_DECLARE_STRUCT ( hermonprm_wqe_segment_data_ptr );
 struct MLX_DECLARE_STRUCT ( hermonprm_wqe_segment_ud );
@@ -193,7 +200,7 @@ struct hermonprm_write_mtt {
 	struct hermonprm_mtt mtt;
 } __attribute__ (( packed ));
 
-#define HERMON_MAX_GATHER 1
+#define HERMON_MAX_GATHER 2
 
 struct hermonprm_ud_send_wqe {
 	struct hermonprm_wqe_segment_ctrl_send ctrl;
@@ -201,6 +208,12 @@ struct hermonprm_ud_send_wqe {
 	struct hermonprm_wqe_segment_data_ptr data[HERMON_MAX_GATHER];
 } __attribute__ (( packed ));
 
+struct hermonprm_mlx_send_wqe {
+	struct hermonprm_wqe_segment_ctrl_mlx ctrl;
+	struct hermonprm_wqe_segment_data_ptr data[HERMON_MAX_GATHER];
+	uint8_t headers[IB_MAX_HEADER_SIZE];
+} __attribute__ (( packed ));
+
 #define HERMON_MAX_SCATTER 1
 
 struct hermonprm_recv_wqe {
@@ -320,7 +333,9 @@ struct hermon_mtt {
 
 /** A Hermon send work queue entry */
 union hermon_send_wqe {
+	struct hermonprm_wqe_segment_ctrl_send ctrl;
 	struct hermonprm_ud_send_wqe ud;
+	struct hermonprm_mlx_send_wqe mlx;
 	uint8_t force_align[HERMON_SEND_WQE_ALIGN];
 } __attribute__ (( packed ));
 
@@ -360,6 +375,16 @@ struct hermon_recv_work_queue {
 	struct hermonprm_qp_db_record doorbell __attribute__ (( aligned (4) ));
 };
 
+/** Number of special queue pairs */
+#define HERMON_NUM_SPECIAL_QPS 8
+
+/** Number of queue pairs reserved for the "special QP" block
+ *
+ * The special QPs must be within a contiguous block aligned on its
+ * own size.
+ */
+#define HERMON_RSVD_SPECIAL_QPS	( ( HERMON_NUM_SPECIAL_QPS << 1 ) - 1 )
+
 /** Maximum number of allocatable queue pairs
  *
  * This is a policy decision, not a device limit.
@@ -475,6 +500,10 @@ struct hermon {
 
 	/** Device capabilities */
 	struct hermon_dev_cap cap;
+	/** Special QPN base */
+	unsigned long special_qpn_base;
+	/** QPN base */
+	unsigned long qpn_base;
 
 	/** Infiniband devices */
 	struct ib_device *ibdev[HERMON_NUM_PORTS];