[eoib] Support non-FullMember gateway devices

Some EoIB implementations utilise an EoIB-to-Ethernet gateway device
that does not perform a FullMember join to the multicast group for the
EoIB broadcast domain.  This has various exciting side-effects, such
as requiring every EoIB node to send every broadcast packet twice.

As an added bonus, the gateway may also break the EoIB MAC address to
GID mapping protocol by sending Ethernet-sourced packets from the
wrong QPN.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

2 files changed, 120 insertions, 2 deletions

diff --git a/src/drivers/net/eoib.c b/src/drivers/net/eoib.c
index a5f9e29ee..f58e74b7d 100644
--- a/src/drivers/net/eoib.c
+++ b/src/drivers/net/eoib.c
@@ -269,6 +269,14 @@ static void eoib_rx_av ( struct eoib_device *eoib, const uint8_t *mac,
 		return;
 	}
 
+	/* Some dubious EoIB implementations utilise an Ethernet-to-
+	 * EoIB gateway that will send packets from the wrong QPN.
+	 */
+	if ( eoib_has_gateway ( eoib ) &&
+	     ( memcmp ( gid, &eoib->gateway.gid, sizeof ( *gid ) ) == 0 ) ) {
+		qpn = eoib->gateway.qpn;
+	}
+
 	/* Do nothing if peer cache entry is complete and correct */
 	if ( ( peer->av.lid == av->lid ) && ( peer->av.qpn == qpn ) ) {
 		DBGCP ( eoib, "EoIB %s %s RX unchanged\n",
@@ -326,9 +334,14 @@ static int eoib_transmit ( struct net_device *netdev,
 	if ( iob_len ( iobuf ) < zlen )
 		iob_pad ( iobuf, zlen );
 
-	/* If we have no unicast address then send as a broadcast */
-	if ( ! av )
+	/* If we have no unicast address then send as a broadcast,
+	 * with a duplicate sent to the gateway if applicable.
+	 */
+	if ( ! av ) {
 		av = &eoib->broadcast;
+		if ( eoib_has_gateway ( eoib ) )
+			eoib->duplicate ( eoib, iobuf );
+	}
 
 	/* Post send work queue entry */
 	return ib_post_send ( eoib->ibdev, eoib->qp, av, iobuf );
@@ -797,3 +810,86 @@ struct net_protocol eoib_heartbeat_protocol __net_protocol = {
 	.rx = eoib_heartbeat_rx,
 	.ntoa = eoib_heartbeat_ntoa,
 };
+
+/****************************************************************************
+ *
+ * EoIB gateway
+ *
+ ****************************************************************************
+ *
+ * Some dubious EoIB implementations require all broadcast traffic to
+ * be sent twice: once to the actual broadcast group, and once as a
+ * unicast to the EoIB-to-Ethernet gateway.  This somewhat curious
+ * design arises since the EoIB-to-Ethernet gateway hardware lacks the
+ * ability to attach a queue pair to a multicast GID (or LID), and so
+ * cannot receive traffic sent to the broadcast group.
+ *
+ */
+
+/**
+ * Transmit duplicate packet to the EoIB gateway
+ *
+ * @v eoib		EoIB device
+ * @v original		Original I/O buffer
+ */
+static void eoib_duplicate ( struct eoib_device *eoib,
+			     struct io_buffer *original ) {
+	struct net_device *netdev = eoib->netdev;
+	struct ib_device *ibdev = eoib->ibdev;
+	struct ib_address_vector *av = &eoib->gateway;
+	size_t len = iob_len ( original );
+	struct io_buffer *copy;
+	int rc;
+
+	/* Create copy of I/O buffer */
+	copy = alloc_iob ( len );
+	if ( ! copy ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memcpy ( iob_put ( copy, len ), original->data, len );
+
+	/* Append to network device's transmit queue */
+	list_add_tail ( &copy->list, &original->list );
+
+	/* Resolve path to gateway */
+	if ( ( rc = ib_resolve_path ( ibdev, av ) ) != 0 ) {
+		DBGC ( eoib, "EoIB %s no path to gateway: %s\n",
+		       eoib->name, strerror ( rc ) );
+		goto err_path;
+	}
+
+	/* Force use of GRH even for local destinations */
+	av->gid_present = 1;
+
+	/* Post send work queue entry */
+	if ( ( rc = ib_post_send ( eoib->ibdev, eoib->qp, av, copy ) ) != 0 )
+		goto err_post_send;
+
+	return;
+
+ err_post_send:
+ err_path:
+ err_alloc:
+	netdev_tx_complete_err ( netdev, copy, rc );
+}
+
+/**
+ * Set EoIB gateway
+ *
+ * @v eoib		EoIB device
+ * @v av		Address vector, or NULL to clear gateway
+ */
+void eoib_set_gateway ( struct eoib_device *eoib,
+			struct ib_address_vector *av ) {
+
+	if ( av ) {
+		DBGC ( eoib, "EoIB %s using gateway " IB_GID_FMT "\n",
+		       eoib->name, IB_GID_ARGS ( &av->gid ) );
+		memcpy ( &eoib->gateway, av, sizeof ( eoib->gateway ) );
+		eoib->duplicate = eoib_duplicate;
+	} else {
+		DBGC ( eoib, "EoIB %s not using gateway\n", eoib->name );
+		eoib->duplicate = NULL;
+	}
+}
diff --git a/src/include/ipxe/eoib.h b/src/include/ipxe/eoib.h
index acae542b6..93f496c36 100644
--- a/src/include/ipxe/eoib.h
+++ b/src/include/ipxe/eoib.h
@@ -49,11 +49,31 @@ struct eoib_device {
 	/** Peer cache */
 	struct list_head peers;
 
+	/** Send duplicate packet to gateway (or NULL)
+	 *
+	 * @v eoib		EoIB device
+	 * @v original		Original I/O buffer
+	 */
+	void ( * duplicate ) ( struct eoib_device *eoib,
+			       struct io_buffer *original );
+	/** Gateway (if any) */
+	struct ib_address_vector gateway;
 	/** Multicast group additional component mask */
 	unsigned int mask;
 };
 
 /**
+ * Check if EoIB device uses a gateway
+ *
+ * @v eoib		EoIB device
+ * @v has_gw		EoIB device uses a gateway
+ */
+static inline int eoib_has_gateway ( struct eoib_device *eoib ) {
+
+	return ( eoib->duplicate != NULL );
+}
+
+/**
  * Force creation of multicast group
  *
  * @v eoib		EoIB device
@@ -77,5 +97,7 @@ extern int eoib_create ( struct ib_device *ibdev, const uint8_t *hw_addr,
 extern struct eoib_device * eoib_find ( struct ib_device *ibdev,
 					const uint8_t *hw_addr );
 extern void eoib_destroy ( struct eoib_device *eoib );
+extern void eoib_set_gateway ( struct eoib_device *eoib,
+			       struct ib_address_vector *av );
 
 #endif /* _IPXE_EOIB_H */