Read port GID directly using MAD IFC.

3 files changed, 242 insertions, 15 deletions

diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h
index c4b536a5..28893f51 100644
--- a/src/drivers/net/mlx_ipoib/arbel.h
+++ b/src/drivers/net/mlx_ipoib/arbel.h
@@ -33,6 +33,7 @@
 #define ARBEL_HCR_INIT2RTR_QPEE		0x001a
 #define ARBEL_HCR_RTR2RTS_QPEE		0x001b
 #define ARBEL_HCR_2RST_QPEE		0x0021
+#define ARBEL_HCR_MAD_IFC		0x0024
 #define ARBEL_HCR_READ_MGM		0x0025
 #define ARBEL_HCR_WRITE_MGM		0x0026
 #define ARBEL_HCR_MGID_HASH		0x0027
@@ -67,6 +68,7 @@ struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record );
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
 struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
+struct MLX_DECLARE_STRUCT ( arbelprm_mad_ifc );
 struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry );
 struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash );
 struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record );
@@ -126,6 +128,11 @@ union arbelprm_doorbell_register {
 	uint32_t dword[2];
 } __attribute__ (( packed ));
 
+union arbelprm_mad {
+	struct arbelprm_mad_ifc ifc;
+	union ib_mad mad;
+} __attribute__ (( packed ));
+
 /*
  * gPXE-specific definitions
  *
diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c
index 13b7d78b..aed6d208 100644
--- a/src/drivers/net/mlx_ipoib/mt25218.c
+++ b/src/drivers/net/mlx_ipoib/mt25218.c
@@ -550,6 +550,15 @@ arbel_cmd_2rst_qpee ( struct arbel *arbel, unsigned long qpn ) {
 }
 
 static inline int
+arbel_cmd_mad_ifc ( struct arbel *arbel, union arbelprm_mad *mad ) {
+	return arbel_cmd ( arbel,
+			   ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MAD_IFC,
+						 1, sizeof ( *mad ),
+						 1, sizeof ( *mad ) ),
+			   0x03, mad, PXE_IB_PORT, mad );
+}
+
+static inline int
 arbel_cmd_read_mgm ( struct arbel *arbel, unsigned int index,
 		     struct arbelprm_mgm_entry *mgm ) {
 	return arbel_cmd ( arbel,
@@ -1234,6 +1243,15 @@ static int arbel_complete ( struct ib_device *ibdev,
 }			     
 
 /**
+ * Drain event queue
+ *
+ * @v arbel		Arbel device
+ */
+static void arbel_drain_eq ( struct arbel *arbel ) {
+#warning "drain the event queue"
+}
+
+/**
  * Poll completion queue
  *
  * @v ibdev		Infiniband device
@@ -1252,6 +1270,9 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
 	unsigned int cqe_idx_mask;
 	int rc;
 
+	/* Drain the event queue */
+	arbel_drain_eq ( arbel );
+
 	while ( 1 ) {
 		/* Look for completion entry */
 		cqe_idx_mask = ( cq->num_cqes - 1 );
@@ -1377,8 +1398,6 @@ static void arbel_mcast_detach ( struct ib_device *ibdev,
 	}
 }
 
-
-
 /** Arbel Infiniband operations */
 static struct ib_device_operations arbel_ib_operations = {
 	.create_cq	= arbel_create_cq,
@@ -1392,20 +1411,83 @@ static struct ib_device_operations arbel_ib_operations = {
 	.mcast_detach	= arbel_mcast_detach,
 };
 
-/**
- * Remove PCI device
- *
- * @v pci		PCI device
- */
-static void arbel_remove ( struct pci_device *pci ) {
-	struct net_device *netdev = pci_get_drvdata ( pci );
 
-	unregister_netdev ( netdev );
-	ib_driver_close ( 0 );
-	netdev_nullify ( netdev );
-	netdev_put ( netdev );
+static int arbel_mad_ifc ( struct arbel *arbel,
+			   union arbelprm_mad *mad ) {
+	struct ib_mad_hdr *hdr = &mad->mad.mad_hdr;
+	int rc;
+
+	hdr->base_version = IB_MGMT_BASE_VERSION;
+	if ( ( rc = arbel_cmd_mad_ifc ( arbel, mad ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
+		       arbel, strerror ( rc ) );
+		return rc;
+	}
+	if ( hdr->status != 0 ) {
+		DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
+		       arbel, ntohs ( hdr->status ) );
+		return -EIO;
+	}
+	return 0;
+}
+
+static int arbel_get_port_info ( struct arbel *arbel,
+				 struct ib_mad_port_info *port_info ) {
+	union arbelprm_mad mad;
+	struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
+	int rc;
+
+	memset ( &mad, 0, sizeof ( mad ) );
+	hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+	hdr->class_version = 1;
+	hdr->method = IB_MGMT_METHOD_GET;
+	hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
+	hdr->attr_mod = htonl ( PXE_IB_PORT );
+	if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p could not get port info: %s\n",
+		       arbel, strerror ( rc ) );
+		return rc;
+	}
+	memcpy ( port_info, &mad.mad.port_info, sizeof ( *port_info ) );
+	return 0;
+}
+
+static int arbel_get_guid_info ( struct arbel *arbel,
+				 struct ib_mad_guid_info *guid_info ) {
+	union arbelprm_mad mad;
+	struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
+	int rc;
+
+	memset ( &mad, 0, sizeof ( mad ) );
+	hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+	hdr->class_version = 1;
+	hdr->method = IB_MGMT_METHOD_GET;
+	hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
+	if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p could not get GUID info: %s\n",
+		       arbel, strerror ( rc ) );
+		return rc;
+	}
+	memcpy ( guid_info, &mad.mad.guid_info, sizeof ( *guid_info ) );
+	return 0;
 }
 
+static int arbel_get_port_gid ( struct arbel *arbel, struct ib_gid *gid ) {
+	struct ib_mad_port_info port_info;
+	struct ib_mad_guid_info guid_info;
+	int rc;
+
+	if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
+		return rc;
+	if ( ( rc = arbel_get_guid_info ( arbel, &guid_info ) ) != 0 )
+		return rc;
+	memcpy ( &gid->bytes[0], port_info.gid_prefix, 8 );
+	memcpy ( &gid->bytes[8], guid_info.gid_local, 8 );
+	return 0;
+}
+
+
+
 /**
  * Probe PCI device
  *
@@ -1514,11 +1596,20 @@ static int arbel_probe ( struct pci_device *pci,
 		      strerror ( rc ) );
 		return rc;
 	}
-				      
+
+	if ( ( rc = arbel_get_port_gid ( arbel, &ibdev->port_gid ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p could not determine port GID: %s\n",
+		       arbel, strerror ( rc ) );
+		return rc;
+	}
+
+	DBG ( "Port GID:\n" );
+	DBG_HD ( &ibdev->port_gid, sizeof ( ibdev->port_gid ) );
+		
 
 	mac = ( ( struct ib_mac * ) netdev->ll_addr );
 	mac->qpn = htonl ( mlx->own_qp->qpn );
-	memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) );	
+	memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) );	
 #endif
 
 #if 0
@@ -1545,6 +1636,20 @@ static int arbel_probe ( struct pci_device *pci,
 	return rc;
 }
 
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void arbel_remove ( struct pci_device *pci ) {
+	struct net_device *netdev = pci_get_drvdata ( pci );
+
+	unregister_netdev ( netdev );
+	ib_driver_close ( 0 );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
 static struct pci_device_id arbel_nics[] = {
 	PCI_ROM ( 0x15b3, 0x6282, "MT25218", "MT25218 HCA driver" ),
 	PCI_ROM ( 0x15b3, 0x6274, "MT25204", "MT25204 HCA driver" ),
diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 6a38a1b8..3f09808c 100644
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -277,6 +277,8 @@ struct ib_device_operations {
 
 /** An Infiniband device */
 struct ib_device {	
+	/** Port GID */
+	struct ib_gid port_gid;
 	/** Infiniband operations */
 	struct ib_device_operations *op;
 	/** Device private data */
@@ -323,6 +325,119 @@ ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	ibdev->op->mcast_detach ( ibdev, qp, gid );
 }
 
+/*****************************************************************************
+ *
+ * Management datagrams
+ *
+ * Portions Copyright (c) 2004 Mellanox Technologies Ltd.  All rights
+ * reserved.
+ *
+ */
+
+/* Management base version */
+#define IB_MGMT_BASE_VERSION			1
+
+/* Management classes */
+#define IB_MGMT_CLASS_SUBN_LID_ROUTED		0x01
+#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE	0x81
+#define IB_MGMT_CLASS_SUBN_ADM			0x03
+#define IB_MGMT_CLASS_PERF_MGMT			0x04
+#define IB_MGMT_CLASS_BM			0x05
+#define IB_MGMT_CLASS_DEVICE_MGMT		0x06
+#define IB_MGMT_CLASS_CM			0x07
+#define IB_MGMT_CLASS_SNMP			0x08
+#define IB_MGMT_CLASS_VENDOR_RANGE2_START	0x30
+#define IB_MGMT_CLASS_VENDOR_RANGE2_END		0x4F
+
+/* Management methods */
+#define IB_MGMT_METHOD_GET			0x01
+#define IB_MGMT_METHOD_SET			0x02
+#define IB_MGMT_METHOD_GET_RESP			0x81
+#define IB_MGMT_METHOD_SEND			0x03
+#define IB_MGMT_METHOD_TRAP			0x05
+#define IB_MGMT_METHOD_REPORT			0x06
+#define IB_MGMT_METHOD_REPORT_RESP		0x86
+#define IB_MGMT_METHOD_TRAP_REPRESS		0x07
+#define IB_MGMT_METHOD_DELETE			0x15
+#define IB_MGMT_METHOD_RESP			0x80
+
+/* Subnet management attributes */
+#define IB_SMP_ATTR_NOTICE			0x0002
+#define IB_SMP_ATTR_NODE_DESC			0x0010
+#define IB_SMP_ATTR_NODE_INFO			0x0011
+#define IB_SMP_ATTR_SWITCH_INFO			0x0012
+#define IB_SMP_ATTR_GUID_INFO			0x0014
+#define IB_SMP_ATTR_PORT_INFO			0x0015
+#define IB_SMP_ATTR_PKEY_TABLE			0x0016
+#define IB_SMP_ATTR_SL_TO_VL_TABLE		0x0017
+#define IB_SMP_ATTR_VL_ARB_TABLE		0x0018
+#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE	0x0019
+#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE	0x001A
+#define IB_SMP_ATTR_MCAST_FORWARD_TABLE		0x001B
+#define IB_SMP_ATTR_SM_INFO			0x0020
+#define IB_SMP_ATTR_VENDOR_DIAG			0x0030
+#define IB_SMP_ATTR_LED_INFO			0x0031
+#define IB_SMP_ATTR_VENDOR_MASK			0xFF00
+
+struct ib_mad_hdr {
+	uint8_t base_version;
+	uint8_t mgmt_class;
+	uint8_t class_version;
+	uint8_t method;
+	uint16_t status;
+	uint16_t class_specific;
+	uint64_t tid;
+	uint16_t attr_id;
+	uint16_t resv;
+	uint32_t attr_mod;
+} __attribute__ (( packed ));
+
+struct ib_mad_data {
+	struct ib_mad_hdr mad_hdr;
+	uint8_t data[232];
+} __attribute__ (( packed ));
+
+struct ib_mad_guid_info {
+	struct ib_mad_hdr mad_hdr;
+	uint32_t mkey[2];
+	uint32_t reserved[8];
+	uint8_t gid_local[8];
+} __attribute__ (( packed ));
+
+struct ib_mad_port_info {
+	struct ib_mad_hdr mad_hdr;
+	uint32_t mkey[2];
+	uint32_t reserved[8];
+	uint32_t mkey2[2];
+	uint8_t gid_prefix[8];
+	uint16_t lid;
+	uint16_t mastersm_lid;
+	uint32_t cap_mask;
+	uint16_t diag_code;
+	uint16_t mkey_lease_period;
+	uint8_t local_port_num;
+	uint8_t link_width_enabled;
+	uint8_t link_width_supported;
+	uint8_t link_width_active;
+	uint8_t port_state__link_speed_supported;
+	uint8_t link_down_def_state__port_phys_state;
+	uint8_t lmc__r1__mkey_prot_bits;
+	uint8_t link_speed_enabled__link_speed_active;
+} __attribute__ (( packed ));
+
+union ib_mad {
+	struct ib_mad_hdr mad_hdr;
+	struct ib_mad_data data;
+	struct ib_mad_guid_info guid_info;
+	struct ib_mad_port_info port_info;
+} __attribute__ (( packed ));
+
+
+
+
+
+
+
 
 extern struct ll_protocol infiniband_protocol;