From 663904a7bc4ced3b80d87df125b0e7cf90c1bfc2 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Thu, 6 Nov 2008 22:31:19 +0000 Subject: [infiniband] Split subnet management agent client out into ib_smc.c Not all Infiniband cards have embedded subnet management agents. Split out the code that communicates with such an embedded SMA into a separate ib_smc.c file, and have drivers call ib_smc_update() explicitly when they suspect that the answers given by the embedded SMA may have changed. --- src/drivers/infiniband/arbel.c | 97 ++++++------ src/drivers/infiniband/hermon.c | 97 ++++++------ src/drivers/infiniband/ib_smc.c | 166 +++++++++++++++++++ src/drivers/net/ipoib.c | 18 ++- src/include/gpxe/ib_mad.h | 341 ++++++++++++++++++++++++++++++++++++++++ src/include/gpxe/ib_packet.h | 114 ++++++++++++++ src/include/gpxe/ib_smc.h | 18 +++ src/include/gpxe/infiniband.h | 323 ++++++------------------------------- src/net/infiniband.c | 151 +----------------- 9 files changed, 805 insertions(+), 520 deletions(-) create mode 100644 src/drivers/infiniband/ib_smc.c create mode 100644 src/include/gpxe/ib_mad.h create mode 100644 src/include/gpxe/ib_packet.h create mode 100644 src/include/gpxe/ib_smc.h diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c index 8a31ad8b..e516031b 100644 --- a/src/drivers/infiniband/arbel.c +++ b/src/drivers/infiniband/arbel.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "arbel.h" /** @@ -482,6 +483,50 @@ arbel_cmd_map_fa ( struct arbel *arbel, 0, map, 1, NULL ); } +/*************************************************************************** + * + * MAD operations + * + *************************************************************************** + */ + +/** + * Issue management datagram + * + * @v ibdev Infiniband device + * @v mad Management datagram + * @ret rc Return status code + */ +static int arbel_mad ( struct ib_device *ibdev, union ib_mad *mad ) { + struct arbel *arbel = ib_get_drvdata ( ibdev ); + union arbelprm_mad mad_ifc; + int rc; + + linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ), + mad_size_mismatch ); + + /* Copy in request packet */ + memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) ); + + /* Issue MAD */ + if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port, + &mad_ifc ) ) != 0 ) { + DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n", + arbel, strerror ( rc ) ); + return rc; + } + + /* Copy out reply packet */ + memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) ); + + if ( mad->hdr.status != 0 ) { + DBGC ( arbel, "Arbel %p MAD IFC status %04x\n", + arbel, ntohs ( mad->hdr.status ) ); + return -EIO; + } + return 0; +} + /*************************************************************************** * * Completion queue operations @@ -1394,6 +1439,9 @@ static void arbel_event_port_state_change ( struct arbel *arbel, return; } + /* Update MAD parameters */ + ib_smc_update ( arbel->ibdev[port], arbel_mad ); + /* Notify Infiniband core of link state change */ ib_link_state_changed ( arbel->ibdev[port] ); } @@ -1483,6 +1531,9 @@ static int arbel_open ( struct ib_device *ibdev ) { return rc; } + /* Update MAD parameters */ + ib_smc_update ( ibdev, arbel_mad ); + return 0; } @@ -1598,51 +1649,6 @@ static void arbel_mcast_detach ( struct ib_device *ibdev, } } -/*************************************************************************** - * - * MAD operations - * - *************************************************************************** - */ - -/** - * Issue management datagram - * - * @v ibdev Infiniband device - * @v mad Management datagram - * @v len Length of management datagram - * @ret rc Return status code - */ -static int arbel_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, - size_t len ) { - struct arbel *arbel = ib_get_drvdata ( ibdev ); - union arbelprm_mad mad_ifc; - int rc; - - /* Copy in request packet */ - memset ( &mad_ifc, 0, sizeof ( mad_ifc ) ); - assert ( len <= sizeof ( mad_ifc.mad ) ); - memcpy ( &mad_ifc.mad, mad, len ); - - /* Issue MAD */ - if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port, - &mad_ifc ) ) != 0 ) { - DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n", - arbel, strerror ( rc ) ); - return rc; - } - - /* Copy out reply packet */ - memcpy ( mad, &mad_ifc.mad, len ); - - if ( mad->status != 0 ) { - DBGC ( arbel, "Arbel %p MAD IFC status %04x\n", - arbel, ntohs ( mad->status ) ); - return -EIO; - } - return 0; -} - /** Arbel Infiniband operations */ static struct ib_device_operations arbel_ib_operations = { .create_cq = arbel_create_cq, @@ -1658,7 +1664,6 @@ static struct ib_device_operations arbel_ib_operations = { .close = arbel_close, .mcast_attach = arbel_mcast_attach, .mcast_detach = arbel_mcast_detach, - .mad = arbel_mad, }; /*************************************************************************** diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c index db7619ad..01eab396 100644 --- a/src/drivers/infiniband/hermon.c +++ b/src/drivers/infiniband/hermon.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "hermon.h" /** @@ -608,6 +609,50 @@ static void hermon_free_mtt ( struct hermon *hermon, mtt->num_pages ); } +/*************************************************************************** + * + * MAD operations + * + *************************************************************************** + */ + +/** + * Issue management datagram + * + * @v ibdev Infiniband device + * @v mad Management datagram + * @ret rc Return status code + */ +static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) { + struct hermon *hermon = ib_get_drvdata ( ibdev ); + union hermonprm_mad mad_ifc; + int rc; + + linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ), + mad_size_mismatch ); + + /* Copy in request packet */ + memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) ); + + /* Issue MAD */ + if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port, + &mad_ifc ) ) != 0 ) { + DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n", + hermon, strerror ( rc ) ); + return rc; + } + + /* Copy out reply packet */ + memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) ); + + if ( mad->hdr.status != 0 ) { + DBGC ( hermon, "Hermon %p MAD IFC status %04x\n", + hermon, ntohs ( mad->hdr.status ) ); + return -EIO; + } + return 0; +} + /*************************************************************************** * * Completion queue operations @@ -1377,6 +1422,9 @@ static void hermon_event_port_state_change ( struct hermon *hermon, return; } + /* Update MAD parameters */ + ib_smc_update ( hermon->ibdev[port], hermon_mad ); + /* Notify Infiniband core of link state change */ ib_link_state_changed ( hermon->ibdev[port] ); } @@ -1465,6 +1513,9 @@ static int hermon_open ( struct ib_device *ibdev ) { return rc; } + /* Update MAD parameters */ + ib_smc_update ( ibdev, hermon_mad ); + return 0; } @@ -1579,51 +1630,6 @@ static void hermon_mcast_detach ( struct ib_device *ibdev, } } -/*************************************************************************** - * - * MAD operations - * - *************************************************************************** - */ - -/** - * Issue management datagram - * - * @v ibdev Infiniband device - * @v mad Management datagram - * @v len Length of management datagram - * @ret rc Return status code - */ -static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, - size_t len ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - union hermonprm_mad mad_ifc; - int rc; - - /* Copy in request packet */ - memset ( &mad_ifc, 0, sizeof ( mad_ifc ) ); - assert ( len <= sizeof ( mad_ifc.mad ) ); - memcpy ( &mad_ifc.mad, mad, len ); - - /* Issue MAD */ - if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port, - &mad_ifc ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - - /* Copy out reply packet */ - memcpy ( mad, &mad_ifc.mad, len ); - - if ( mad->status != 0 ) { - DBGC ( hermon, "Hermon %p MAD IFC status %04x\n", - hermon, ntohs ( mad->status ) ); - return -EIO; - } - return 0; -} - /** Hermon Infiniband operations */ static struct ib_device_operations hermon_ib_operations = { .create_cq = hermon_create_cq, @@ -1639,7 +1645,6 @@ static struct ib_device_operations hermon_ib_operations = { .close = hermon_close, .mcast_attach = hermon_mcast_attach, .mcast_detach = hermon_mcast_detach, - .mad = hermon_mad, }; /*************************************************************************** diff --git a/src/drivers/infiniband/ib_smc.c b/src/drivers/infiniband/ib_smc.c new file mode 100644 index 00000000..c07a4fef --- /dev/null +++ b/src/drivers/infiniband/ib_smc.c @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2008 Michael Brown . + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @file + * + * Infiniband Subnet Management Client + * + */ + +/** + * Get port information + * + * @v ibdev Infiniband device + * @v local_mad Method for issuing local MADs + * @v mad Management datagram to fill in + * @ret rc Return status code + */ +static int ib_smc_get_port_info ( struct ib_device *ibdev, + ib_local_mad_t local_mad, + union ib_mad *mad ) { + int rc; + + /* Construct MAD */ + memset ( mad, 0, sizeof ( *mad ) ); + mad->hdr.base_version = IB_MGMT_BASE_VERSION; + mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + mad->hdr.class_version = 1; + mad->hdr.method = IB_MGMT_METHOD_GET; + mad->hdr.attr_id = htons ( IB_SMP_ATTR_PORT_INFO ); + mad->hdr.attr_mod = htonl ( ibdev->port ); + + if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p could not get port info: %s\n", + ibdev, strerror ( rc ) ); + return rc; + } + return 0; +} + +/** + * Get GUID information + * + * @v ibdev Infiniband device + * @v local_mad Method for issuing local MADs + * @v mad Management datagram to fill in + * @ret rc Return status code + */ +static int ib_smc_get_guid_info ( struct ib_device *ibdev, + ib_local_mad_t local_mad, + union ib_mad *mad ) { + int rc; + + /* Construct MAD */ + memset ( mad, 0, sizeof ( *mad ) ); + mad->hdr.base_version = IB_MGMT_BASE_VERSION; + mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + mad->hdr.class_version = 1; + mad->hdr.method = IB_MGMT_METHOD_GET; + mad->hdr.attr_id = htons ( IB_SMP_ATTR_GUID_INFO ); + + if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n", + ibdev, strerror ( rc ) ); + return rc; + } + return 0; +} + +/** + * Get partition key table + * + * @v ibdev Infiniband device + * @v local_mad Method for issuing local MADs + * @v mad Management datagram to fill in + * @ret rc Return status code + */ +static int ib_smc_get_pkey_table ( struct ib_device *ibdev, + ib_local_mad_t local_mad, + union ib_mad *mad ) { + int rc; + + /* Construct MAD */ + memset ( mad, 0, sizeof ( *mad ) ); + mad->hdr.base_version = IB_MGMT_BASE_VERSION; + mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + mad->hdr.class_version = 1; + mad->hdr.method = IB_MGMT_METHOD_GET; + mad->hdr.attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE ); + + if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n", + ibdev, strerror ( rc ) ); + return rc; + } + return 0; +} + +/** + * Get MAD parameters + * + * @v ibdev Infiniband device + * @v local_mad Method for issuing local MADs + * @ret rc Return status code + */ +int ib_smc_update ( struct ib_device *ibdev, ib_local_mad_t local_mad ) { + union ib_mad mad; + union ib_smp_data *smp = &mad.smp.smp_data; + int rc; + + /* Port info gives us the link state, the first half of the + * port GID and the SM LID. + */ + if ( ( rc = ib_smc_get_port_info ( ibdev, local_mad, &mad ) ) != 0 ) + return rc; + ibdev->port_state = + ( smp->port_info.link_speed_supported__port_state & 0x0f ); + memcpy ( &ibdev->gid.u.half[0], smp->port_info.gid_prefix, + sizeof ( ibdev->gid.u.half[0] ) ); + ibdev->lid = ntohs ( smp->port_info.lid ); + ibdev->sm_lid = ntohs ( smp->port_info.mastersm_lid ); + ibdev->sm_sl = ( smp->port_info.neighbour_mtu__mastersm_sl & 0xf ); + + /* GUID info gives us the second half of the port GID */ + if ( ( rc = ib_smc_get_guid_info ( ibdev, local_mad, &mad ) ) != 0 ) + return rc; + memcpy ( &ibdev->gid.u.half[1], smp->guid_info.guid[0], + sizeof ( ibdev->gid.u.half[1] ) ); + + /* Get partition key */ + if ( ( rc = ib_smc_get_pkey_table ( ibdev, local_mad, &mad ) ) != 0 ) + return rc; + ibdev->pkey = ntohs ( smp->pkey_table.pkey[0] ); + + DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n", ibdev, + htonl ( ibdev->gid.u.dwords[0] ), + htonl ( ibdev->gid.u.dwords[1] ), + htonl ( ibdev->gid.u.dwords[2] ), + htonl ( ibdev->gid.u.dwords[3] ) ); + + return 0; +} diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index aa68fe38..8a26f162 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -383,12 +383,13 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib, path_record->sa_hdr.comp_mask[1] = htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID ); memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) ); - memcpy ( &path_record->sgid, &ibdev->port_gid, + memcpy ( &path_record->sgid, &ibdev->gid, sizeof ( path_record->sgid ) ); /* Construct address vector */ memset ( &av, 0, sizeof ( av ) ); av.lid = ibdev->sm_lid; + av.sl = ibdev->sm_sl; av.qpn = IB_SA_QPN; av.qkey = IB_GLOBAL_QKEY; @@ -443,12 +444,13 @@ static int ipoib_mc_member_record ( struct ipoib_device *ipoib, mc_member_record->scope__join_state = 1; memcpy ( &mc_member_record->mgid, gid, sizeof ( mc_member_record->mgid ) ); - memcpy ( &mc_member_record->port_gid, &ibdev->port_gid, + memcpy ( &mc_member_record->port_gid, &ibdev->gid, sizeof ( mc_member_record->port_gid ) ); /* Construct address vector */ memset ( &av, 0, sizeof ( av ) ); av.lid = ibdev->sm_lid; + av.sl = ibdev->sm_sl; av.qpn = IB_SA_QPN; av.qkey = IB_GLOBAL_QKEY; @@ -491,7 +493,7 @@ static int ipoib_transmit ( struct net_device *netdev, /* Attempting transmission while link is down will put the * queue pair into an error state, so don't try it. */ - if ( ! ibdev->link_up ) + if ( ! ib_link_ok ( ibdev ) ) return -ENETUNREACH; /* Construct address vector */ @@ -691,13 +693,13 @@ ipoib_meta_complete_recv ( struct ib_device *ibdev __unused, } mad = iobuf->data; - if ( mad->mad_hdr.status != 0 ) { + if ( mad->hdr.status != 0 ) { DBGC ( ipoib, "IPoIB %p metadata RX err status %04x\n", - ipoib, ntohs ( mad->mad_hdr.status ) ); + ipoib, ntohs ( mad->hdr.status ) ); goto done; } - switch ( mad->mad_hdr.tid[0] ) { + switch ( mad->hdr.tid[0] ) { case IPOIB_TID_GET_PATH_REC: ipoib_recv_path_record ( ipoib, &mad->path_record ); break; @@ -928,7 +930,7 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) { /* Calculate GID portion of MAC address based on port GID */ mac = ( ( struct ipoib_mac * ) netdev->ll_addr ); - memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) ); + memcpy ( &mac->gid, &ibdev->gid, sizeof ( mac->gid ) ); /* Calculate broadcast GID based on partition key */ memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid, @@ -936,7 +938,7 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) { ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey ); /* Set net device link state to reflect Infiniband link state */ - if ( ibdev->link_up ) { + if ( ib_link_ok ( ibdev ) ) { netdev_link_up ( netdev ); } else { netdev_link_down ( netdev ); diff --git a/src/include/gpxe/ib_mad.h b/src/include/gpxe/ib_mad.h new file mode 100644 index 00000000..dbaf09a2 --- /dev/null +++ b/src/include/gpxe/ib_mad.h @@ -0,0 +1,341 @@ +#ifndef _GPXE_IB_MAD_H +#define _GPXE_IB_MAD_H + +/** @file + * + * Infiniband management datagrams + * + */ + +#include +#include + +/** A management datagram common header + * + * Defined in section 13.4.2 of the IBA. + */ +struct ib_mad_hdr { + uint8_t base_version; + uint8_t mgmt_class; + uint8_t class_version; + uint8_t method; + uint16_t status; + uint16_t class_specific; + uint32_t tid[2]; + uint16_t attr_id; + uint8_t reserved[2]; + uint32_t attr_mod; +} __attribute__ (( packed )); + +/* Management base version */ +#define IB_MGMT_BASE_VERSION 1 + +/* Management classes */ +#define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01 +#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81 +#define IB_MGMT_CLASS_SUBN_ADM 0x03 +#define IB_MGMT_CLASS_PERF_MGMT 0x04 +#define IB_MGMT_CLASS_BM 0x05 +#define IB_MGMT_CLASS_DEVICE_MGMT 0x06 +#define IB_MGMT_CLASS_CM 0x07 +#define IB_MGMT_CLASS_SNMP 0x08 +#define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30 +#define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F + +/* Management methods */ +#define IB_MGMT_METHOD_GET 0x01 +#define IB_MGMT_METHOD_SET 0x02 +#define IB_MGMT_METHOD_GET_RESP 0x81 +#define IB_MGMT_METHOD_SEND 0x03 +#define IB_MGMT_METHOD_TRAP 0x05 +#define IB_MGMT_METHOD_REPORT 0x06 +#define IB_MGMT_METHOD_REPORT_RESP 0x86 +#define IB_MGMT_METHOD_TRAP_REPRESS 0x07 +#define IB_MGMT_METHOD_DELETE 0x15 + +/* Status codes */ +#define IB_MGMT_STATUS_OK 0x0000 +#define IB_MGMT_STATUS_BAD_VERSION 0x0001 +#define IB_MGMT_STATUS_UNSUPPORTED_METHOD 0x0002 +#define IB_MGMT_STATUS_UNSUPPORTED_METHOD_ATTR 0x0003 +#define IB_MGMT_STATUS_INVALID_VALUE 0x0004 + +/** A LID routed SMP header + * + * Defined in section 14.2.1.1 of the IBA. + */ +struct ib_smp_lr_hdr { + uint64_t mkey; + uint8_t reserved[32]; +} __attribute__ (( packed )); + +/** A directed route SMP header + * + * Defined in section 14.2.1.2 of the IBA. + */ +struct ib_smp_dr_hdr { + uint64_t mkey; + uint16_t slid; + uint16_t dlid; + uint8_t reserved[28]; +} __attribute__ (( packed )); + +/** A subnet management header */ +union ib_smp_hdr { + uint64_t mkey; + struct ib_smp_lr_hdr lr; + struct ib_smp_dr_hdr dr; +} __attribute__ (( packed )); + +/** Subnet management class version */ +#define IB_SMP_CLASS_VERSION 1 + +/** Subnet management direction bit + * + * This bit resides in the "status" field in the MAD header. + */ +#define IB_SMP_STATUS_D_INBOUND 0x8000 + +/* Subnet management attributes */ +#define IB_SMP_ATTR_NOTICE 0x0002 +#define IB_SMP_ATTR_NODE_DESC 0x0010 +#define IB_SMP_ATTR_NODE_INFO 0x0011 +#define IB_SMP_ATTR_SWITCH_INFO 0x0012 +#define IB_SMP_ATTR_GUID_INFO 0x0014 +#define IB_SMP_ATTR_PORT_INFO 0x0015 +#define IB_SMP_ATTR_PKEY_TABLE 0x0016 +#define IB_SMP_ATTR_SL_TO_VL_TABLE 0x0017 +#define IB_SMP_ATTR_VL_ARB_TABLE 0x0018 +#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE 0x0019 +#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE 0x001A +#define IB_SMP_ATTR_MCAST_FORWARD_TABLE 0x001B +#define IB_SMP_ATTR_SM_INFO 0x0020 +#define IB_SMP_ATTR_VENDOR_DIAG 0x0030 +#define IB_SMP_ATTR_LED_INFO 0x0031 +#define IB_SMP_ATTR_VENDOR_MASK 0xFF00 + +/** + * A Node Description attribute + * + * Defined in section 14.2.5.2 of the IBA + */ +struct ib_node_desc { + char node_string[64]; +} __attribute__ (( packed )); + +/** A Node Information attribute + * + * Defined in section 14.2.5.3 of the IBA. + */ +struct ib_node_info { + uint8_t base_version; + uint8_t class_version; + uint8_t node_type; + uint8_t num_ports; + uint8_t sys_guid[8]; + uint8_t node_guid[8]; + uint8_t port_guid[8]; + uint16_t partition_cap; + uint16_t device_id; + uint32_t revision; + uint8_t local_port_num; + uint8_t vendor_id[3]; +} __attribute__ ((packed)); + +#define IB_NODE_TYPE_HCA 0x01 +#define IB_NODE_TYPE_SWITCH 0x02 +#define IB_NODE_TYPE_ROUTER 0x03 + +/** A GUID Information attribute + * + * Defined in section 14.2.5.5 of the IBA. + */ +struct ib_guid_info { + uint8_t guid[8][8]; +} __attribute__ (( packed )); + +/** A Port Information attribute + * + * Defined in section 14.2.5.6 of the IBA. + */ +struct ib_port_info { + uint64_t mkey; + uint8_t gid_prefix[8]; + uint16_t lid; + uint16_t mastersm_lid; + uint32_t cap_mask; + uint16_t diag_code; + uint16_t mkey_lease_period; + uint8_t local_port_num; + uint8_t link_width_enabled; + uint8_t link_width_supported; + uint8_t link_width_active; + uint8_t link_speed_supported__port_state; + uint8_t port_phys_state__link_down_def_state; + uint8_t mkey_prot_bits__lmc; + uint8_t link_speed_active__link_speed_enabled; + uint8_t neighbour_mtu__mastersm_sl; + uint8_t vl_cap__init_type; + uint8_t vl_high_limit; + uint8_t vl_arbitration_high_cap; + uint8_t vl_arbitration_low_cap; + uint8_t init_type_reply__mtu_cap; + uint8_t vl_stall_count__hoq_life; + uint8_t operational_vls__enforcement; + uint16_t mkey_violations; + uint16_t pkey_violations; + uint16_t qkey_violations; + uint8_t guid_cap; + uint8_t client_reregister__subnet_timeout; + uint8_t resp_time_value; + uint8_t local_phy_errors__overrun_errors; + uint16_t max_credit_hint; + uint32_t link_round_trip_latency; +} __attribute__ (( packed )); + +#define IB_LINK_WIDTH_1X 0x01 +#define IB_LINK_WIDTH_4X 0x02 +#define IB_LINK_WIDTH_8X 0x04 +#define IB_LINK_WIDTH_12X 0x08 + +#define IB_LINK_SPEED_SDR 0x01 +#define IB_LINK_SPEED_DDR 0x02 +#define IB_LINK_SPEED_QDR 0x04 + +#define IB_PORT_STATE_DOWN 0x01 +#define IB_PORT_STATE_INIT 0x02 +#define IB_PORT_STATE_ARMED 0x03 +#define IB_PORT_STATE_ACTIVE 0x04 + +#define IB_PORT_PHYS_STATE_SLEEP 0x01 +#define IB_PORT_PHYS_STATE_POLLING 0x02 + +#define IB_MTU_256 0x01 +#define IB_MTU_512 0x02 +#define IB_MTU_1024 0x03 +#define IB_MTU_2048 0x04 +#define IB_MTU_4096 0x05 + +#define IB_VL_0 0x01 +#define IB_VL_0_1 0x02 +#define IB_VL_0_3 0x03 +#define IB_VL_0_7 0x04 +#define IB_VL_0_14 0x05 + +/** A Partition Key Table attribute + * + * Defined in section 14.2.5.7 of the IBA. + */ +struct ib_pkey_table { + uint16_t pkey[32]; +} __attribute__ (( packed )); + +/** A subnet management attribute */ +union ib_smp_data { + struct ib_node_desc node_desc; + struct ib_node_info node_info; + struct ib_guid_info guid_info; + struct ib_port_info port_info; + struct ib_pkey_table pkey_table; + uint8_t bytes[64]; +} __attribute__ (( packed )); + +/** A subnet management directed route path */ +struct ib_smp_dr_path { + uint8_t reserved; + uint8_t hops[63]; +} __attribute__ (( packed )); + +/** A subnet management MAD */ +struct ib_mad_smp { + struct ib_mad_hdr mad_hdr; + union ib_smp_hdr smp_hdr; + union ib_smp_data smp_data; + struct ib_smp_dr_path initial_path; + struct ib_smp_dr_path return_path; +} __attribute__ (( packed )); + +struct ib_sa_hdr { + uint32_t sm_key[2]; + uint16_t reserved; + uint16_t attrib_offset; + uint32_t comp_mask[2]; +} __attribute__ (( packed )); + +struct ib_rmpp_hdr { + uint32_t raw[3]; +} __attribute__ (( packed )); + +struct ib_mad_path_record { + struct ib_mad_hdr mad_hdr; + struct ib_rmpp_hdr rmpp_hdr; + struct ib_sa_hdr sa_hdr; + uint32_t reserved0[2]; + struct ib_gid dgid; + struct ib_gid sgid; + uint16_t dlid; + uint16_t slid; + uint32_t hop_limit__flow_label__raw_traffic; + uint32_t pkey__numb_path__reversible__tclass; + uint8_t reserved1; + uint8_t reserved__sl; + uint8_t mtu_selector__mtu; + uint8_t rate_selector__rate; + uint32_t preference__packet_lifetime__packet_lifetime_selector; + uint32_t reserved2[35]; +} __attribute__ (( packed )); + +struct ib_mad_mc_member_record { + struct ib_mad_hdr mad_hdr; + struct ib_rmpp_hdr rmpp_hdr; + struct ib_sa_hdr sa_hdr; + struct ib_gid mgid; + struct ib_gid port_gid; + uint32_t qkey; + uint16_t mlid; + uint8_t mtu_selector__mtu; + uint8_t tclass; + uint16_t pkey; + uint8_t rate_selector__rate; + uint8_t packet_lifetime_selector__packet_lifetime; + uint32_t sl__flow_label__hop_limit; + uint8_t scope__join_state; + uint8_t proxy_join__reserved; + uint16_t reserved0; + uint32_t reserved1[37]; +} __attribute__ (( packed )); + +#define IB_SA_ATTR_MC_MEMBER_REC 0x38 +#define IB_SA_ATTR_PATH_REC 0x35 + +#define IB_SA_MCMEMBER_REC_MGID (1<<0) +#define IB_SA_MCMEMBER_REC_PORT_GID (1<<1) +#define IB_SA_MCMEMBER_REC_QKEY (1<<2) +#define IB_SA_MCMEMBER_REC_MLID (1<<3) +#define IB_SA_MCMEMBER_REC_MTU_SELECTOR (1<<4) +#define IB_SA_MCMEMBER_REC_MTU (1<<5) +#define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS (1<<6) +#define IB_SA_MCMEMBER_REC_PKEY (1<<7) +#define IB_SA_MCMEMBER_REC_RATE_SELECTOR (1<<8) +#define IB_SA_MCMEMBER_REC_RATE (1<<9) +#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR (1<<10) +#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME (1<<11) +#define IB_SA_MCMEMBER_REC_SL (1<<12) +#define IB_SA_MCMEMBER_REC_FLOW_LABEL (1<<13) +#define IB_SA_MCMEMBER_REC_HOP_LIMIT (1<<14) +#define IB_SA_MCMEMBER_REC_SCOPE (1<<15) +#define IB_SA_MCMEMBER_REC_JOIN_STATE (1<<16) +#define IB_SA_MCMEMBER_REC_PROXY_JOIN (1<<17) + +#define IB_SA_PATH_REC_DGID (1<<2) +#define IB_SA_PATH_REC_SGID (1<<3) + +union ib_mad { + struct ib_mad_hdr hdr; + struct ib_mad_smp smp; + struct ib_mad_path_record path_record; + struct ib_mad_mc_member_record mc_member_record; + uint8_t bytes[256]; +} __attribute__ (( packed )); + +#endif /* _GPXE_IB_MAD_H */ diff --git a/src/include/gpxe/ib_packet.h b/src/include/gpxe/ib_packet.h new file mode 100644 index 00000000..f9ef2f6e --- /dev/null +++ b/src/include/gpxe/ib_packet.h @@ -0,0 +1,114 @@ +#ifndef _GPXE_IB_PACKET_H +#define _GPXE_IB_PACKET_H + +/** @file + * + * Infiniband packet format + * + */ + +/** Half of an Infiniband Global Identifier */ +struct ib_gid_half { + uint8_t bytes[8]; +}; + +/** An Infiniband Global Identifier */ +struct ib_gid { + union { + uint8_t bytes[16]; + uint16_t words[8]; + uint32_t dwords[4]; + struct ib_gid_half half[2]; + } u; +}; + +/** An Infiniband Local Route Header */ +struct ib_local_route_header { + /** Virtual lane and link version */ + uint8_t vl__lver; + /** Service level and next link header */ + uint8_t sl__lnh; + /** Destination LID */ + uint16_t dlid; + /** Packet length */ + uint16_t length; + /** Source LID */ + uint16_t slid; +} __attribute__ (( packed )); + +/** Infiniband virtual lanes */ +enum ib_vl { + IB_VL_DEFAULT = 0, + IB_VL_SMP = 15, +}; + +/** An Infiniband Link Next Header value */ +enum ib_lnh { + IB_LNH_RAW = 0, + IB_LNH_IPv6 = 1, + IB_LNH_BTH = 2, + IB_LNH_GRH = 3 +}; + +/** Default Infiniband LID */ +#define IB_LID_NONE 0xffff + +/** An Infiniband Global Route Header */ +struct ib_global_route_header { + /** IP version, traffic class, and flow label + * + * 4 bits : Version of the GRH + * 8 bits : Traffic class + * 20 bits : Flow label + */ + uint32_t ipver__tclass__flowlabel; + /** Payload length */ + uint16_t paylen; + /** Next header */ + uint8_t nxthdr; + /** Hop limit */ + uint8_t hoplmt; + /** Source GID */ + struct ib_gid sgid; + /** Destiniation GID */ + struct ib_gid dgid; +} __attribute__ (( packed )); + +#define IB_GRH_IPVER_IPv6 0x06 +#define IB_GRH_NXTHDR_IBA 0x1b +#define IB_GRH_HOPLMT_MAX 0xff + +/** An Infiniband Base Transport Header */ +struct ib_base_transport_header { + /** Opcode */ + uint8_t opcode; + /** Transport header version, pad count, migration and solicitation */ + uint8_t se__m__padcnt__tver; + /** Partition key */ + uint16_t pkey; + /** Destination queue pair */ + uint32_t dest_qp; + /** Packet sequence number and acknowledge request */ + uint32_t ack__psn; +} __attribute__ (( packed )); + +/** An Infiniband BTH opcode */ +enum ib_bth_opcode { + BTH_OPCODE_UD_SEND = 0x64, +}; + +/** Default Infiniband partition key */ +#define IB_PKEY_NONE 0xffff + +/** Subnet management queue pair number */ +#define IB_QPN_SMP 0 + +/** An Infiniband Datagram Extended Transport Header */ +struct ib_datagram_extended_transport_header { + /** Queue key */ + uint32_t qkey; + /** Source queue pair */ + uint32_t src_qp; +} __attribute__ (( packed )); + +#endif /* _GPXE_IB_PACKET_H */ diff --git a/src/include/gpxe/ib_smc.h b/src/include/gpxe/ib_smc.h new file mode 100644 index 00000000..bb9020bf --- /dev/null +++ b/src/include/gpxe/ib_smc.h @@ -0,0 +1,18 @@ +#ifndef _GPXE_IB_SMC_H +#define _GPXE_IB_SMC_H + +/** @file + * + * Infiniband Subnet Management Client + * + */ + +#include + +typedef int ( * ib_local_mad_t ) ( struct ib_device *ibdev, + union ib_mad *mad ); + +extern int ib_smc_update ( struct ib_device *ibdev, + ib_local_mad_t local_mad ); + +#endif /* _GPXE_IB_SMC_H */ diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index ed0d1a66..2691ffc9 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include /** Subnet administrator QPN */ #define IB_SA_QPN 1 @@ -20,36 +22,6 @@ /** Subnet administrator queue key */ #define IB_GLOBAL_QKEY 0x80010000UL -/** An Infiniband Global Identifier */ -struct ib_gid { - union { - uint8_t bytes[16]; - uint16_t words[8]; - uint32_t dwords[4]; - } u; -}; - -/** An Infiniband Global Route Header */ -struct ib_global_route_header { - /** IP version, traffic class, and flow label - * - * 4 bits : Version of the GRH - * 8 bits : Traffic class - * 20 bits : Flow label - */ - uint32_t ipver_tclass_flowlabel; - /** Payload length */ - uint16_t paylen; - /** Next header */ - uint8_t nxthdr; - /** Hop limit */ - uint8_t hoplmt; - /** Source GID */ - struct ib_gid sgid; - /** Destiniation GID */ - struct ib_gid dgid; -} __attribute__ (( packed )); - struct ib_device; struct ib_queue_pair; struct ib_address_vector; @@ -178,8 +150,6 @@ struct ib_completion_queue { void *drv_priv; }; -struct ib_mad_hdr; - /** * Infiniband device operations * @@ -306,16 +276,6 @@ struct ib_device_operations { void ( * mcast_detach ) ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_gid *gid ); - /** - * Issue management datagram - * - * @v ibdev Infiniband device - * @v mad Management datagram - * @v len Length of management datagram - * @ret rc Return status code - */ - int ( * mad ) ( struct ib_device *ibdev, struct ib_mad_hdr *mad, - size_t len ); }; /** An Infiniband device */ @@ -330,14 +290,24 @@ struct ib_device { struct ib_device_operations *op; /** Port number */ unsigned int port; - /** Link state */ - int link_up; + + /** Port state */ + uint8_t port_state; + /** Link width */ + uint8_t link_width; + /** Link speed */ + uint8_t link_speed; /** Port GID */ - struct ib_gid port_gid; + struct ib_gid gid; + /** Port LID */ + uint16_t lid; /** Subnet manager LID */ - unsigned long sm_lid; + uint16_t sm_lid; + /** Subnet manager SL */ + uint8_t sm_sl; /** Partition key */ - unsigned int pkey; + uint16_t pkey; + /** Driver private data */ void *drv_priv; /** Owner private data */ @@ -375,6 +345,11 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size ); extern int register_ibdev ( struct ib_device *ibdev ); extern void unregister_ibdev ( struct ib_device *ibdev ); extern void ib_link_state_changed ( struct ib_device *ibdev ); +extern struct list_head ib_devices; + +/** Iterate over all network devices */ +#define for_each_ibdev( ibdev ) \ + list_for_each_entry ( (ibdev), &ib_devices, list ) /** * Poll completion queue @@ -382,7 +357,7 @@ extern void ib_link_state_changed ( struct ib_device *ibdev ); * @v ibdev Infiniband device * @v cq Completion queue */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { ibdev->op->poll_cq ( ibdev, cq ); } @@ -393,7 +368,7 @@ ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { * @v ibdev Infiniband device * @ret rc Return status code */ -static inline __attribute__ (( always_inline )) int +static inline __always_inline int ib_open ( struct ib_device *ibdev ) { return ibdev->op->open ( ibdev ); } @@ -403,11 +378,22 @@ ib_open ( struct ib_device *ibdev ) { * * @v ibdev Infiniband device */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_close ( struct ib_device *ibdev ) { ibdev->op->close ( ibdev ); } +/** + * Check link state + * + * @v ibdev Infiniband device + * @ret link_up Link is up + */ +static inline __always_inline int +ib_link_ok ( struct ib_device *ibdev ) { + return ( ibdev->port_state == IB_PORT_STATE_ACTIVE ); +} + /** * Attach to multicast group * @@ -416,7 +402,7 @@ ib_close ( struct ib_device *ibdev ) { * @v gid Multicast GID * @ret rc Return status code */ -static inline __attribute__ (( always_inline )) int +static inline __always_inline int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_gid *gid ) { return ibdev->op->mcast_attach ( ibdev, qp, gid ); @@ -429,32 +415,19 @@ ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp, * @v qp Queue pair * @v gid Multicast GID */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_gid *gid ) { ibdev->op->mcast_detach ( ibdev, qp, gid ); } -/** - * Issue management datagram - * - * @v ibdev Infiniband device - * @v mad Management datagram - * @v len Length of management datagram - * @ret rc Return status code - */ -static inline __attribute__ (( always_inline )) int -ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) { - return ibdev->op->mad ( ibdev, mad, len ); -} - /** * Get reference to Infiniband device * * @v ibdev Infiniband device * @ret ibdev Infiniband device */ -static inline __attribute__ (( always_inline )) struct ib_device * +static inline __always_inline struct ib_device * ibdev_get ( struct ib_device *ibdev ) { ref_get ( &ibdev->refcnt ); return ibdev; @@ -465,7 +438,7 @@ ibdev_get ( struct ib_device *ibdev ) { * * @v ibdev Infiniband device */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ibdev_put ( struct ib_device *ibdev ) { ref_put ( &ibdev->refcnt ); } @@ -476,7 +449,7 @@ ibdev_put ( struct ib_device *ibdev ) { * @v wq Work queue * @v priv Private data */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) { wq->drv_priv = priv; } @@ -487,7 +460,7 @@ ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) { * @v wq Work queue * @ret priv Private data */ -static inline __attribute__ (( always_inline )) void * +static inline __always_inline void * ib_wq_get_drvdata ( struct ib_work_queue *wq ) { return wq->drv_priv; } @@ -498,7 +471,7 @@ ib_wq_get_drvdata ( struct ib_work_queue *wq ) { * @v qp Queue pair * @v priv Private data */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) { qp->drv_priv = priv; } @@ -509,7 +482,7 @@ ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) { * @v qp Queue pair * @ret priv Private data */ -static inline __attribute__ (( always_inline )) void * +static inline __always_inline void * ib_qp_get_drvdata ( struct ib_queue_pair *qp ) { return qp->drv_priv; } @@ -520,7 +493,7 @@ ib_qp_get_drvdata ( struct ib_queue_pair *qp ) { * @v qp Queue pair * @v priv Private data */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) { qp->owner_priv = priv; } @@ -531,7 +504,7 @@ ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) { * @v qp Queue pair * @ret priv Private data */ -static inline __attribute__ (( always_inline )) void * +static inline __always_inline void * ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) { return qp->owner_priv; } @@ -542,7 +515,7 @@ ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) { * @v cq Completion queue * @v priv Private data */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) { cq->drv_priv = priv; } @@ -553,7 +526,7 @@ ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) { * @v cq Completion queue * @ret priv Private data */ -static inline __attribute__ (( always_inline )) void * +static inline __always_inline void * ib_cq_get_drvdata ( struct ib_completion_queue *cq ) { return cq->drv_priv; } @@ -564,7 +537,7 @@ ib_cq_get_drvdata ( struct ib_completion_queue *cq ) { * @v ibdev Infiniband device * @v priv Private data */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_set_drvdata ( struct ib_device *ibdev, void *priv ) { ibdev->drv_priv = priv; } @@ -575,7 +548,7 @@ ib_set_drvdata ( struct ib_device *ibdev, void *priv ) { * @v ibdev Infiniband device * @ret priv Private data */ -static inline __attribute__ (( always_inline )) void * +static inline __always_inline void * ib_get_drvdata ( struct ib_device *ibdev ) { return ibdev->drv_priv; } @@ -586,7 +559,7 @@ ib_get_drvdata ( struct ib_device *ibdev ) { * @v ibdev Infiniband device * @v priv Private data */ -static inline __attribute__ (( always_inline )) void +static inline __always_inline void ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) { ibdev->owner_priv = priv; } @@ -597,201 +570,9 @@ ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) { * @v ibdev Infiniband device * @ret priv Private data */ -static inline __attribute__ (( always_inline )) void * +static inline __always_inline void * ib_get_ownerdata ( struct ib_device *ibdev ) { return ibdev->owner_priv; } -/***************************************************************************** - * - * Management datagrams - * - * Portions Copyright (c) 2004 Mellanox Technologies Ltd. All rights - * reserved. - * - */ - -/* Management base version */ -#define IB_MGMT_BASE_VERSION 1 - -/* Management classes */ -#define IB_MGMT_CLASS_SUBN_LID_ROUTED 0x01 -#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 0x81 -#define IB_MGMT_CLASS_SUBN_ADM 0x03 -#define IB_MGMT_CLASS_PERF_MGMT 0x04 -#define IB_MGMT_CLASS_BM 0x05 -#define IB_MGMT_CLASS_DEVICE_MGMT 0x06 -#define IB_MGMT_CLASS_CM 0x07 -#define IB_MGMT_CLASS_SNMP 0x08 -#define IB_MGMT_CLASS_VENDOR_RANGE2_START 0x30 -#define IB_MGMT_CLASS_VENDOR_RANGE2_END 0x4F - -/* Management methods */ -#define IB_MGMT_METHOD_GET 0x01 -#define IB_MGMT_METHOD_SET 0x02 -#define IB_MGMT_METHOD_GET_RESP 0x81 -#define IB_MGMT_METHOD_SEND 0x03 -#define IB_MGMT_METHOD_TRAP 0x05 -#define IB_MGMT_METHOD_REPORT 0x06 -#define IB_MGMT_METHOD_REPORT_RESP 0x86 -#define IB_MGMT_METHOD_TRAP_REPRESS 0x07 -#define IB_MGMT_METHOD_DELETE 0x15 -#define IB_MGMT_METHOD_RESP 0x80 - -/* Subnet management attributes */ -#define IB_SMP_ATTR_NOTICE 0x0002 -#define IB_SMP_ATTR_NODE_DESC 0x0010 -#define IB_SMP_ATTR_NODE_INFO 0x0011 -#define IB_SMP_ATTR_SWITCH_INFO 0x0012 -#define IB_SMP_ATTR_GUID_INFO 0x0014 -#define IB_SMP_ATTR_PORT_INFO 0x0015 -#define IB_SMP_ATTR_PKEY_TABLE 0x0016 -#define IB_SMP_ATTR_SL_TO_VL_TABLE 0x0017 -#define IB_SMP_ATTR_VL_ARB_TABLE 0x0018 -#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE 0x0019 -#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE 0x001A -#define IB_SMP_ATTR_MCAST_FORWARD_TABLE 0x001B -#define IB_SMP_ATTR_SM_INFO 0x0020 -#define IB_SMP_ATTR_VENDOR_DIAG 0x0030 -#define IB_SMP_ATTR_LED_INFO 0x0031 -#define IB_SMP_ATTR_VENDOR_MASK 0xFF00 - -#define IB_SA_ATTR_MC_MEMBER_REC 0x38 -#define IB_SA_ATTR_PATH_REC 0x35 - -#define IB_SA_MCMEMBER_REC_MGID (1<<0) -#define IB_SA_MCMEMBER_REC_PORT_GID (1<<1) -#define IB_SA_MCMEMBER_REC_QKEY (1<<2) -#define IB_SA_MCMEMBER_REC_MLID (1<<3) -#define IB_SA_MCMEMBER_REC_MTU_SELECTOR (1<<4) -#define IB_SA_MCMEMBER_REC_MTU (1<<5) -#define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS (1<<6) -#define IB_SA_MCMEMBER_REC_PKEY (1<<7) -#define IB_SA_MCMEMBER_REC_RATE_SELECTOR (1<<8) -#define IB_SA_MCMEMBER_REC_RATE (1<<9) -#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR (1<<10) -#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME (1<<11) -#define IB_SA_MCMEMBER_REC_SL (1<<12) -#define IB_SA_MCMEMBER_REC_FLOW_LABEL (1<<13) -#define IB_SA_MCMEMBER_REC_HOP_LIMIT (1<<14) -#define IB_SA_MCMEMBER_REC_SCOPE (1<<15) -#define IB_SA_MCMEMBER_REC_JOIN_STATE (1<<16) -#define IB_SA_MCMEMBER_REC_PROXY_JOIN (1<<17) - -#define IB_SA_PATH_REC_DGID (1<<2) -#define IB_SA_PATH_REC_SGID (1<<3) - -struct ib_mad_hdr { - uint8_t base_version; - uint8_t mgmt_class; - uint8_t class_version; - uint8_t method; - uint16_t status; - uint16_t class_specific; - uint32_t tid[2]; - uint16_t attr_id; - uint16_t resv; - uint32_t attr_mod; -} __attribute__ (( packed )); - -struct ib_sa_hdr { - uint32_t sm_key[2]; - uint16_t reserved; - uint16_t attrib_offset; - uint32_t comp_mask[2]; -} __attribute__ (( packed )); - -struct ib_rmpp_hdr { - uint32_t raw[3]; -} __attribute__ (( packed )); - -struct ib_mad_data { - struct ib_mad_hdr mad_hdr; - uint8_t data[232]; -} __attribute__ (( packed )); - -struct ib_mad_guid_info { - struct ib_mad_hdr mad_hdr; - uint32_t mkey[2]; - uint32_t reserved[8]; - uint8_t gid_local[8]; -} __attribute__ (( packed )); - -struct ib_mad_port_info { - struct ib_mad_hdr mad_hdr; - uint32_t mkey[2]; - uint32_t reserved[8]; - uint32_t mkey2[2]; - uint8_t gid_prefix[8]; - uint16_t lid; - uint16_t mastersm_lid; - uint32_t cap_mask; - uint16_t diag_code; - uint16_t mkey_lease_period; - uint8_t local_port_num; - uint8_t link_width_enabled; - uint8_t link_width_supported; - uint8_t link_width_active; - uint8_t port_state__link_speed_supported; - uint8_t link_down_def_state__port_phys_state; - uint8_t lmc__r1__mkey_prot_bits; - uint8_t link_speed_enabled__link_speed_active; -} __attribute__ (( packed )); - -struct ib_mad_pkey_table { - struct ib_mad_hdr mad_hdr; - uint32_t mkey[2]; - uint32_t reserved[8]; - uint16_t pkey[16][2]; -} __attribute__ (( packed )); - -struct ib_mad_path_record { - struct ib_mad_hdr mad_hdr; - struct ib_rmpp_hdr rmpp_hdr; - struct ib_sa_hdr sa_hdr; - uint32_t reserved0[2]; - struct ib_gid dgid; - struct ib_gid sgid; - uint16_t dlid; - uint16_t slid; - uint32_t hop_limit__flow_label__raw_traffic; - uint32_t pkey__numb_path__reversible__tclass; - uint8_t reserved1; - uint8_t reserved__sl; - uint8_t mtu_selector__mtu; - uint8_t rate_selector__rate; - uint32_t preference__packet_lifetime__packet_lifetime_selector; - uint32_t reserved2[35]; -} __attribute__ (( packed )); - -struct ib_mad_mc_member_record { - struct ib_mad_hdr mad_hdr; - struct ib_rmpp_hdr rmpp_hdr; - struct ib_sa_hdr sa_hdr; - struct ib_gid mgid; - struct ib_gid port_gid; - uint32_t qkey; - uint16_t mlid; - uint8_t mtu_selector__mtu; - uint8_t tclass; - uint16_t pkey; - uint8_t rate_selector__rate; - uint8_t packet_lifetime_selector__packet_lifetime; - uint32_t sl__flow_label__hop_limit; - uint8_t scope__join_state; - uint8_t proxy_join__reserved; - uint16_t reserved0; - uint32_t reserved1[37]; -} __attribute__ (( packed )); - -union ib_mad { - struct ib_mad_hdr mad_hdr; - struct ib_mad_data data; - struct ib_mad_guid_info guid_info; - struct ib_mad_port_info port_info; - struct ib_mad_pkey_table pkey_table; - struct ib_mad_path_record path_record; - struct ib_mad_mc_member_record mc_member_record; -} __attribute__ (( packed )); - #endif /* _GPXE_INFINIBAND_H */ diff --git a/src/net/infiniband.c b/src/net/infiniband.c index ba7842cf..7ded51e6 100644 --- a/src/net/infiniband.c +++ b/src/net/infiniband.c @@ -333,142 +333,6 @@ void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, qp->recv.fill--; } -/*************************************************************************** - * - * Management datagram operations - * - *************************************************************************** - */ - -/** - * Get port information - * - * @v ibdev Infiniband device - * @v port_info Port information datagram to fill in - * @ret rc Return status code - */ -static int ib_get_port_info ( struct ib_device *ibdev, - struct ib_mad_port_info *port_info ) { - struct ib_mad_hdr *hdr = &port_info->mad_hdr; - int rc; - - /* Construct MAD */ - memset ( port_info, 0, sizeof ( *port_info ) ); - hdr->base_version = IB_MGMT_BASE_VERSION; - hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - hdr->class_version = 1; - hdr->method = IB_MGMT_METHOD_GET; - hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO ); - hdr->attr_mod = htonl ( ibdev->port ); - - if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *port_info ) ) ) != 0 ) { - DBGC ( ibdev, "IBDEV %p could not get port info: %s\n", - ibdev, strerror ( rc ) ); - return rc; - } - return 0; -} - -/** - * Get GUID information - * - * @v ibdev Infiniband device - * @v guid_info GUID information datagram to fill in - * @ret rc Return status code - */ -static int ib_get_guid_info ( struct ib_device *ibdev, - struct ib_mad_guid_info *guid_info ) { - struct ib_mad_hdr *hdr = &guid_info->mad_hdr; - int rc; - - /* Construct MAD */ - memset ( guid_info, 0, sizeof ( *guid_info ) ); - hdr->base_version = IB_MGMT_BASE_VERSION; - hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - hdr->class_version = 1; - hdr->method = IB_MGMT_METHOD_GET; - hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO ); - - if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *guid_info ) ) ) != 0 ) { - DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n", - ibdev, strerror ( rc ) ); - return rc; - } - return 0; -} - -/** - * Get partition key table - * - * @v ibdev Infiniband device - * @v guid_info Partition key table datagram to fill in - * @ret rc Return status code - */ -static int ib_get_pkey_table ( struct ib_device *ibdev, - struct ib_mad_pkey_table *pkey_table ) { - struct ib_mad_hdr *hdr = &pkey_table->mad_hdr; - int rc; - - /* Construct MAD */ - memset ( pkey_table, 0, sizeof ( *pkey_table ) ); - hdr->base_version = IB_MGMT_BASE_VERSION; - hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - hdr->class_version = 1; - hdr->method = IB_MGMT_METHOD_GET; - hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE ); - - if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *pkey_table ) ) ) != 0 ) { - DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n", - ibdev, strerror ( rc ) ); - return rc; - } - return 0; -} - -/** - * Get MAD parameters - * - * @v ibdev Infiniband device - * @ret rc Return status code - */ -static int ib_get_mad_params ( struct ib_device *ibdev ) { - union { - /* This union exists just to save stack space */ - struct ib_mad_port_info port_info; - struct ib_mad_guid_info guid_info; - struct ib_mad_pkey_table pkey_table; - } u; - int rc; - - /* Port info gives us the link state, the first half of the - * port GID and the SM LID. - */ - if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 ) - return rc; - ibdev->link_up = ( ( u.port_info.port_state__link_speed_supported - & 0xf ) == 4 ); - memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 ); - ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid ); - - /* GUID info gives us the second half of the port GID */ - if ( ( rc = ib_get_guid_info ( ibdev, &u.guid_info ) ) != 0 ) - return rc; - memcpy ( &ibdev->port_gid.u.bytes[8], u.guid_info.gid_local, 8 ); - - /* Get partition key */ - if ( ( rc = ib_get_pkey_table ( ibdev, &u.pkey_table ) ) != 0 ) - return rc; - ibdev->pkey = ntohs ( u.pkey_table.pkey[0][0] ); - - DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n", - ibdev, htonl ( ibdev->port_gid.u.dwords[0] ), - htonl ( ibdev->port_gid.u.dwords[1] ), - htonl ( ibdev->port_gid.u.dwords[2] ), - htonl ( ibdev->port_gid.u.dwords[3] ) ); - - return 0; -} - /*************************************************************************** * * Event queues @@ -482,14 +346,6 @@ static int ib_get_mad_params ( struct ib_device *ibdev ) { * @v ibdev Infiniband device */ void ib_link_state_changed ( struct ib_device *ibdev ) { - int rc; - - /* Update MAD parameters */ - if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) { - DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n", - ibdev, strerror ( rc ) ); - return; - } /* Notify IPoIB of link state change */ ipoib_link_state_changed ( ibdev ); @@ -536,6 +392,8 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) { if ( ibdev ) { drv_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) ); ib_set_drvdata ( ibdev, drv_priv ); + ibdev->lid = IB_LID_NONE; + ibdev->pkey = IB_PKEY_NONE; } return ibdev; } @@ -557,10 +415,6 @@ int register_ibdev ( struct ib_device *ibdev ) { if ( ( rc = ib_open ( ibdev ) ) != 0 ) goto err_open; - /* Get MAD parameters */ - if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) - goto err_get_mad_params; - /* Add IPoIB device */ if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) { DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n", @@ -573,7 +427,6 @@ int register_ibdev ( struct ib_device *ibdev ) { return 0; err_ipoib_probe: - err_get_mad_params: ib_close ( ibdev ); err_open: list_del ( &ibdev->list ); -- cgit v1.2.3-55-g7522