From 34bfc04e4c2ea74a025994fc37c83d640942a651 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Thu, 6 Aug 2009 01:18:38 +0100 Subject: [infiniband] Update all other MAD users to use a management interface --- src/drivers/net/ipoib.c | 32 ++- src/include/gpxe/ib_cm.h | 60 ++++- src/include/gpxe/ib_gma.h | 65 ------ src/include/gpxe/ib_mad.h | 11 + src/include/gpxe/ib_mcast.h | 33 ++- src/include/gpxe/ib_pathrec.h | 59 +++++ src/include/gpxe/infiniband.h | 5 +- src/net/infiniband.c | 17 +- src/net/infiniband/ib_cm.c | 473 ++++++++++++++++++++-------------------- src/net/infiniband/ib_gma.c | 403 ---------------------------------- src/net/infiniband/ib_mcast.c | 248 ++++++++++----------- src/net/infiniband/ib_pathrec.c | 333 +++++++++++++++++----------- 12 files changed, 747 insertions(+), 992 deletions(-) delete mode 100644 src/include/gpxe/ib_gma.h delete mode 100644 src/net/infiniband/ib_gma.c diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index 9110784c..3f149247 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -58,12 +58,14 @@ struct ipoib_device { struct ib_queue_pair *qp; /** Broadcast MAC */ struct ipoib_mac broadcast; - /** Joined to multicast group + /** Joined to IPv4 broadcast multicast group * * This flag indicates whether or not we have initiated the - * join to the IPv4 multicast group. + * join to the IPv4 broadcast multicast group. */ int broadcast_joined; + /** IPv4 broadcast multicast group membership */ + struct ib_mc_membership broadcast_membership; }; /** Broadcast IPoIB address */ @@ -456,6 +458,26 @@ static void ipoib_irq ( struct net_device *netdev __unused, /* No implementation */ } +/** + * Handle IPv4 broadcast multicast group join completion + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v membership Multicast group membership + * @v rc Status code + * @v mad Response MAD (or NULL on error) + */ +void ipoib_join_complete ( struct ib_device *ibdev __unused, + struct ib_queue_pair *qp __unused, + struct ib_mc_membership *membership, int rc, + union ib_mad *mad __unused ) { + struct ipoib_device *ipoib = container_of ( membership, + struct ipoib_device, broadcast_membership ); + + /* Record join status as link status */ + netdev_link_err ( ipoib->netdev, rc ); +} + /** * Join IPv4 broadcast multicast group * @@ -466,7 +488,9 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) { int rc; if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp, - &ipoib->broadcast.gid ) ) != 0 ) { + &ipoib->broadcast_membership, + &ipoib->broadcast.gid, + ipoib_join_complete ) ) != 0 ) { DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n", ipoib, strerror ( rc ) ); return rc; @@ -485,7 +509,7 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) { if ( ipoib->broadcast_joined ) { ib_mcast_leave ( ipoib->ibdev, ipoib->qp, - &ipoib->broadcast.gid ); + &ipoib->broadcast_membership ); ipoib->broadcast_joined = 0; } } diff --git a/src/include/gpxe/ib_cm.h b/src/include/gpxe/ib_cm.h index a4446224..07f5b716 100644 --- a/src/include/gpxe/ib_cm.h +++ b/src/include/gpxe/ib_cm.h @@ -10,12 +10,60 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include +#include -extern int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid, - struct ib_gid_half *service_id, - void *private_data, size_t private_data_len, - void ( * notify ) ( struct ib_queue_pair *qp, - int rc, void *private_data, - size_t private_data_len ) ); +struct ib_mad_transaction; +struct ib_connection; + +/** Infiniband connection operations */ +struct ib_connection_operations { + /** Handle change of connection status + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v conn Connection + * @v rc Connection status code + * @v private_data Private data, if available + * @v private_data_len Length of private data + */ + void ( * changed ) ( struct ib_device *ibdev, struct ib_queue_pair *qp, + struct ib_connection *conn, int rc, + void *private_data, size_t private_data_len ); +}; + +/** An Infiniband connection */ +struct ib_connection { + /** Infiniband device */ + struct ib_device *ibdev; + /** Queue pair */ + struct ib_queue_pair *qp; + /** Local communication ID */ + uint32_t local_id; + /** Remote communication ID */ + uint32_t remote_id; + /** Target service ID */ + struct ib_gid_half service_id; + /** Connection operations */ + struct ib_connection_operations *op; + + /** Path to target */ + struct ib_path *path; + /** Connection request management transaction */ + struct ib_mad_transaction *madx; + + /** Length of connection request private data */ + size_t private_data_len; + /** Connection request private data */ + uint8_t private_data[0]; +}; + +extern struct ib_connection * +ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp, + struct ib_gid *dgid, struct ib_gid_half *service_id, + void *req_private_data, size_t req_private_data_len, + struct ib_connection_operations *op ); +extern void ib_destroy_conn ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_connection *conn ); #endif /* _GPXE_IB_CM_H */ diff --git a/src/include/gpxe/ib_gma.h b/src/include/gpxe/ib_gma.h deleted file mode 100644 index 1b33514b..00000000 --- a/src/include/gpxe/ib_gma.h +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef _GPXE_IB_GMA_H -#define _GPXE_IB_GMA_H - -/** @file - * - * Infiniband General Management Agent - * - */ - -FILE_LICENCE ( GPL2_OR_LATER ); - -#include -#include -#include -#include - -struct ib_gma; - -/** A GMA attribute handler */ -struct ib_gma_handler { - /** Management class */ - uint8_t mgmt_class; - /** Management class don't-care bits */ - uint8_t mgmt_class_ignore; - /** Class version */ - uint8_t class_version; - /** Method */ - uint8_t method; - /** Attribute (in network byte order) */ - uint16_t attr_id; - /** Handle attribute - * - * @v gma General management agent - * @v mad MAD - * @ret response MAD response, or NULL to send no response - */ - union ib_mad * ( * handle ) ( struct ib_gma *gma, union ib_mad *mad ); -}; - -/** GMA attribute handlers */ -#define IB_GMA_HANDLERS __table ( struct ib_gma_handler, "ib_gma_handlers" ) - -/** Declare a GMA attribute handler */ -#define __ib_gma_handler __table_entry ( IB_GMA_HANDLERS, 01 ) - -/** An Infiniband General Management Agent */ -struct ib_gma { - /** Infiniband device */ - struct ib_device *ibdev; - /** Completion queue */ - struct ib_completion_queue *cq; - /** Queue pair */ - struct ib_queue_pair *qp; - - /** List of outstanding MAD requests */ - struct list_head requests; -}; - -extern int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad, - struct ib_address_vector *av, int retry ); -extern struct ib_gma * ib_create_gma ( struct ib_device *ibdev, - enum ib_queue_pair_type type ); -extern void ib_destroy_gma ( struct ib_gma *gma ); - -#endif /* _GPXE_IB_GMA_H */ diff --git a/src/include/gpxe/ib_mad.h b/src/include/gpxe/ib_mad.h index ac9a1d4b..54d0a2af 100644 --- a/src/include/gpxe/ib_mad.h +++ b/src/include/gpxe/ib_mad.h @@ -303,6 +303,16 @@ union ib_sa_data { #define IB_CM_ATTR_LOAD_ALTERNATE_PATH 0x0019 #define IB_CM_ATTR_ALTERNATE_PATH_RESPONSE 0x001a +/** Communication management common fields */ +struct ib_cm_common { + /** Local communication ID */ + uint32_t local_id; + /** Remote communication ID */ + uint32_t remote_id; + /** Reserved */ + uint8_t reserved[224]; +} __attribute__ (( packed )); + /** A communication management path */ struct ib_cm_path { /** Local port LID */ @@ -438,6 +448,7 @@ struct ib_cm_ready_to_use { /** A communication management attribute */ union ib_cm_data { + struct ib_cm_common common; struct ib_cm_connect_request connect_request; struct ib_cm_connect_reject connect_reject; struct ib_cm_connect_reply connect_reply; diff --git a/src/include/gpxe/ib_mcast.h b/src/include/gpxe/ib_mcast.h index 2ca3382b..74eccd0b 100644 --- a/src/include/gpxe/ib_mcast.h +++ b/src/include/gpxe/ib_mcast.h @@ -11,9 +11,38 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include +struct ib_mad_transaction; + +/** An Infiniband multicast group membership */ +struct ib_mc_membership { + /** Queue pair */ + struct ib_queue_pair *qp; + /** Multicast GID */ + struct ib_gid gid; + /** Multicast group join transaction */ + struct ib_mad_transaction *madx; + /** Handle join success/failure + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v membership Multicast group membership + * @v rc Status code + * @v mad Response MAD (or NULL on error) + */ + void ( * complete ) ( struct ib_device *ibdev, struct ib_queue_pair *qp, + struct ib_mc_membership *membership, int rc, + union ib_mad *mad ); +}; + extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp, - struct ib_gid *gid ); + struct ib_mc_membership *membership, + struct ib_gid *gid, + void ( * joined ) ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_mc_membership *memb, + int rc, union ib_mad *mad ) ); + extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp, - struct ib_gid *gid ); + struct ib_mc_membership *membership ); #endif /* _GPXE_IB_MCAST_H */ diff --git a/src/include/gpxe/ib_pathrec.h b/src/include/gpxe/ib_pathrec.h index 44515563..5884d536 100644 --- a/src/include/gpxe/ib_pathrec.h +++ b/src/include/gpxe/ib_pathrec.h @@ -11,6 +11,65 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include +struct ib_mad_transaction; +struct ib_path; + +/** Infiniband path operations */ +struct ib_path_operations { + /** Handle path transaction completion + * + * @v ibdev Infiniband device + * @v path Path + * @v rc Status code + * @v av Address vector, or NULL on error + */ + void ( * complete ) ( struct ib_device *ibdev, + struct ib_path *path, int rc, + struct ib_address_vector *av ); +}; + +/** An Infiniband path */ +struct ib_path { + /** Infiniband device */ + struct ib_device *ibdev; + /** Address vector */ + struct ib_address_vector av; + /** Management transaction */ + struct ib_mad_transaction *madx; + /** Path operations */ + struct ib_path_operations *op; + /** Owner private data */ + void *owner_priv; +}; + +/** + * Set Infiniband path owner-private data + * + * @v path Path + * @v priv Private data + */ +static inline __always_inline void +ib_path_set_ownerdata ( struct ib_path *path, void *priv ) { + path->owner_priv = priv; +} + +/** + * Get Infiniband path owner-private data + * + * @v path Path + * @ret priv Private data + */ +static inline __always_inline void * +ib_path_get_ownerdata ( struct ib_path *path ) { + return path->owner_priv; +} + +extern struct ib_path * +ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av, + struct ib_path_operations *op ); +extern void ib_destroy_path ( struct ib_device *ibdev, + struct ib_path *path ); + extern int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ); diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index d300a77b..c4bc2ff4 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -46,7 +46,6 @@ struct ib_queue_pair; struct ib_address_vector; struct ib_completion_queue; struct ib_mad_interface; -struct ib_gma; /** Infiniband transmission rates */ enum ib_rate { @@ -416,8 +415,8 @@ struct ib_device { /** Subnet management interface */ struct ib_mad_interface *smi; - /** General management agent */ - struct ib_gma *gma; + /** General services interface */ + struct ib_mad_interface *gsi; /** Driver private data */ void *drv_priv; diff --git a/src/net/infiniband.c b/src/net/infiniband.c index a0dfc5b3..2d8b63ec 100644 --- a/src/net/infiniband.c +++ b/src/net/infiniband.c @@ -35,7 +35,6 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include -#include /** @file * @@ -551,12 +550,12 @@ int ib_open ( struct ib_device *ibdev ) { goto err_create_sma; } - /* Create general management agent */ - ibdev->gma = ib_create_gma ( ibdev, IB_QPT_GSI ); - if ( ! ibdev->gma ) { - DBGC ( ibdev, "IBDEV %p could not create GMA\n", ibdev ); + /* Create general services interface */ + ibdev->gsi = ib_create_mi ( ibdev, IB_QPT_GSI ); + if ( ! ibdev->gsi ) { + DBGC ( ibdev, "IBDEV %p could not create GSI\n", ibdev ); rc = -ENOMEM; - goto err_create_gma; + goto err_create_gsi; } /* Open device */ @@ -571,8 +570,8 @@ int ib_open ( struct ib_device *ibdev ) { ibdev->op->close ( ibdev ); err_open: - ib_destroy_gma ( ibdev->gma ); - err_create_gma: + ib_destroy_mi ( ibdev, ibdev->gsi ); + err_create_gsi: ib_destroy_sma ( ibdev, ibdev->smi ); err_create_sma: ib_destroy_mi ( ibdev, ibdev->smi ); @@ -594,7 +593,7 @@ void ib_close ( struct ib_device *ibdev ) { /* Close device if this was the last remaining requested opening */ if ( ibdev->open_count == 0 ) { - ib_destroy_gma ( ibdev->gma ); + ib_destroy_mi ( ibdev, ibdev->gsi ); ib_destroy_sma ( ibdev, ibdev->smi ); ib_destroy_mi ( ibdev, ibdev->smi ); ibdev->op->close ( ibdev ); diff --git a/src/net/infiniband/ib_cm.c b/src/net/infiniband/ib_cm.c index b95ce9ff..01ce21c2 100644 --- a/src/net/infiniband/ib_cm.c +++ b/src/net/infiniband/ib_cm.c @@ -24,10 +24,8 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include -#include -#include #include -#include +#include #include #include @@ -38,64 +36,170 @@ FILE_LICENCE ( GPL2_OR_LATER ); * */ -/** An outstanding connection request */ -struct ib_cm_request { - /** List of all outstanding requests */ - struct list_head list; - /** Local communication ID */ - uint32_t local_id; - /** Remote communication ID */ - uint32_t remote_id; - /** Queue pair */ - struct ib_queue_pair *qp; - /** Target service ID */ - struct ib_gid_half service_id; - /** Connection process */ - struct process process; - /** Notification handler - * - * @v qp Queue pair - * @v rc Connection status code - * @v private_data Private data - * @v private_data_len Length of private data - */ - void ( * notify ) ( struct ib_queue_pair *qp, int rc, - void *private_data, size_t private_data_len ); - /** Private data length */ - size_t private_data_len; - /** Private data */ - uint8_t private_data[0]; -}; +/** + * Send "ready to use" response + * + * @v ibdev Infiniband device + * @v mi Management interface + * @v conn Connection + * @v av Address vector + * @ret rc Return status code + */ +static int ib_cm_send_rtu ( struct ib_device *ibdev, + struct ib_mad_interface *mi, + struct ib_connection *conn, + struct ib_address_vector *av ) { + union ib_mad mad; + struct ib_cm_ready_to_use *ready = + &mad.cm.cm_data.ready_to_use; + int rc; + + /* Construct "ready to use" response */ + memset ( &mad, 0, sizeof ( mad ) ); + mad.hdr.mgmt_class = IB_MGMT_CLASS_CM; + mad.hdr.class_version = IB_CM_CLASS_VERSION; + mad.hdr.method = IB_MGMT_METHOD_SEND; + mad.hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE ); + ready->local_id = htonl ( conn->local_id ); + ready->remote_id = htonl ( conn->remote_id ); + if ( ( rc = ib_mi_send ( ibdev, mi, &mad, av ) ) != 0 ){ + DBGC ( conn, "CM %p could not send RTU: %s\n", + conn, strerror ( rc ) ); + return rc; + } -/** List of all outstanding connection requests */ -static LIST_HEAD ( ib_cm_requests ); + return 0; +} /** - * Send connection request + * Handle connection request transaction completion * - * @v request Connection request - * @ret rc Return status code + * @v ibdev Infiniband device + * @v mi Management interface + * @v madx Management transaction + * @v rc Status code + * @v mad Received MAD (or NULL on error) + * @v av Source address vector (or NULL on error) */ -static int ib_cm_send_request ( struct ib_cm_request *request ) { - struct ib_queue_pair *qp = request->qp; - struct ib_device *ibdev = qp->ibdev; - struct ib_gma *gma = ibdev->gma; +static void ib_cm_req_complete ( struct ib_device *ibdev, + struct ib_mad_interface *mi, + struct ib_mad_transaction *madx, + int rc, union ib_mad *mad, + struct ib_address_vector *av ) { + struct ib_connection *conn = ib_madx_get_ownerdata ( madx ); + struct ib_queue_pair *qp = conn->qp; + struct ib_cm_common *common = &mad->cm.cm_data.common; + struct ib_cm_connect_reply *connect_rep = + &mad->cm.cm_data.connect_reply; + struct ib_cm_connect_reject *connect_rej = + &mad->cm.cm_data.connect_reject; + void *private_data = NULL; + size_t private_data_len = 0; + + /* Report failures */ + if ( rc != 0 ) { + DBGC ( conn, "CM %p connection request failed: %s\n", + conn, strerror ( rc ) ); + goto out; + } + + /* Record remote communication ID */ + conn->remote_id = ntohl ( common->local_id ); + + /* Handle response */ + switch ( mad->hdr.attr_id ) { + + case htons ( IB_CM_ATTR_CONNECT_REPLY ) : + /* Extract fields */ + qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 ); + qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 ); + private_data = &connect_rep->private_data; + private_data_len = sizeof ( connect_rep->private_data ); + DBGC ( conn, "CM %p connected to QPN %lx PSN %x\n", + conn, qp->av.qpn, qp->send.psn ); + + /* Modify queue pair */ + if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) { + DBGC ( conn, "CM %p could not modify queue pair: %s\n", + conn, strerror ( rc ) ); + goto out; + } + + /* Send "ready to use" reply */ + if ( ( rc = ib_cm_send_rtu ( ibdev, mi, conn, av ) ) != 0 ) { + /* Treat as non-fatal */ + rc = 0; + } + break; + + case htons ( IB_CM_ATTR_CONNECT_REJECT ) : + /* Extract fields */ + DBGC ( conn, "CM %p connection rejected (reason %d)\n", + conn, ntohs ( connect_rej->reason ) ); + private_data = &connect_rej->private_data; + private_data_len = sizeof ( connect_rej->private_data ); + rc = -ENOTCONN; + break; + + default: + DBGC ( conn, "CM %p unexpected response (attribute %04x)\n", + conn, ntohs ( mad->hdr.attr_id ) ); + rc = -EIO; + break; + } + + out: + /* Destroy the completed transaction */ + ib_destroy_madx ( ibdev, ibdev->gsi, madx ); + conn->madx = NULL; + + /* Hand off to the upper completion handler */ + conn->op->changed ( ibdev, qp, conn, rc, private_data, + private_data_len ); +} + +/** Connection request operations */ +static struct ib_mad_transaction_operations ib_cm_req_op = { + .complete = ib_cm_req_complete, +}; + +/** + * Handle connection path transaction completion + * + * @v ibdev Infiniband device + * @v path Path + * @v rc Status code + * @v av Address vector, or NULL on error + */ +static void ib_cm_path_complete ( struct ib_device *ibdev, + struct ib_path *path, int rc, + struct ib_address_vector *av ) { + struct ib_connection *conn = ib_path_get_ownerdata ( path ); + struct ib_queue_pair *qp = conn->qp; union ib_mad mad; - struct ib_mad_cm *cm = &mad.cm; struct ib_cm_connect_request *connect_req = - &cm->cm_data.connect_request; + &mad.cm.cm_data.connect_request; size_t private_data_len; - int rc; + + /* Report failures */ + if ( rc != 0 ) { + DBGC ( conn, "CM %p path lookup failed: %s\n", + conn, strerror ( rc ) ); + conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 ); + goto out; + } + + /* Update queue pair peer path */ + memcpy ( &qp->av, av, sizeof ( qp->av ) ); /* Construct connection request */ - memset ( cm, 0, sizeof ( *cm ) ); - cm->mad_hdr.base_version = IB_MGMT_BASE_VERSION; - cm->mad_hdr.mgmt_class = IB_MGMT_CLASS_CM; - cm->mad_hdr.class_version = IB_CM_CLASS_VERSION; - cm->mad_hdr.method = IB_MGMT_METHOD_SEND; - cm->mad_hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST ); - connect_req->local_id = htonl ( request->local_id ); - memcpy ( &connect_req->service_id, &request->service_id, + memset ( &mad, 0, sizeof ( mad ) ); + mad.hdr.mgmt_class = IB_MGMT_CLASS_CM; + mad.hdr.class_version = IB_CM_CLASS_VERSION; + mad.hdr.method = IB_MGMT_METHOD_SEND; + mad.hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST ); + connect_req->local_id = htonl ( conn->local_id ); + memcpy ( &connect_req->service_id, &conn->service_id, sizeof ( connect_req->service_id ) ); ib_get_hca_info ( ibdev, &connect_req->local_ca ); connect_req->local_qpn__responder_resources = @@ -113,217 +217,116 @@ static int ib_cm_send_request ( struct ib_cm_request *request ) { connect_req->max_cm_retries__srq = ( ( 0x0f << 4 ) | ( 0 << 3 ) ); connect_req->primary.local_lid = htons ( ibdev->lid ); - connect_req->primary.remote_lid = htons ( request->qp->av.lid ); + connect_req->primary.remote_lid = htons ( conn->qp->av.lid ); memcpy ( &connect_req->primary.local_gid, &ibdev->gid, sizeof ( connect_req->primary.local_gid ) ); - memcpy ( &connect_req->primary.remote_gid, &request->qp->av.gid, + memcpy ( &connect_req->primary.remote_gid, &conn->qp->av.gid, sizeof ( connect_req->primary.remote_gid ) ); connect_req->primary.flow_label__rate = - htonl ( ( 0 << 12 ) | ( request->qp->av.rate << 0 ) ); + htonl ( ( 0 << 12 ) | ( conn->qp->av.rate << 0 ) ); connect_req->primary.hop_limit = 0; connect_req->primary.sl__subnet_local = - ( ( request->qp->av.sl << 4 ) | ( 1 << 3 ) ); + ( ( conn->qp->av.sl << 4 ) | ( 1 << 3 ) ); connect_req->primary.local_ack_timeout = ( 0x13 << 3 ); - private_data_len = request->private_data_len; + private_data_len = conn->private_data_len; if ( private_data_len > sizeof ( connect_req->private_data ) ) private_data_len = sizeof ( connect_req->private_data ); - memcpy ( &connect_req->private_data, &request->private_data, + memcpy ( &connect_req->private_data, &conn->private_data, private_data_len ); - /* Send request */ - if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) { - DBGC ( gma, "GMA %p could not send connection request: %s\n", - gma, strerror ( rc ) ); - return rc; - } - - return 0; - -} - -/** - * Connection request process step - * - * @v process Connection request process - */ -static void ib_cm_step ( struct process *process ) { - struct ib_cm_request *request = - container_of ( process, struct ib_cm_request, process ); - struct ib_queue_pair *qp = request->qp; - struct ib_device *ibdev = qp->ibdev; - int rc; - - /* Wait until path can be resolved */ - if ( ( rc = ib_resolve_path ( ibdev, &request->qp->av ) ) != 0 ) - return; - - /* Wait until request can be sent */ - if ( ( rc = ib_cm_send_request ( request ) ) != 0 ) - return; - - /* Stop process */ - process_del ( process ); -} - -/** - * Identify connection request by communication ID - * - * @v local_id Local communication ID - * @v remote_id Remote communication ID - * @ret request Connection request, or NULL - */ -static struct ib_cm_request * ib_cm_find_request ( uint32_t local_id, - uint32_t remote_id ) { - struct ib_cm_request *request; - - list_for_each_entry ( request, &ib_cm_requests, list ) { - if ( request->local_id == local_id ) { - request->remote_id = remote_id; - return request; - } - } - return NULL; -} - -/** - * Handle connection reply - * - * @v gma General management agent - * @v mad MAD - * @ret response MAD response - */ -static union ib_mad * ib_cm_connect_reply ( struct ib_gma *gma, - union ib_mad *mad ) { - struct ib_cm_connect_reply *connect_rep = - &mad->cm.cm_data.connect_reply; - struct ib_cm_ready_to_use *ready = - &mad->cm.cm_data.ready_to_use; - struct ib_cm_request *request; - int rc; - - /* Identify request */ - request = ib_cm_find_request ( ntohl ( connect_rep->remote_id ), - ntohl ( connect_rep->local_id ) ); - if ( ! request ) { - DBGC ( gma, "GMA %p received connection reply with unknown " - "ID %08x\n", gma, ntohl ( connect_rep->remote_id ) ); - return NULL; + /* Create connection request */ + conn->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL, + &ib_cm_req_op ); + if ( ! conn->madx ) { + DBGC ( conn, "CM %p could not create connection request\n", + conn ); + conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 ); + goto out; } + ib_madx_set_ownerdata ( conn->madx, conn ); - /* Extract fields */ - request->qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 ); - request->qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 ); - DBGC ( gma, "GMA %p QPN %lx connected to QPN %lx PSN %x\n", gma, - request->qp->qpn, request->qp->av.qpn, request->qp->send.psn ); - - /* Modify queue pair */ - if ( ( rc = ib_modify_qp ( request->qp->ibdev, request->qp ) ) != 0 ) { - DBGC ( gma, "GMA %p QPN %lx could not modify queue pair: %s\n", - gma, request->qp->qpn, strerror ( rc ) ); - return NULL; - } - - /* Inform recipient that we are now connected */ - request->notify ( request->qp, 0, &connect_rep->private_data, - sizeof ( connect_rep->private_data ) ); - - /* Construct ready to use reply */ - mad->hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE ); - memset ( ready, 0, sizeof ( *ready ) ); - ready->local_id = htonl ( request->local_id ); - ready->remote_id = htonl ( request->remote_id ); - - return mad; + out: + /* Destroy the completed transaction */ + ib_destroy_path ( ibdev, path ); + conn->path = NULL; } -/** - * Handle connection rejection - * - * @v gma General management agent - * @v mad MAD - * @ret response MAD response - */ -static union ib_mad * ib_cm_connect_reject ( struct ib_gma *gma, - union ib_mad *mad ) { - struct ib_cm_connect_reject *connect_rej = - &mad->cm.cm_data.connect_reject; - struct ib_cm_request *request; - uint16_t reason; - - /* Identify request */ - request = ib_cm_find_request ( ntohl ( connect_rej->remote_id ), - ntohl ( connect_rej->local_id ) ); - if ( ! request ) { - DBGC ( gma, "GMA %p received connection rejection with " - "unknown ID %08x\n", gma, - ntohl ( connect_rej->remote_id ) ); - return NULL; - } - - /* Extract fields */ - reason = ntohs ( connect_rej->reason ); - DBGC ( gma, "GMA %p QPN %lx connection rejected (reason %d)\n", - gma, request->qp->qpn, reason ); - - /* Inform recipient that we are now disconnected */ - request->notify ( request->qp, -ENOTCONN, &connect_rej->private_data, - sizeof ( connect_rej->private_data ) ); - - return NULL; -} - -/** Communication management MAD handlers */ -struct ib_gma_handler ib_cm_handlers[] __ib_gma_handler = { - { - .mgmt_class = IB_MGMT_CLASS_CM, - .class_version = IB_CM_CLASS_VERSION, - .method = IB_MGMT_METHOD_SEND, - .attr_id = htons ( IB_CM_ATTR_CONNECT_REPLY ), - .handle = ib_cm_connect_reply, - }, - { - .mgmt_class = IB_MGMT_CLASS_CM, - .class_version = IB_CM_CLASS_VERSION, - .method = IB_MGMT_METHOD_SEND, - .attr_id = htons ( IB_CM_ATTR_CONNECT_REJECT ), - .handle = ib_cm_connect_reject, - }, +/** Connection path operations */ +static struct ib_path_operations ib_cm_path_op = { + .complete = ib_cm_path_complete, }; /** - * Connect to remote QP + * Create connection to remote QP * + * @v ibdev Infiniband device * @v qp Queue pair * @v dgid Target GID * @v service_id Target service ID - * @v private_data Private data - * @v private_data_len Length of private data - * @ret rc Return status code + * @v private_data Connection request private data + * @v private_data_len Length of connection request private data + * @v op Connection operations + * @ret conn Connection */ -int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid, - struct ib_gid_half *service_id, - void *private_data, size_t private_data_len, - void ( * notify ) ( struct ib_queue_pair *qp, int rc, - void *private_data, - size_t private_data_len ) ) { - struct ib_cm_request *request; +struct ib_connection * +ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp, + struct ib_gid *dgid, struct ib_gid_half *service_id, + void *private_data, size_t private_data_len, + struct ib_connection_operations *op ) { + struct ib_connection *conn; /* Allocate and initialise request */ - request = zalloc ( sizeof ( *request ) + private_data_len ); - if ( ! request ) - return -ENOMEM; - list_add ( &request->list, &ib_cm_requests ); - request->local_id = random(); - request->qp = qp; + conn = zalloc ( sizeof ( *conn ) + private_data_len ); + if ( ! conn ) + goto err_alloc_conn; + conn->ibdev = ibdev; + conn->qp = qp; memset ( &qp->av, 0, sizeof ( qp->av ) ); qp->av.gid_present = 1; memcpy ( &qp->av.gid, dgid, sizeof ( qp->av.gid ) ); - memcpy ( &request->service_id, service_id, - sizeof ( request->service_id ) ); - request->notify = notify; - request->private_data_len = private_data_len; - memcpy ( &request->private_data, private_data, private_data_len ); - process_init ( &request->process, ib_cm_step, NULL ); + conn->local_id = random(); + memcpy ( &conn->service_id, service_id, sizeof ( conn->service_id ) ); + conn->op = op; + conn->private_data_len = private_data_len; + memcpy ( &conn->private_data, private_data, private_data_len ); + + /* Create path */ + conn->path = ib_create_path ( ibdev, &qp->av, &ib_cm_path_op ); + if ( ! conn->path ) + goto err_create_path; + ib_path_set_ownerdata ( conn->path, conn ); + + DBGC ( conn, "CM %p created for IBDEV %p QPN %lx\n", + conn, ibdev, qp->qpn ); + DBGC ( conn, "CM %p connecting to %08x:%08x:%08x:%08x %08x:%08x\n", + conn, ntohl ( dgid->u.dwords[0] ), ntohl ( dgid->u.dwords[1] ), + ntohl ( dgid->u.dwords[2] ), ntohl ( dgid->u.dwords[3] ), + ntohl ( service_id->u.dwords[0] ), + ntohl ( service_id->u.dwords[1] ) ); + + return conn; + + ib_destroy_path ( ibdev, conn->path ); + err_create_path: + free ( conn ); + err_alloc_conn: + return NULL; +} - return 0; +/** + * Destroy connection to remote QP + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v conn Connection + */ +void ib_destroy_conn ( struct ib_device *ibdev, + struct ib_queue_pair *qp __unused, + struct ib_connection *conn ) { + + if ( conn->madx ) + ib_destroy_madx ( ibdev, ibdev->gsi, conn->madx ); + if ( conn->path ) + ib_destroy_path ( ibdev, conn->path ); + free ( conn ); } diff --git a/src/net/infiniband/ib_gma.c b/src/net/infiniband/ib_gma.c deleted file mode 100644 index f11a9fab..00000000 --- a/src/net/infiniband/ib_gma.c +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Copyright (C) 2009 Michael Brown . - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -FILE_LICENCE ( GPL2_OR_LATER ); - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/** - * @file - * - * Infiniband General Management Agent - * - */ - -/** A MAD request */ -struct ib_mad_request { - /** Associated GMA */ - struct ib_gma *gma; - /** List of outstanding MAD requests */ - struct list_head list; - /** Retry timer */ - struct retry_timer timer; - /** Destination address */ - struct ib_address_vector av; - /** MAD request */ - union ib_mad mad; -}; - -/** GMA number of send WQEs - * - * This is a policy decision. - */ -#define IB_GMA_NUM_SEND_WQES 4 - -/** GMA number of receive WQEs - * - * This is a policy decision. - */ -#define IB_GMA_NUM_RECV_WQES 2 - -/** GMA number of completion queue entries - * - * This is a policy decision - */ -#define IB_GMA_NUM_CQES 8 - -/** TID magic signature */ -#define IB_GMA_TID_MAGIC ( ( 'g' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' ) - -/** TID to use for next MAD request */ -static unsigned int next_request_tid; - -/***************************************************************************** - * - * General management agent - * - ***************************************************************************** - */ - -/** - * Call attribute handler - * - * @v gma General management agent - * @v mad MAD - * @ret mad MAD response - */ -static union ib_mad * ib_handle_mad ( struct ib_gma *gma, union ib_mad *mad ) { - struct ib_mad_hdr *hdr = &mad->hdr; - struct ib_gma_handler *handler; - - for_each_table_entry ( handler, IB_GMA_HANDLERS ) { - if ( ( ( handler->mgmt_class & ~handler->mgmt_class_ignore ) == - ( hdr->mgmt_class & ~handler->mgmt_class_ignore ) ) && - ( handler->class_version == hdr->class_version ) && - ( handler->method == hdr->method ) && - ( handler->attr_id == hdr->attr_id ) ) { - return handler->handle ( gma, mad ); - } - } - - hdr->method = IB_MGMT_METHOD_TRAP; - hdr->status = htons ( IB_MGMT_STATUS_UNSUPPORTED_METHOD_ATTR ); - return mad; -} - -/** - * Complete GMA receive - * - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v av Address vector - * @v iobuf I/O buffer - * @v rc Completion status code - */ -static void ib_gma_complete_recv ( struct ib_device *ibdev, - struct ib_queue_pair *qp, - struct ib_address_vector *av, - struct io_buffer *iobuf, int rc ) { - struct ib_gma *gma = ib_qp_get_ownerdata ( qp ); - struct ib_mad_request *request; - union ib_mad *mad; - struct ib_mad_hdr *hdr; - union ib_mad *response; - - /* Ignore errors */ - if ( rc != 0 ) { - DBGC ( gma, "GMA %p RX error: %s\n", gma, strerror ( rc ) ); - goto out; - } - - /* Sanity checks */ - if ( iob_len ( iobuf ) != sizeof ( *mad ) ) { - DBGC ( gma, "GMA %p RX bad size (%zd bytes)\n", - gma, iob_len ( iobuf ) ); - DBGC_HDA ( gma, 0, iobuf->data, iob_len ( iobuf ) ); - goto out; - } - mad = iobuf->data; - hdr = &mad->hdr; - if ( hdr->base_version != IB_MGMT_BASE_VERSION ) { - DBGC ( gma, "GMA %p unsupported base version %x\n", - gma, hdr->base_version ); - DBGC_HDA ( gma, 0, mad, sizeof ( *mad ) ); - goto out; - } - DBGC ( gma, "GMA %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status " - "%04x\n", gma, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ), - hdr->mgmt_class, hdr->class_version, hdr->method, - ntohs ( hdr->attr_id ), ntohs ( hdr->status ) ); - DBGC2_HDA ( gma, 0, mad, sizeof ( *mad ) ); - - /* Dequeue request if applicable */ - list_for_each_entry ( request, &gma->requests, list ) { - if ( memcmp ( &request->mad.hdr.tid, &hdr->tid, - sizeof ( request->mad.hdr.tid ) ) == 0 ) { - stop_timer ( &request->timer ); - list_del ( &request->list ); - free ( request ); - break; - } - } - - /* Handle MAD */ - if ( ( response = ib_handle_mad ( gma, mad ) ) == NULL ) - goto out; - - /* Re-use I/O buffer for response */ - memcpy ( mad, response, sizeof ( *mad ) ); - DBGC ( gma, "GMA %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status " - "%04x\n", gma, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ), - hdr->mgmt_class, hdr->class_version, hdr->method, - ntohs ( hdr->attr_id ), ntohs ( hdr->status ) ); - DBGC2_HDA ( gma, 0, mad, sizeof ( *mad ) ); - - /* Send MAD response, if applicable */ - if ( ( rc = ib_post_send ( ibdev, qp, av, - iob_disown ( iobuf ) ) ) != 0 ) { - DBGC ( gma, "GMA %p could not send MAD response: %s\n", - gma, strerror ( rc ) ); - goto out; - } - - out: - free_iob ( iobuf ); -} - -/** GMA completion operations */ -static struct ib_completion_queue_operations ib_gma_completion_ops = { - .complete_recv = ib_gma_complete_recv, -}; - -/** - * Transmit MAD request - * - * @v gma General management agent - * @v request MAD request - * @ret rc Return status code - */ -static int ib_gma_send ( struct ib_gma *gma, struct ib_mad_request *request ) { - struct io_buffer *iobuf; - int rc; - - DBGC ( gma, "GMA %p TX TID %08x%08x (%02x,%02x,%02x,%04x)\n", - gma, ntohl ( request->mad.hdr.tid[0] ), - ntohl ( request->mad.hdr.tid[1] ), request->mad.hdr.mgmt_class, - request->mad.hdr.class_version, request->mad.hdr.method, - ntohs ( request->mad.hdr.attr_id ) ); - DBGC2_HDA ( gma, 0, &request->mad, sizeof ( request->mad ) ); - - /* Construct I/O buffer */ - iobuf = alloc_iob ( sizeof ( request->mad ) ); - if ( ! iobuf ) { - DBGC ( gma, "GMA %p could not allocate buffer for TID " - "%08x%08x\n", gma, ntohl ( request->mad.hdr.tid[0] ), - ntohl ( request->mad.hdr.tid[1] ) ); - return -ENOMEM; - } - memcpy ( iob_put ( iobuf, sizeof ( request->mad ) ), &request->mad, - sizeof ( request->mad ) ); - - /* Send I/O buffer */ - if ( ( rc = ib_post_send ( gma->ibdev, gma->qp, &request->av, - iobuf ) ) != 0 ) { - DBGC ( gma, "GMA %p could not send TID %08x%08x: %s\n", - gma, ntohl ( request->mad.hdr.tid[0] ), - ntohl ( request->mad.hdr.tid[1] ), strerror ( rc ) ); - free_iob ( iobuf ); - return rc; - } - - return 0; -} - -/** - * Handle MAD request timer expiry - * - * @v timer Retry timer - * @v expired Failure indicator - */ -static void ib_gma_timer_expired ( struct retry_timer *timer, int expired ) { - struct ib_mad_request *request = - container_of ( timer, struct ib_mad_request, timer ); - struct ib_gma *gma = request->gma; - - /* Abandon TID if we have tried too many times */ - if ( expired ) { - DBGC ( gma, "GMA %p abandoning TID %08x%08x\n", - gma, ntohl ( request->mad.hdr.tid[0] ), - ntohl ( request->mad.hdr.tid[1] ) ); - list_del ( &request->list ); - free ( request ); - return; - } - - /* Restart retransmission timer */ - start_timer ( timer ); - - /* Resend request */ - ib_gma_send ( gma, request ); -} - -/** - * Issue MAD request - * - * @v gma General management agent - * @v mad MAD request - * @v av Destination address, or NULL for SM - * @v retry Request should be retried until a response arrives - * @ret rc Return status code - */ -int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad, - struct ib_address_vector *av, int retry ) { - struct ib_device *ibdev = gma->ibdev; - struct ib_mad_request *request; - - /* Allocate and initialise structure */ - request = zalloc ( sizeof ( *request ) ); - if ( ! request ) { - DBGC ( gma, "GMA %p could not allocate MAD request\n", gma ); - return -ENOMEM; - } - request->gma = gma; - request->timer.expired = ib_gma_timer_expired; - - /* Determine address vector */ - if ( av ) { - memcpy ( &request->av, av, sizeof ( request->av ) ); - } else { - request->av.lid = ibdev->sm_lid; - request->av.sl = ibdev->sm_sl; - request->av.qpn = IB_QPN_GSI; - request->av.qkey = IB_QKEY_GSI; - } - - /* Copy MAD body */ - memcpy ( &request->mad, mad, sizeof ( request->mad ) ); - - /* Allocate TID */ - request->mad.hdr.tid[0] = htonl ( IB_GMA_TID_MAGIC ); - request->mad.hdr.tid[1] = htonl ( ++next_request_tid ); - - /* Send initial request. Ignore errors; the retry timer will - * take care of those we care about. - */ - ib_gma_send ( gma, request ); - - /* Add to list and start timer if applicable */ - if ( retry ) { - list_add ( &request->list, &gma->requests ); - start_timer ( &request->timer ); - } else { - free ( request ); - } - - return 0; -} - -/** - * Create GMA - * - * @v ibdev Infiniband device - * @v type Queue pair type - * @ret gma General management agent, or NULL - */ -struct ib_gma * ib_create_gma ( struct ib_device *ibdev, - enum ib_queue_pair_type type ) { - struct ib_gma *gma; - int rc; - - /* Allocate and initialise fields */ - gma = zalloc ( sizeof ( *gma ) ); - if ( ! gma ) - goto err_alloc; - gma->ibdev = ibdev; - INIT_LIST_HEAD ( &gma->requests ); - - /* Create completion queue */ - gma->cq = ib_create_cq ( ibdev, IB_GMA_NUM_CQES, - &ib_gma_completion_ops ); - if ( ! gma->cq ) { - DBGC ( gma, "GMA %p could not allocate completion queue\n", - gma ); - goto err_create_cq; - } - - /* Create queue pair */ - gma->qp = ib_create_qp ( ibdev, type, IB_GMA_NUM_SEND_WQES, gma->cq, - IB_GMA_NUM_RECV_WQES, gma->cq ); - if ( ! gma->qp ) { - DBGC ( gma, "GMA %p could not allocate queue pair\n", gma ); - goto err_create_qp; - } - ib_qp_set_ownerdata ( gma->qp, gma ); - DBGC ( gma, "GMA %p running on QPN %#lx\n", gma, gma->qp->qpn ); - - /* Set queue key */ - gma->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI ); - if ( ( rc = ib_modify_qp ( ibdev, gma->qp ) ) != 0 ) { - DBGC ( gma, "GMA %p could not set queue key: %s\n", - gma, strerror ( rc ) ); - goto err_modify_qp; - } - - /* Fill receive ring */ - ib_refill_recv ( ibdev, gma->qp ); - return gma; - - err_modify_qp: - ib_destroy_qp ( ibdev, gma->qp ); - err_create_qp: - ib_destroy_cq ( ibdev, gma->cq ); - err_create_cq: - free ( gma ); - err_alloc: - return NULL; -} - -/** - * Destroy GMA - * - * @v gma General management agent - */ -void ib_destroy_gma ( struct ib_gma *gma ) { - struct ib_device *ibdev = gma->ibdev; - struct ib_mad_request *request; - struct ib_mad_request *tmp; - - /* Flush any outstanding requests */ - list_for_each_entry_safe ( request, tmp, &gma->requests, list ) { - stop_timer ( &request->timer ); - list_del ( &request->list ); - free ( request ); - } - - ib_destroy_qp ( ibdev, gma->qp ); - ib_destroy_cq ( ibdev, gma->cq ); - free ( gma ); -} diff --git a/src/net/infiniband/ib_mcast.c b/src/net/infiniband/ib_mcast.c index 58e555ad..c1d18d21 100644 --- a/src/net/infiniband/ib_mcast.c +++ b/src/net/infiniband/ib_mcast.c @@ -24,7 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include -#include +#include #include /** @file @@ -34,22 +34,19 @@ FILE_LICENCE ( GPL2_OR_LATER ); */ /** - * Transmit multicast group membership request + * Generate multicast membership MAD * - * @v gma General management agent + * @v ibdev Infiniband device * @v gid Multicast GID * @v join Join (rather than leave) group - * @ret rc Return status code + * @v mad MAD to fill in */ -static int ib_mc_member_request ( struct ib_gma *gma, struct ib_gid *gid, - int join ) { - union ib_mad mad; - struct ib_mad_sa *sa = &mad.sa; - int rc; +static void ib_mcast_mad ( struct ib_device *ibdev, struct ib_gid *gid, + int join, union ib_mad *mad ) { + struct ib_mad_sa *sa = &mad->sa; /* Construct multicast membership record request */ memset ( sa, 0, sizeof ( *sa ) ); - sa->mad_hdr.base_version = IB_MGMT_BASE_VERSION; sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; sa->mad_hdr.class_version = IB_SA_CLASS_VERSION; sa->mad_hdr.method = @@ -61,51 +58,123 @@ static int ib_mc_member_request ( struct ib_gma *gma, struct ib_gid *gid, sa->sa_data.mc_member_record.scope__join_state = 1; memcpy ( &sa->sa_data.mc_member_record.mgid, gid, sizeof ( sa->sa_data.mc_member_record.mgid ) ); - memcpy ( &sa->sa_data.mc_member_record.port_gid, &gma->ibdev->gid, + memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid, sizeof ( sa->sa_data.mc_member_record.port_gid ) ); +} - /* Issue multicast membership record request */ - if ( ( rc = ib_gma_request ( gma, &mad, NULL, join ) ) != 0 ) { - DBGC ( gma, "GMA %p could not join group: %s\n", - gma, strerror ( rc ) ); - return rc; +/** + * Handle multicast membership record join response + * + * @v ibdev Infiniband device + * @v mi Management interface + * @v madx Management transaction + * @v rc Status code + * @v mad Received MAD (or NULL on error) + * @v av Source address vector (or NULL on error) + */ +static void ib_mcast_complete ( struct ib_device *ibdev, + struct ib_mad_interface *mi __unused, + struct ib_mad_transaction *madx, + int rc, union ib_mad *mad, + struct ib_address_vector *av __unused ) { + struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx ); + struct ib_queue_pair *qp = membership->qp; + struct ib_gid *gid = &membership->gid; + struct ib_mc_member_record *mc_member_record = + &mad->sa.sa_data.mc_member_record; + int joined; + unsigned long qkey; + + /* Report failures */ + if ( rc != 0 ) { + DBGC ( ibdev, "IBDEV %p QPN %lx join failed: %s\n", + ibdev, qp->qpn, strerror ( rc ) ); + goto out; } - return 0; + /* Extract values from MAD */ + joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP ); + qkey = ntohl ( mc_member_record->qkey ); + DBGC ( ibdev, "IBDEV %p QPN %lx %s %08x:%08x:%08x:%08x qkey %lx\n", + ibdev, qp->qpn, ( joined ? "joined" : "left" ), + ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ), + ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ), + qkey ); + + /* Set queue key */ + qp->qkey = qkey; + if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p QPN %lx could not modify qkey: %s\n", + ibdev, qp->qpn, strerror ( rc ) ); + goto out; + } + + out: + /* Destroy the completed transaction */ + ib_destroy_madx ( ibdev, mi, madx ); + membership->madx = NULL; + + /* Hand off to upper completion handler */ + membership->complete ( ibdev, qp, membership, rc, mad ); } +/** Multicast membership management transaction completion operations */ +static struct ib_mad_transaction_operations ib_mcast_op = { + .complete = ib_mcast_complete, +}; + /** * Join multicast group * * @v ibdev Infiniband device * @v qp Queue pair - * @v gid Multicast GID + * @v membership Multicast group membership + * @v gid Multicast GID to join + * @v joined Join completion handler * @ret rc Return status code */ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp, - struct ib_gid *gid ) { - struct ib_gma *gma = ibdev->gma; + struct ib_mc_membership *membership, struct ib_gid *gid, + void ( * complete ) ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_mc_membership *membership, + int rc, union ib_mad *mad ) ) { + union ib_mad mad; int rc; - DBGC ( gma, "GMA %p QPN %lx joining %08x:%08x:%08x:%08x\n", - gma, qp->qpn, ntohl ( gid->u.dwords[0] ), + DBGC ( ibdev, "IBDEV %p QPN %lx joining %08x:%08x:%08x:%08x\n", + ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ) ); + /* Initialise structure */ + membership->qp = qp; + memcpy ( &membership->gid, gid, sizeof ( membership->gid ) ); + membership->complete = complete; + /* Attach queue pair to multicast GID */ if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) { - DBGC ( gma, "GMA %p could not attach: %s\n", - gma, strerror ( rc ) ); + DBGC ( ibdev, "IBDEV %p QPN %lx could not attach: %s\n", + ibdev, qp->qpn, strerror ( rc ) ); goto err_mcast_attach; } /* Initiate multicast membership join */ - if ( ( rc = ib_mc_member_request ( gma, gid, 1 ) ) != 0 ) - goto err_mc_member_record; + ib_mcast_mad ( ibdev, gid, 1, &mad ); + membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL, + &ib_mcast_op ); + if ( ! membership->madx ) { + DBGC ( ibdev, "IBDEV %p QPN %lx could not create join " + "transaction\n", ibdev, qp->qpn ); + rc = -ENOMEM; + goto err_create_madx; + } + ib_madx_set_ownerdata ( membership->madx, membership ); return 0; - err_mc_member_record: + ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx ); + err_create_madx: ib_mcast_detach ( ibdev, qp, gid ); err_mcast_attach: return rc; @@ -116,121 +185,32 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp, * * @v ibdev Infiniband device * @v qp Queue pair - * @v gid Multicast GID + * @v membership Multicast group membership */ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp, - struct ib_gid *gid ) { - struct ib_gma *gma = ibdev->gma; + struct ib_mc_membership *membership ) { + struct ib_gid *gid = &membership->gid; + union ib_mad mad; + int rc; - DBGC ( gma, "GMA %p QPN %lx leaving %08x:%08x:%08x:%08x\n", - gma, qp->qpn, ntohl ( gid->u.dwords[0] ), + DBGC ( ibdev, "IBDEV %p QPN %lx leaving %08x:%08x:%08x:%08x\n", + ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ) ); - /* Detach queue pair from multicast GID */ - ib_mcast_detach ( ibdev, qp, gid ); - - /* Initiate multicast membership leave */ - ib_mc_member_request ( gma, gid, 0 ); -} - -/** - * Handle multicast membership record join response - * - * @v gma General management agent - * @v mad MAD - * @ret mad MAD response - */ -static union ib_mad * ib_handle_mc_member_join ( struct ib_gma *gma, - union ib_mad *mad ) { - struct ib_device *ibdev = gma->ibdev; - struct ib_mc_member_record *mc_member_record = - &mad->sa.sa_data.mc_member_record; - struct ib_queue_pair *qp; - struct ib_gid *gid; - unsigned long qkey; - int rc; - - /* Ignore if not a success */ - if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) { - DBGC ( gma, "GMA %p join failed with status %04x\n", - gma, ntohs ( mad->hdr.status ) ); - return NULL; - } - - /* Extract MAD parameters */ - gid = &mc_member_record->mgid; - qkey = ntohl ( mc_member_record->qkey ); - - /* Locate matching queue pair */ - qp = ib_find_qp_mgid ( ibdev, gid ); - if ( ! qp ) { - DBGC ( gma, "GMA %p has no QP to join %08x:%08x:%08x:%08x\n", - gma, ntohl ( gid->u.dwords[0] ), - ntohl ( gid->u.dwords[1] ), - ntohl ( gid->u.dwords[2] ), - ntohl ( gid->u.dwords[3] ) ); - return NULL; - } - DBGC ( gma, "GMA %p QPN %lx joined %08x:%08x:%08x:%08x qkey %lx\n", - gma, qp->qpn, ntohl ( gid->u.dwords[0] ), - ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ), - ntohl ( gid->u.dwords[3] ), qkey ); + /* Detach from multicast GID */ + ib_mcast_detach ( ibdev, qp, &membership->gid ); - /* Set queue key */ - qp->qkey = qkey; - if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) { - DBGC ( gma, "GMA %p QPN %lx could not modify qkey: %s\n", - gma, qp->qpn, strerror ( rc ) ); - return NULL; + /* Cancel multicast membership join, if applicable */ + if ( membership->madx ) { + ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx ); + membership->madx = NULL; } - return NULL; -} - -/** - * Handle multicast membership record leave response - * - * @v gma General management agent - * @v mad MAD - * @v response MAD response - */ -static union ib_mad * ib_handle_mc_member_leave ( struct ib_gma *gma, - union ib_mad *mad ) { - struct ib_mc_member_record *mc_member_record = - &mad->sa.sa_data.mc_member_record; - struct ib_gid *gid; - - /* Ignore if not a success */ - if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) { - DBGC ( gma, "GMA %p leave failed with status %04x\n", - gma, ntohs ( mad->hdr.status ) ); - return NULL; + /* Send a single group leave MAD */ + ib_mcast_mad ( ibdev, &membership->gid, 0, &mad ); + if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) { + DBGC ( ibdev, "IBDEV %p QPN %lx could not send leave request: " + "%s\n", ibdev, qp->qpn, strerror ( rc ) ); } - - /* Extract MAD parameters */ - gid = &mc_member_record->mgid; - DBGC ( gma, "GMA %p left %08x:%08x:%08x:%08x\n", gma, - ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ), - ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ) ); - - return NULL; } - -/** Multicast membership record response handler */ -struct ib_gma_handler ib_mc_member_record_handlers[] __ib_gma_handler = { - { - .mgmt_class = IB_MGMT_CLASS_SUBN_ADM, - .class_version = IB_SA_CLASS_VERSION, - .method = IB_MGMT_METHOD_GET_RESP, - .attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ), - .handle = ib_handle_mc_member_join, - }, - { - .mgmt_class = IB_MGMT_CLASS_SUBN_ADM, - .class_version = IB_SA_CLASS_VERSION, - .method = IB_SA_METHOD_DELETE_RESP, - .attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ), - .handle = ib_handle_mc_member_leave, - }, -}; diff --git a/src/net/infiniband/ib_pathrec.c b/src/net/infiniband/ib_pathrec.c index f0c72988..983623c7 100644 --- a/src/net/infiniband/ib_pathrec.c +++ b/src/net/infiniband/ib_pathrec.c @@ -19,11 +19,12 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include +#include #include #include #include #include -#include +#include #include /** @file @@ -32,56 +33,162 @@ FILE_LICENCE ( GPL2_OR_LATER ); * */ -/** Number of path record cache entries +/** + * Handle path transaction completion + * + * @v ibdev Infiniband device + * @v mi Management interface + * @v madx Management transaction + * @v rc Status code + * @v mad Received MAD (or NULL on error) + * @v av Source address vector (or NULL on error) + */ +static void ib_path_complete ( struct ib_device *ibdev, + struct ib_mad_interface *mi, + struct ib_mad_transaction *madx, + int rc, union ib_mad *mad, + struct ib_address_vector *av __unused ) { + struct ib_path *path = ib_madx_get_ownerdata ( madx ); + struct ib_gid *dgid = &path->av.gid; + struct ib_path_record *pathrec = &mad->sa.sa_data.path_record; + + /* Report failures */ + if ( rc != 0 ) { + DBGC ( ibdev, "IBDEV %p path lookup for %08x:%08x:%08x:%08x " + "failed: %s\n", ibdev, htonl ( dgid->u.dwords[0] ), + htonl ( dgid->u.dwords[1] ), + htonl ( dgid->u.dwords[2] ), + htonl ( dgid->u.dwords[3] ), strerror ( rc ) ); + goto out; + } + + /* Extract values from MAD */ + path->av.lid = ntohs ( pathrec->dlid ); + path->av.sl = ( pathrec->reserved__sl & 0x0f ); + path->av.rate = ( pathrec->rate_selector__rate & 0x3f ); + DBGC ( ibdev, "IBDEV %p path to %08x:%08x:%08x:%08x is %04x sl %d " + "rate %d\n", ibdev, htonl ( dgid->u.dwords[0] ), + htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ), + htonl ( dgid->u.dwords[3] ), path->av.lid, path->av.sl, + path->av.rate ); + + out: + /* Destroy the completed transaction */ + ib_destroy_madx ( ibdev, mi, madx ); + path->madx = NULL; + + /* Hand off to upper completion handler */ + path->op->complete ( ibdev, path, rc, &path->av ); +} + +/** Path transaction completion operations */ +static struct ib_mad_transaction_operations ib_path_op = { + .complete = ib_path_complete, +}; + +/** + * Create path + * + * @v ibdev Infiniband device + * @v av Address vector to complete + * @v op Path operations + * @ret path Path + */ +struct ib_path * +ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av, + struct ib_path_operations *op ) { + struct ib_path *path; + union ib_mad mad; + struct ib_mad_sa *sa = &mad.sa; + + /* Allocate and initialise structure */ + path = zalloc ( sizeof ( *path ) ); + if ( ! path ) + goto err_alloc_path; + path->ibdev = ibdev; + memcpy ( &path->av, av, sizeof ( path->av ) ); + path->op = op; + + /* Construct path request */ + memset ( sa, 0, sizeof ( *sa ) ); + sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; + sa->mad_hdr.class_version = IB_SA_CLASS_VERSION; + sa->mad_hdr.method = IB_MGMT_METHOD_GET; + sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC ); + sa->sa_hdr.comp_mask[1] = + htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID ); + memcpy ( &sa->sa_data.path_record.dgid, &path->av.gid, + sizeof ( sa->sa_data.path_record.dgid ) ); + memcpy ( &sa->sa_data.path_record.sgid, &ibdev->gid, + sizeof ( sa->sa_data.path_record.sgid ) ); + + /* Create management transaction */ + path->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL, + &ib_path_op ); + if ( ! path->madx ) + goto err_create_madx; + ib_madx_set_ownerdata ( path->madx, path ); + + return path; + + ib_destroy_madx ( ibdev, ibdev->gsi, path->madx ); + err_create_madx: + free ( path ); + err_alloc_path: + return NULL; +} + +/** + * Destroy path + * + * @v ibdev Infiniband device + * @v path Path + */ +void ib_destroy_path ( struct ib_device *ibdev, struct ib_path *path ) { + + if ( path->madx ) + ib_destroy_madx ( ibdev, ibdev->gsi, path->madx ); + free ( path ); +} + +/** Number of path cache entries * * Must be a power of two. */ #define IB_NUM_CACHED_PATHS 4 -/** A path record cache entry */ -struct ib_cached_path_record { - /** Infiniband device's port GID - * - * Used to disambiguate cache entries when we have multiple - * Infiniband devices, without having to maintain a pointer to - * the Infiniband device. - */ - struct ib_gid sgid; - /** Destination GID */ - struct ib_gid dgid; - /** Destination LID */ - unsigned int dlid; - /** Rate */ - unsigned int rate; - /** Service level */ - unsigned int sl; +/** A cached path */ +struct ib_cached_path { + /** Path */ + struct ib_path *path; }; -/** Path record cache */ -static struct ib_cached_path_record ib_path_cache[IB_NUM_CACHED_PATHS]; +/** Path cache */ +static struct ib_cached_path ib_path_cache[IB_NUM_CACHED_PATHS]; -/** Oldest path record cache entry index */ +/** Oldest path cache entry index */ static unsigned int ib_path_cache_idx; /** - * Find path record cache entry + * Find path cache entry * * @v ibdev Infiniband device * @v dgid Destination GID - * @ret cached Path record cache entry, or NULL + * @ret path Path cache entry, or NULL */ -static struct ib_cached_path_record * +static struct ib_cached_path * ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) { - struct ib_cached_path_record *cached; + struct ib_cached_path *cached; unsigned int i; for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) { cached = &ib_path_cache[i]; - if ( memcmp ( &cached->sgid, &ibdev->gid, - sizeof ( cached->sgid ) ) != 0 ) + if ( ! cached->path ) + continue; + if ( cached->path->ibdev != ibdev ) continue; - if ( memcmp ( &cached->dgid, dgid, - sizeof ( cached->dgid ) ) != 0 ) + if ( memcmp ( &cached->path->av.gid, dgid, + sizeof ( cached->path->av.gid ) ) != 0 ) continue; return cached; } @@ -90,134 +197,98 @@ ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) { } /** - * Resolve path record + * Handle cached path transaction completion + * + * @v ibdev Infiniband device + * @v path Path + * @v rc Status code + * @v av Address vector, or NULL on error + */ +static void ib_cached_path_complete ( struct ib_device *ibdev, + struct ib_path *path, int rc, + struct ib_address_vector *av __unused ) { + struct ib_cached_path *cached = ib_path_get_ownerdata ( path ); + + /* If the transaction failed, erase the cache entry */ + if ( rc != 0 ) { + /* Destroy the old cache entry */ + ib_destroy_path ( ibdev, path ); + memset ( cached, 0, sizeof ( *cached ) ); + return; + } + + /* Do not destroy the completed transaction; we still need to + * refer to the resolved path. + */ +} + +/** Cached path transaction completion operations */ +static struct ib_path_operations ib_cached_path_op = { + .complete = ib_cached_path_complete, +}; + +/** + * Resolve path * * @v ibdev Infiniband device * @v av Address vector to complete * @ret rc Return status code + * + * This provides a non-transactional way to resolve a path, via a + * cache similar to ARP. */ -int ib_resolve_path ( struct ib_device *ibdev, - struct ib_address_vector *av ) { - struct ib_gma *gma = ibdev->gma; +int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ) { struct ib_gid *gid = &av->gid; - struct ib_cached_path_record *cached; - union ib_mad mad; - struct ib_mad_sa *sa = &mad.sa; + struct ib_cached_path *cached; unsigned int cache_idx; - int rc; /* Sanity check */ if ( ! av->gid_present ) { - DBGC ( gma, "GMA %p attempt to look up path record " - "without GID\n", gma ); + DBGC ( ibdev, "IBDEV %p attempt to look up path " + "without GID\n", ibdev ); return -EINVAL; } /* Look in cache for a matching entry */ cached = ib_find_path_cache_entry ( ibdev, gid ); - if ( cached && cached->dlid ) { + if ( cached && cached->path->av.lid ) { /* Populated entry found */ - av->lid = cached->dlid; - av->rate = cached->rate; - av->sl = cached->sl; - DBGC2 ( gma, "GMA %p cache hit for %08x:%08x:%08x:%08x\n", - gma, htonl ( gid->u.dwords[0] ), + av->lid = cached->path->av.lid; + av->rate = cached->path->av.rate; + av->sl = cached->path->av.sl; + DBGC2 ( ibdev, "IBDEV %p cache hit for %08x:%08x:%08x:%08x\n", + ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ) ); return 0; } - DBGC ( gma, "GMA %p cache miss for %08x:%08x:%08x:%08x%s\n", gma, - htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ), + DBGC ( ibdev, "IBDEV %p cache miss for %08x:%08x:%08x:%08x%s\n", + ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ), ( cached ? " (in progress)" : "" ) ); - /* If no unresolved entry was found, then create a new one */ - if ( ! cached ) { - cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS ); - cached = &ib_path_cache[cache_idx]; - memset ( cached, 0, sizeof ( *cached ) ); - memcpy ( &cached->sgid, &ibdev->gid, sizeof ( cached->sgid ) ); - memcpy ( &cached->dgid, gid, sizeof ( cached->dgid ) ); - } + /* If lookup is already in progress, do nothing */ + if ( cached ) + return -ENOENT; - /* Construct path record request */ - memset ( sa, 0, sizeof ( *sa ) ); - sa->mad_hdr.base_version = IB_MGMT_BASE_VERSION; - sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; - sa->mad_hdr.class_version = IB_SA_CLASS_VERSION; - sa->mad_hdr.method = IB_MGMT_METHOD_GET; - sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC ); - sa->sa_hdr.comp_mask[1] = - htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID ); - memcpy ( &sa->sa_data.path_record.dgid, &cached->dgid, - sizeof ( sa->sa_data.path_record.dgid ) ); - memcpy ( &sa->sa_data.path_record.sgid, &cached->sgid, - sizeof ( sa->sa_data.path_record.sgid ) ); + /* Locate a new cache entry to use */ + cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS ); + cached = &ib_path_cache[cache_idx]; + + /* Destroy the old cache entry */ + if ( cached->path ) + ib_destroy_path ( ibdev, cached->path ); + memset ( cached, 0, sizeof ( *cached ) ); - /* Issue path record request */ - if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) { - DBGC ( gma, "GMA %p could not get path record: %s\n", - gma, strerror ( rc ) ); - return rc; + /* Create new path */ + cached->path = ib_create_path ( ibdev, av, &ib_cached_path_op ); + if ( ! cached->path ) { + DBGC ( ibdev, "IBDEV %p could not create path\n", + ibdev ); + return -ENOMEM; } + ib_path_set_ownerdata ( cached->path, cached ); /* Not found yet */ return -ENOENT; } - -/** - * Handle path record response - * - * @v gma General management agent - * @v mad MAD - * @ret response MAD response - */ -static union ib_mad * ib_handle_path_record ( struct ib_gma *gma, - union ib_mad *mad ) { - struct ib_device *ibdev = gma->ibdev; - struct ib_path_record *path_record = &mad->sa.sa_data.path_record; - struct ib_gid *dgid = &path_record->dgid; - struct ib_cached_path_record *cached; - unsigned int dlid; - unsigned int sl; - unsigned int rate; - - /* Ignore if not a success */ - if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) { - DBGC ( gma, "GMA %p path record lookup failed with status " - "%04x\n", gma, ntohs ( mad->hdr.status ) ); - return NULL; - } - - /* Extract values from MAD */ - dlid = ntohs ( path_record->dlid ); - sl = ( path_record->reserved__sl & 0x0f ); - rate = ( path_record->rate_selector__rate & 0x3f ); - DBGC ( gma, "GMA %p path to %08x:%08x:%08x:%08x is %04x sl %d " - "rate %d\n", gma, htonl ( dgid->u.dwords[0] ), - htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ), - htonl ( dgid->u.dwords[3] ), dlid, sl, rate ); - - /* Look for a matching cache entry to fill in */ - if ( ( cached = ib_find_path_cache_entry ( ibdev, dgid ) ) != NULL ) { - DBGC ( gma, "GMA %p cache add for %08x:%08x:%08x:%08x\n", - gma, htonl ( dgid->u.dwords[0] ), - htonl ( dgid->u.dwords[1] ), - htonl ( dgid->u.dwords[2] ), - htonl ( dgid->u.dwords[3] ) ); - cached->dlid = dlid; - cached->rate = rate; - cached->sl = sl; - } - - return NULL; -} - -/** Path record response handler */ -struct ib_gma_handler ib_path_record_handler __ib_gma_handler = { - .mgmt_class = IB_MGMT_CLASS_SUBN_ADM, - .class_version = IB_SA_CLASS_VERSION, - .method = IB_MGMT_METHOD_GET_RESP, - .attr_id = htons ( IB_SA_ATTR_PATH_REC ), - .handle = ib_handle_path_record, -}; -- cgit v1.2.3-55-g7522