From ad66465b3c436b828d9631e41deb00f5a32f606a Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Thu, 6 Aug 2009 01:55:38 +0100 Subject: [hermon] Add support for multiple ports and detecting non-IB ports Originally-fixed-by: Itay Gazit --- src/drivers/infiniband/MT25408_PRM.h | 6 +- src/drivers/infiniband/hermon.c | 117 ++++++++++++++++++++++++++--------- src/drivers/infiniband/hermon.h | 18 +++++- 3 files changed, 110 insertions(+), 31 deletions(-) diff --git a/src/drivers/infiniband/MT25408_PRM.h b/src/drivers/infiniband/MT25408_PRM.h index 39ecd17f..419e25ac 100644 --- a/src/drivers/infiniband/MT25408_PRM.h +++ b/src/drivers/infiniband/MT25408_PRM.h @@ -2071,7 +2071,11 @@ struct hermonprm_query_dev_cap_st { /* Little Endian */ pseudo_bit_t pkv[0x00001]; /* PKey Violation Counter Supported */ pseudo_bit_t qkv[0x00001]; /* QKey Violation Coutner Supported */ pseudo_bit_t vmm[0x00001]; /* Hermon New */ - pseudo_bit_t reserved27[0x00005]; + pseudo_bit_t fcoe[0x00001]; + pseudo_bit_t dpdp[0x00001]; /* Dual Port Different Protocols */ + pseudo_bit_t raw_ethertype[0x00001]; + pseudo_bit_t raw_ipv6[0x00001]; + pseudo_bit_t blh[0x00001]; pseudo_bit_t mw[0x00001]; /* Memory windows supported */ pseudo_bit_t apm[0x00001]; /* Automatic Path Migration Supported */ pseudo_bit_t atm[0x00001]; /* Atomic operations supported (atomicity is guaranteed between QPs on this HCA) */ diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c index f4e228e2..136b843a 100644 --- a/src/drivers/infiniband/hermon.c +++ b/src/drivers/infiniband/hermon.c @@ -541,6 +541,16 @@ hermon_cmd_map_fa ( struct hermon *hermon, 0, map, 1, NULL ); } +static inline int +hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port, + struct hermonprm_sense_port *port_type ) { + return hermon_cmd ( hermon, + HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT, + 1, sizeof ( *port_type ) ), + 0, NULL, port, port_type ); +} + + /*************************************************************************** * * Memory translation table operations @@ -1664,7 +1674,7 @@ static void hermon_event_port_state_change ( struct hermon *hermon, ( link_up ? "up" : "down" ) ); /* Sanity check */ - if ( port >= HERMON_NUM_PORTS ) { + if ( port >= hermon->cap.num_ports ) { DBGC ( hermon, "Hermon %p port %d does not exist!\n", hermon, ( port + 1 ) ); return; @@ -1735,6 +1745,36 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) { *************************************************************************** */ +/** + * Sense port type + * + * @v ibdev Infiniband device + * @ret port_type Port type, or negative error + */ +static int hermon_sense_port_type ( struct ib_device *ibdev ) { + struct hermon *hermon = ib_get_drvdata ( ibdev ); + struct hermonprm_sense_port sense_port; + int port_type; + int rc; + + /* If DPDP is not supported, always assume Infiniband */ + if ( ! hermon->cap.dpdp ) + return HERMON_PORT_TYPE_IB; + + /* Sense the port type */ + if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port, + &sense_port ) ) != 0 ) { + DBGC ( hermon, "Hermon %p port %d sense failed: %s\n", + hermon, ibdev->port, strerror ( rc ) ); + return rc; + } + port_type = MLX_GET ( &sense_port, port_type ); + + DBGC ( hermon, "Hermon %p port %d type %d\n", + hermon, ibdev->port, port_type ); + return port_type; +} + /** * Initialise Infiniband link * @@ -1744,8 +1784,19 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) { static int hermon_open ( struct ib_device *ibdev ) { struct hermon *hermon = ib_get_drvdata ( ibdev ); struct hermonprm_init_port init_port; + int port_type; int rc; + /* Check we are connected to an Infiniband network */ + if ( ( rc = port_type = hermon_sense_port_type ( ibdev ) ) < 0 ) + return rc; + if ( port_type != HERMON_PORT_TYPE_IB ) { + DBGC ( hermon, "Hermon %p port %d not connected to an " + "Infiniband network", hermon, ibdev->port ); + return -ENOTCONN; + } + + /* Init Port */ memset ( &init_port, 0, sizeof ( init_port ) ); MLX_FILL_2 ( &init_port, 0, port_width_cap, 3, @@ -2099,6 +2150,15 @@ static int hermon_get_cap ( struct hermon *hermon ) { ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) ); hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz ); hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars ); + hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports ); + hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp ); + + /* Sanity check */ + if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) { + DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n", + hermon, hermon->cap.num_ports, HERMON_MAX_PORTS ); + hermon->cap.num_ports = HERMON_MAX_PORTS; + } return 0; } @@ -2478,7 +2538,7 @@ static int hermon_probe ( struct pci_device *pci, struct hermon *hermon; struct ib_device *ibdev; struct hermonprm_init_hca init_hca; - int i; + unsigned int i; int rc; /* Allocate Hermon device */ @@ -2489,20 +2549,6 @@ static int hermon_probe ( struct pci_device *pci, } pci_set_drvdata ( pci, hermon ); - /* Allocate Infiniband devices */ - for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) { - ibdev = alloc_ibdev ( 0 ); - if ( ! ibdev ) { - rc = -ENOMEM; - goto err_alloc_ibdev; - } - hermon->ibdev[i] = ibdev; - ibdev->op = &hermon_ib_operations; - ibdev->dev = &pci->dev; - ibdev->port = ( HERMON_PORT_BASE + i ); - ib_set_drvdata ( ibdev, hermon ); - } - /* Fix up PCI device */ adjust_pci_device ( pci ); @@ -2534,6 +2580,20 @@ static int hermon_probe ( struct pci_device *pci, if ( ( rc = hermon_get_cap ( hermon ) ) != 0 ) goto err_get_cap; + /* Allocate Infiniband devices */ + for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { + ibdev = alloc_ibdev ( 0 ); + if ( ! ibdev ) { + rc = -ENOMEM; + goto err_alloc_ibdev; + } + hermon->ibdev[i] = ibdev; + ibdev->op = &hermon_ib_operations; + ibdev->dev = &pci->dev; + ibdev->port = ( HERMON_PORT_BASE + i ); + ib_set_drvdata ( ibdev, hermon ); + } + /* Allocate ICM */ memset ( &init_hca, 0, sizeof ( init_hca ) ); if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 ) @@ -2552,7 +2612,7 @@ static int hermon_probe ( struct pci_device *pci, /* Set up memory protection */ if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 ) goto err_setup_mpt; - for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) + for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) hermon->ibdev[i]->rdma_key = hermon->lkey; /* Set up event queue */ @@ -2563,12 +2623,13 @@ static int hermon_probe ( struct pci_device *pci, if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 ) goto err_conf_special_qps; - /* Update MAD parameters */ - for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) + /* Update IPoIB MAC address */ + for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { ib_smc_update ( hermon->ibdev[i], hermon_mad ); + } /* Register Infiniband devices */ - for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) { + for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) { DBGC ( hermon, "Hermon %p could not register IB " "device: %s\n", hermon, strerror ( rc ) ); @@ -2578,9 +2639,9 @@ static int hermon_probe ( struct pci_device *pci, return 0; - i = HERMON_NUM_PORTS; + i = hermon->cap.num_ports; err_register_ibdev: - for ( i-- ; i >= 0 ; i-- ) + for ( i-- ; ( signed int ) i >= 0 ; i-- ) unregister_ibdev ( hermon->ibdev[i] ); err_conf_special_qps: hermon_destroy_eq ( hermon ); @@ -2590,6 +2651,10 @@ static int hermon_probe ( struct pci_device *pci, err_init_hca: hermon_free_icm ( hermon ); err_alloc_icm: + i = hermon->cap.num_ports; + err_alloc_ibdev: + for ( i-- ; ( signed int ) i >= 0 ; i-- ) + ibdev_put ( hermon->ibdev[i] ); err_get_cap: hermon_stop_firmware ( hermon ); err_start_firmware: @@ -2597,10 +2662,6 @@ static int hermon_probe ( struct pci_device *pci, err_mailbox_out: free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE ); err_mailbox_in: - i = HERMON_NUM_PORTS; - err_alloc_ibdev: - for ( i-- ; i >= 0 ; i-- ) - ibdev_put ( hermon->ibdev[i] ); free ( hermon ); err_alloc_hermon: return rc; @@ -2615,7 +2676,7 @@ static void hermon_remove ( struct pci_device *pci ) { struct hermon *hermon = pci_get_drvdata ( pci ); int i; - for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- ) + for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) unregister_ibdev ( hermon->ibdev[i] ); hermon_destroy_eq ( hermon ); hermon_cmd_close_hca ( hermon ); @@ -2624,7 +2685,7 @@ static void hermon_remove ( struct pci_device *pci ) { hermon_stop_firmware ( hermon ); free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE ); free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE ); - for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- ) + for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) ibdev_put ( hermon->ibdev[i] ); free ( hermon ); } diff --git a/src/drivers/infiniband/hermon.h b/src/drivers/infiniband/hermon.h index 8e5d35e8..ea973c0b 100644 --- a/src/drivers/infiniband/hermon.h +++ b/src/drivers/infiniband/hermon.h @@ -21,7 +21,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); */ /* Ports in existence */ -#define HERMON_NUM_PORTS 2 +#define HERMON_MAX_PORTS 2 #define HERMON_PORT_BASE 1 /* PCI BARs */ @@ -61,6 +61,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); #define HERMON_HCR_READ_MCG 0x0025 #define HERMON_HCR_WRITE_MCG 0x0026 #define HERMON_HCR_MGID_HASH 0x0027 +#define HERMON_HCR_SENSE_PORT 0x004d #define HERMON_HCR_RUN_FW 0x0ff6 #define HERMON_HCR_DISABLE_LAM 0x0ff7 #define HERMON_HCR_ENABLE_LAM 0x0ff8 @@ -164,6 +165,14 @@ struct hermonprm_port_state_change_event_st { struct hermonprm_port_state_change_st data; } __attribute__ (( packed )); +/** Hermon sense port */ +struct hermonprm_sense_port_st { + pseudo_bit_t port_type[0x00020]; +/* -------------- */ + pseudo_bit_t reserved[0x00020]; +}; +#define HERMON_PORT_TYPE_IB 1 + /* * Wrapper structures for hardware datatypes * @@ -192,6 +201,7 @@ struct MLX_DECLARE_STRUCT ( hermonprm_query_dev_cap ); struct MLX_DECLARE_STRUCT ( hermonprm_query_fw ); struct MLX_DECLARE_STRUCT ( hermonprm_queue_pair_ee_context_entry ); struct MLX_DECLARE_STRUCT ( hermonprm_scalar_parameter ); +struct MLX_DECLARE_STRUCT ( hermonprm_sense_port ); struct MLX_DECLARE_STRUCT ( hermonprm_send_db_register ); struct MLX_DECLARE_STRUCT ( hermonprm_ud_address_vector ); struct MLX_DECLARE_STRUCT ( hermonprm_virtual_physical_mapping ); @@ -296,6 +306,10 @@ struct hermon_dev_cap { size_t dmpt_entry_size; /** Number of reserved UARs */ unsigned int reserved_uars; + /** Number of ports */ + unsigned int num_ports; + /** Dual-port different protocol */ + int dpdp; }; /** Number of cMPT entries of each type */ @@ -523,7 +537,7 @@ struct hermon { unsigned long qpn_base; /** Infiniband devices */ - struct ib_device *ibdev[HERMON_NUM_PORTS]; + struct ib_device *ibdev[HERMON_MAX_PORTS]; }; /** Global protection domain */ -- cgit v1.2.3-55-g7522