diff options
Diffstat (limited to 'contrib/syslinux-4.02/gpxe/src/drivers/infiniband/linda.c')
| -rw-r--r-- | contrib/syslinux-4.02/gpxe/src/drivers/infiniband/linda.c | 2432 |
1 files changed, 2432 insertions, 0 deletions
diff --git a/contrib/syslinux-4.02/gpxe/src/drivers/infiniband/linda.c b/contrib/syslinux-4.02/gpxe/src/drivers/infiniband/linda.c new file mode 100644 index 0000000..b9a7ba5 --- /dev/null +++ b/contrib/syslinux-4.02/gpxe/src/drivers/infiniband/linda.c @@ -0,0 +1,2432 @@ +/* + * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +FILE_LICENCE ( GPL2_OR_LATER ); + +#include <stdint.h> +#include <stdlib.h> +#include <errno.h> +#include <unistd.h> +#include <assert.h> +#include <gpxe/io.h> +#include <gpxe/pci.h> +#include <gpxe/infiniband.h> +#include <gpxe/i2c.h> +#include <gpxe/bitbash.h> +#include <gpxe/malloc.h> +#include <gpxe/iobuf.h> +#include "linda.h" + +/** + * @file + * + * QLogic Linda Infiniband HCA + * + */ + +/** A Linda send work queue */ +struct linda_send_work_queue { + /** Send buffer usage */ + uint8_t *send_buf; + /** Producer index */ + unsigned int prod; + /** Consumer index */ + unsigned int cons; +}; + +/** A Linda receive work queue */ +struct linda_recv_work_queue { + /** Receive header ring */ + void *header; + /** Receive header producer offset (written by hardware) */ + struct QIB_7220_scalar header_prod; + /** Receive header consumer offset */ + unsigned int header_cons; + /** Offset within register space of the eager array */ + unsigned long eager_array; + /** Number of entries in eager array */ + unsigned int eager_entries; + /** Eager array producer index */ + unsigned int eager_prod; + /** Eager array consumer index */ + unsigned int eager_cons; +}; + +/** A Linda HCA */ +struct linda { + /** Registers */ + void *regs; + + /** In-use contexts */ + uint8_t used_ctx[LINDA_NUM_CONTEXTS]; + /** Send work queues */ + struct linda_send_work_queue send_wq[LINDA_NUM_CONTEXTS]; + /** Receive work queues */ + struct linda_recv_work_queue recv_wq[LINDA_NUM_CONTEXTS]; + + /** Offset within register space of the first send buffer */ + unsigned long send_buffer_base; + /** Send buffer availability (reported by hardware) */ + struct QIB_7220_SendBufAvail *sendbufavail; + /** Send buffer availability (maintained by software) */ + uint8_t send_buf[LINDA_MAX_SEND_BUFS]; + /** Send buffer availability producer counter */ + unsigned int send_buf_prod; + /** Send buffer availability consumer counter */ + unsigned int send_buf_cons; + /** Number of reserved send buffers (across all QPs) */ + unsigned int reserved_send_bufs; + + /** I2C bit-bashing interface */ + struct i2c_bit_basher i2c; + /** I2C serial EEPROM */ + struct i2c_device eeprom; +}; + +/*************************************************************************** + * + * Linda register access + * + *************************************************************************** + * + * This card requires atomic 64-bit accesses. Strange things happen + * if you try to use 32-bit accesses; sometimes they work, sometimes + * they don't, sometimes you get random data. + * + * These accessors use the "movq" MMX instruction, and so won't work + * on really old Pentiums (which won't have PCIe anyway, so this is + * something of a moot point). + */ + +/** + * Read Linda qword register + * + * @v linda Linda device + * @v dwords Register buffer to read into + * @v offset Register offset + */ +static void linda_readq ( struct linda *linda, uint32_t *dwords, + unsigned long offset ) { + void *addr = ( linda->regs + offset ); + + __asm__ __volatile__ ( "movq (%1), %%mm0\n\t" + "movq %%mm0, (%0)\n\t" + : : "r" ( dwords ), "r" ( addr ) : "memory" ); + + DBGIO ( "[%08lx] => %08x%08x\n", + virt_to_phys ( addr ), dwords[1], dwords[0] ); +} +#define linda_readq( _linda, _ptr, _offset ) \ + linda_readq ( (_linda), (_ptr)->u.dwords, (_offset) ) +#define linda_readq_array8b( _linda, _ptr, _offset, _idx ) \ + linda_readq ( (_linda), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) ) +#define linda_readq_array64k( _linda, _ptr, _offset, _idx ) \ + linda_readq ( (_linda), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ) ) + +/** + * Write Linda qword register + * + * @v linda Linda device + * @v dwords Register buffer to write + * @v offset Register offset + */ +static void linda_writeq ( struct linda *linda, const uint32_t *dwords, + unsigned long offset ) { + void *addr = ( linda->regs + offset ); + + DBGIO ( "[%08lx] <= %08x%08x\n", + virt_to_phys ( addr ), dwords[1], dwords[0] ); + + __asm__ __volatile__ ( "movq (%0), %%mm0\n\t" + "movq %%mm0, (%1)\n\t" + : : "r" ( dwords ), "r" ( addr ) : "memory" ); +} +#define linda_writeq( _linda, _ptr, _offset ) \ + linda_writeq ( (_linda), (_ptr)->u.dwords, (_offset) ) +#define linda_writeq_array8b( _linda, _ptr, _offset, _idx ) \ + linda_writeq ( (_linda), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) ) +#define linda_writeq_array64k( _linda, _ptr, _offset, _idx ) \ + linda_writeq ( (_linda), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ) ) + +/** + * Write Linda dword register + * + * @v linda Linda device + * @v dword Value to write + * @v offset Register offset + */ +static void linda_writel ( struct linda *linda, uint32_t dword, + unsigned long offset ) { + writel ( dword, ( linda->regs + offset ) ); +} + +/*************************************************************************** + * + * Link state management + * + *************************************************************************** + */ + +/** + * Textual representation of link state + * + * @v link_state Link state + * @ret link_text Link state text + */ +static const char * linda_link_state_text ( unsigned int link_state ) { + switch ( link_state ) { + case LINDA_LINK_STATE_DOWN: return "DOWN"; + case LINDA_LINK_STATE_INIT: return "INIT"; + case LINDA_LINK_STATE_ARM: return "ARM"; + case LINDA_LINK_STATE_ACTIVE: return "ACTIVE"; + case LINDA_LINK_STATE_ACT_DEFER:return "ACT_DEFER"; + default: return "UNKNOWN"; + } +} + +/** + * Handle link state change + * + * @v linda Linda device + */ +static void linda_link_state_changed ( struct ib_device *ibdev ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct QIB_7220_IBCStatus ibcstatus; + struct QIB_7220_EXTCtrl extctrl; + unsigned int link_state; + unsigned int link_width; + unsigned int link_speed; + + /* Read link state */ + linda_readq ( linda, &ibcstatus, QIB_7220_IBCStatus_offset ); + link_state = BIT_GET ( &ibcstatus, LinkState ); + link_width = BIT_GET ( &ibcstatus, LinkWidthActive ); + link_speed = BIT_GET ( &ibcstatus, LinkSpeedActive ); + DBGC ( linda, "Linda %p link state %s (%s %s)\n", linda, + linda_link_state_text ( link_state ), + ( link_speed ? "DDR" : "SDR" ), ( link_width ? "x4" : "x1" ) ); + + /* Set LEDs according to link state */ + linda_readq ( linda, &extctrl, QIB_7220_EXTCtrl_offset ); + BIT_SET ( &extctrl, LEDPriPortGreenOn, + ( ( link_state >= LINDA_LINK_STATE_INIT ) ? 1 : 0 ) ); + BIT_SET ( &extctrl, LEDPriPortYellowOn, + ( ( link_state >= LINDA_LINK_STATE_ACTIVE ) ? 1 : 0 ) ); + linda_writeq ( linda, &extctrl, QIB_7220_EXTCtrl_offset ); + + /* Notify Infiniband core of link state change */ + ibdev->port_state = ( link_state + 1 ); + ibdev->link_width_active = + ( link_width ? IB_LINK_WIDTH_4X : IB_LINK_WIDTH_1X ); + ibdev->link_speed_active = + ( link_speed ? IB_LINK_SPEED_DDR : IB_LINK_SPEED_SDR ); + ib_link_state_changed ( ibdev ); +} + +/** + * Wait for link state change to take effect + * + * @v linda Linda device + * @v new_link_state Expected link state + * @ret rc Return status code + */ +static int linda_link_state_check ( struct linda *linda, + unsigned int new_link_state ) { + struct QIB_7220_IBCStatus ibcstatus; + unsigned int link_state; + unsigned int i; + + for ( i = 0 ; i < LINDA_LINK_STATE_MAX_WAIT_US ; i++ ) { + linda_readq ( linda, &ibcstatus, QIB_7220_IBCStatus_offset ); + link_state = BIT_GET ( &ibcstatus, LinkState ); + if ( link_state == new_link_state ) + return 0; + udelay ( 1 ); + } + + DBGC ( linda, "Linda %p timed out waiting for link state %s\n", + linda, linda_link_state_text ( link_state ) ); + return -ETIMEDOUT; +} + +/** + * Set port information + * + * @v ibdev Infiniband device + * @v mad Set port information MAD + */ +static int linda_set_port_info ( struct ib_device *ibdev, union ib_mad *mad ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_port_info *port_info = &mad->smp.smp_data.port_info; + struct QIB_7220_IBCCtrl ibcctrl; + unsigned int port_state; + unsigned int link_state; + + /* Set new link state */ + port_state = ( port_info->link_speed_supported__port_state & 0xf ); + if ( port_state ) { + link_state = ( port_state - 1 ); + DBGC ( linda, "Linda %p set link state to %s (%x)\n", linda, + linda_link_state_text ( link_state ), link_state ); + linda_readq ( linda, &ibcctrl, QIB_7220_IBCCtrl_offset ); + BIT_SET ( &ibcctrl, LinkCmd, link_state ); + linda_writeq ( linda, &ibcctrl, QIB_7220_IBCCtrl_offset ); + + /* Wait for link state change to take effect. Ignore + * errors; the current link state will be returned via + * the GetResponse MAD. + */ + linda_link_state_check ( linda, link_state ); + } + + /* Detect and report link state change */ + linda_link_state_changed ( ibdev ); + + return 0; +} + +/** + * Set partition key table + * + * @v ibdev Infiniband device + * @v mad Set partition key table MAD + */ +static int linda_set_pkey_table ( struct ib_device *ibdev __unused, + union ib_mad *mad __unused ) { + /* Nothing to do */ + return 0; +} + +/*************************************************************************** + * + * Context allocation + * + *************************************************************************** + */ + +/** + * Map context number to QPN + * + * @v ctx Context index + * @ret qpn Queue pair number + */ +static int linda_ctx_to_qpn ( unsigned int ctx ) { + /* This mapping is fixed by hardware */ + return ( ctx * 2 ); +} + +/** + * Map QPN to context number + * + * @v qpn Queue pair number + * @ret ctx Context index + */ +static int linda_qpn_to_ctx ( unsigned int qpn ) { + /* This mapping is fixed by hardware */ + return ( qpn / 2 ); +} + +/** + * Allocate a context + * + * @v linda Linda device + * @ret ctx Context index, or negative error + */ +static int linda_alloc_ctx ( struct linda *linda ) { + unsigned int ctx; + + for ( ctx = 0 ; ctx < LINDA_NUM_CONTEXTS ; ctx++ ) { + + if ( ! linda->used_ctx[ctx] ) { + linda->used_ctx[ctx ] = 1; + DBGC2 ( linda, "Linda %p CTX %d allocated\n", + linda, ctx ); + return ctx; + } + } + + DBGC ( linda, "Linda %p out of available contexts\n", linda ); + return -ENOENT; +} + +/** + * Free a context + * + * @v linda Linda device + * @v ctx Context index + */ +static void linda_free_ctx ( struct linda *linda, unsigned int ctx ) { + + linda->used_ctx[ctx] = 0; + DBGC2 ( linda, "Linda %p CTX %d freed\n", linda, ctx ); +} + +/*************************************************************************** + * + * Send datapath + * + *************************************************************************** + */ + +/** Send buffer toggle bit + * + * We encode send buffers as 7 bits of send buffer index plus a single + * bit which should match the "check" bit in the SendBufAvail array. + */ +#define LINDA_SEND_BUF_TOGGLE 0x80 + +/** + * Allocate a send buffer + * + * @v linda Linda device + * @ret send_buf Send buffer + * + * You must guarantee that a send buffer is available. This is done + * by refusing to allocate more TX WQEs in total than the number of + * available send buffers. + */ +static unsigned int linda_alloc_send_buf ( struct linda *linda ) { + unsigned int send_buf; + + send_buf = linda->send_buf[linda->send_buf_cons]; + send_buf ^= LINDA_SEND_BUF_TOGGLE; + linda->send_buf_cons = ( ( linda->send_buf_cons + 1 ) % + LINDA_MAX_SEND_BUFS ); + return send_buf; +} + +/** + * Free a send buffer + * + * @v linda Linda device + * @v send_buf Send buffer + */ +static void linda_free_send_buf ( struct linda *linda, + unsigned int send_buf ) { + linda->send_buf[linda->send_buf_prod] = send_buf; + linda->send_buf_prod = ( ( linda->send_buf_prod + 1 ) % + LINDA_MAX_SEND_BUFS ); +} + +/** + * Check to see if send buffer is in use + * + * @v linda Linda device + * @v send_buf Send buffer + * @ret in_use Send buffer is in use + */ +static int linda_send_buf_in_use ( struct linda *linda, + unsigned int send_buf ) { + unsigned int send_idx; + unsigned int send_check; + unsigned int inusecheck; + unsigned int inuse; + unsigned int check; + + send_idx = ( send_buf & ~LINDA_SEND_BUF_TOGGLE ); + send_check = ( !! ( send_buf & LINDA_SEND_BUF_TOGGLE ) ); + inusecheck = BIT_GET ( linda->sendbufavail, InUseCheck[send_idx] ); + inuse = ( !! ( inusecheck & 0x02 ) ); + check = ( !! ( inusecheck & 0x01 ) ); + return ( inuse || ( check != send_check ) ); +} + +/** + * Calculate starting offset for send buffer + * + * @v linda Linda device + * @v send_buf Send buffer + * @ret offset Starting offset + */ +static unsigned long linda_send_buffer_offset ( struct linda *linda, + unsigned int send_buf ) { + return ( linda->send_buffer_base + + ( ( send_buf & ~LINDA_SEND_BUF_TOGGLE ) * + LINDA_SEND_BUF_SIZE ) ); +} + +/** + * Create send work queue + * + * @v linda Linda device + * @v qp Queue pair + */ +static int linda_create_send_wq ( struct linda *linda, + struct ib_queue_pair *qp ) { + struct ib_work_queue *wq = &qp->send; + struct linda_send_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + int rc; + + /* Reserve send buffers */ + if ( ( linda->reserved_send_bufs + qp->send.num_wqes ) > + LINDA_MAX_SEND_BUFS ) { + DBGC ( linda, "Linda %p out of send buffers (have %d, used " + "%d, need %d)\n", linda, LINDA_MAX_SEND_BUFS, + linda->reserved_send_bufs, qp->send.num_wqes ); + rc = -ENOBUFS; + goto err_reserve_bufs; + } + linda->reserved_send_bufs += qp->send.num_wqes; + + /* Reset work queue */ + linda_wq->prod = 0; + linda_wq->cons = 0; + + /* Allocate space for send buffer uasge list */ + linda_wq->send_buf = zalloc ( qp->send.num_wqes * + sizeof ( linda_wq->send_buf[0] ) ); + if ( ! linda_wq->send_buf ) { + rc = -ENOBUFS; + goto err_alloc_send_buf; + } + + return 0; + + free ( linda_wq->send_buf ); + err_alloc_send_buf: + linda->reserved_send_bufs -= qp->send.num_wqes; + err_reserve_bufs: + return rc; +} + +/** + * Destroy send work queue + * + * @v linda Linda device + * @v qp Queue pair + */ +static void linda_destroy_send_wq ( struct linda *linda, + struct ib_queue_pair *qp ) { + struct ib_work_queue *wq = &qp->send; + struct linda_send_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + + free ( linda_wq->send_buf ); + linda->reserved_send_bufs -= qp->send.num_wqes; +} + +/** + * Initialise send datapath + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_init_send ( struct linda *linda ) { + struct QIB_7220_SendBufBase sendbufbase; + struct QIB_7220_SendBufAvailAddr sendbufavailaddr; + struct QIB_7220_SendCtrl sendctrl; + unsigned int i; + int rc; + + /* Retrieve SendBufBase */ + linda_readq ( linda, &sendbufbase, QIB_7220_SendBufBase_offset ); + linda->send_buffer_base = BIT_GET ( &sendbufbase, + BaseAddr_SmallPIO ); + DBGC ( linda, "Linda %p send buffers at %lx\n", + linda, linda->send_buffer_base ); + + /* Initialise the send_buf[] array */ + for ( i = 0 ; i < LINDA_MAX_SEND_BUFS ; i++ ) + linda->send_buf[i] = i; + + /* Allocate space for the SendBufAvail array */ + linda->sendbufavail = malloc_dma ( sizeof ( *linda->sendbufavail ), + LINDA_SENDBUFAVAIL_ALIGN ); + if ( ! linda->sendbufavail ) { + rc = -ENOMEM; + goto err_alloc_sendbufavail; + } + memset ( linda->sendbufavail, 0, sizeof ( linda->sendbufavail ) ); + + /* Program SendBufAvailAddr into the hardware */ + memset ( &sendbufavailaddr, 0, sizeof ( sendbufavailaddr ) ); + BIT_FILL_1 ( &sendbufavailaddr, SendBufAvailAddr, + ( virt_to_bus ( linda->sendbufavail ) >> 6 ) ); + linda_writeq ( linda, &sendbufavailaddr, + QIB_7220_SendBufAvailAddr_offset ); + + /* Enable sending and DMA of SendBufAvail */ + memset ( &sendctrl, 0, sizeof ( sendctrl ) ); + BIT_FILL_2 ( &sendctrl, + SendBufAvailUpd, 1, + SPioEnable, 1 ); + linda_writeq ( linda, &sendctrl, QIB_7220_SendCtrl_offset ); + + return 0; + + free_dma ( linda->sendbufavail, sizeof ( *linda->sendbufavail ) ); + err_alloc_sendbufavail: + return rc; +} + +/** + * Shut down send datapath + * + * @v linda Linda device + */ +static void linda_fini_send ( struct linda *linda ) { + struct QIB_7220_SendCtrl sendctrl; + + /* Disable sending and DMA of SendBufAvail */ + memset ( &sendctrl, 0, sizeof ( sendctrl ) ); + linda_writeq ( linda, &sendctrl, QIB_7220_SendCtrl_offset ); + mb(); + + /* Ensure hardware has seen this disable */ + linda_readq ( linda, &sendctrl, QIB_7220_SendCtrl_offset ); + + free_dma ( linda->sendbufavail, sizeof ( *linda->sendbufavail ) ); +} + +/*************************************************************************** + * + * Receive datapath + * + *************************************************************************** + */ + +/** + * Create receive work queue + * + * @v linda Linda device + * @v qp Queue pair + * @ret rc Return status code + */ +static int linda_create_recv_wq ( struct linda *linda, + struct ib_queue_pair *qp ) { + struct ib_work_queue *wq = &qp->recv; + struct linda_recv_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct QIB_7220_RcvHdrAddr0 rcvhdraddr; + struct QIB_7220_RcvHdrTailAddr0 rcvhdrtailaddr; + struct QIB_7220_RcvHdrHead0 rcvhdrhead; + struct QIB_7220_scalar rcvegrindexhead; + struct QIB_7220_RcvCtrl rcvctrl; + unsigned int ctx = linda_qpn_to_ctx ( qp->qpn ); + int rc; + + /* Reset context information */ + memset ( &linda_wq->header_prod, 0, + sizeof ( linda_wq->header_prod ) ); + linda_wq->header_cons = 0; + linda_wq->eager_prod = 0; + linda_wq->eager_cons = 0; + + /* Allocate receive header buffer */ + linda_wq->header = malloc_dma ( LINDA_RECV_HEADERS_SIZE, + LINDA_RECV_HEADERS_ALIGN ); + if ( ! linda_wq->header ) { + rc = -ENOMEM; + goto err_alloc_header; + } + + /* Enable context in hardware */ + memset ( &rcvhdraddr, 0, sizeof ( rcvhdraddr ) ); + BIT_FILL_1 ( &rcvhdraddr, RcvHdrAddr0, + ( virt_to_bus ( linda_wq->header ) >> 2 ) ); + linda_writeq_array8b ( linda, &rcvhdraddr, + QIB_7220_RcvHdrAddr0_offset, ctx ); + memset ( &rcvhdrtailaddr, 0, sizeof ( rcvhdrtailaddr ) ); + BIT_FILL_1 ( &rcvhdrtailaddr, RcvHdrTailAddr0, + ( virt_to_bus ( &linda_wq->header_prod ) >> 2 ) ); + linda_writeq_array8b ( linda, &rcvhdrtailaddr, + QIB_7220_RcvHdrTailAddr0_offset, ctx ); + memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) ); + BIT_FILL_1 ( &rcvhdrhead, counter, 1 ); + linda_writeq_array64k ( linda, &rcvhdrhead, + QIB_7220_RcvHdrHead0_offset, ctx ); + memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) ); + BIT_FILL_1 ( &rcvegrindexhead, Value, 1 ); + linda_writeq_array64k ( linda, &rcvegrindexhead, + QIB_7220_RcvEgrIndexHead0_offset, ctx ); + linda_readq ( linda, &rcvctrl, QIB_7220_RcvCtrl_offset ); + BIT_SET ( &rcvctrl, PortEnable[ctx], 1 ); + BIT_SET ( &rcvctrl, IntrAvail[ctx], 1 ); + linda_writeq ( linda, &rcvctrl, QIB_7220_RcvCtrl_offset ); + + DBGC ( linda, "Linda %p QPN %ld CTX %d hdrs [%lx,%lx) prod %lx\n", + linda, qp->qpn, ctx, virt_to_bus ( linda_wq->header ), + ( virt_to_bus ( linda_wq->header ) + LINDA_RECV_HEADERS_SIZE ), + virt_to_bus ( &linda_wq->header_prod ) ); + return 0; + + free_dma ( linda_wq->header, LINDA_RECV_HEADERS_SIZE ); + err_alloc_header: + return rc; +} + +/** + * Destroy receive work queue + * + * @v linda Linda device + * @v qp Queue pair + */ +static void linda_destroy_recv_wq ( struct linda *linda, + struct ib_queue_pair *qp ) { + struct ib_work_queue *wq = &qp->recv; + struct linda_recv_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct QIB_7220_RcvCtrl rcvctrl; + unsigned int ctx = linda_qpn_to_ctx ( qp->qpn ); + + /* Disable context in hardware */ + linda_readq ( linda, &rcvctrl, QIB_7220_RcvCtrl_offset ); + BIT_SET ( &rcvctrl, PortEnable[ctx], 0 ); + BIT_SET ( &rcvctrl, IntrAvail[ctx], 0 ); + linda_writeq ( linda, &rcvctrl, QIB_7220_RcvCtrl_offset ); + + /* Make sure the hardware has seen that the context is disabled */ + linda_readq ( linda, &rcvctrl, QIB_7220_RcvCtrl_offset ); + mb(); + + /* Free headers ring */ + free_dma ( linda_wq->header, LINDA_RECV_HEADERS_SIZE ); + + /* Free context */ + linda_free_ctx ( linda, ctx ); +} + +/** + * Initialise receive datapath + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_init_recv ( struct linda *linda ) { + struct QIB_7220_RcvCtrl rcvctrl; + struct QIB_7220_scalar rcvegrbase; + struct QIB_7220_scalar rcvhdrentsize; + struct QIB_7220_scalar rcvhdrcnt; + struct QIB_7220_RcvBTHQP rcvbthqp; + unsigned int portcfg; + unsigned long egrbase; + unsigned int eager_array_size_0; + unsigned int eager_array_size_other; + unsigned int ctx; + + /* Select configuration based on number of contexts */ + switch ( LINDA_NUM_CONTEXTS ) { + case 5: + portcfg = LINDA_PORTCFG_5CTX; + eager_array_size_0 = LINDA_EAGER_ARRAY_SIZE_5CTX_0; + eager_array_size_other = LINDA_EAGER_ARRAY_SIZE_5CTX_OTHER; + break; + case 9: + portcfg = LINDA_PORTCFG_9CTX; + eager_array_size_0 = LINDA_EAGER_ARRAY_SIZE_9CTX_0; + eager_array_size_other = LINDA_EAGER_ARRAY_SIZE_9CTX_OTHER; + break; + case 17: + portcfg = LINDA_PORTCFG_17CTX; + eager_array_size_0 = LINDA_EAGER_ARRAY_SIZE_17CTX_0; + eager_array_size_other = LINDA_EAGER_ARRAY_SIZE_17CTX_OTHER; + break; + default: + linker_assert ( 0, invalid_LINDA_NUM_CONTEXTS ); + return -EINVAL; + } + + /* Configure number of contexts */ + memset ( &rcvctrl, 0, sizeof ( rcvctrl ) ); + BIT_FILL_3 ( &rcvctrl, + TailUpd, 1, + PortCfg, portcfg, + RcvQPMapEnable, 1 ); + linda_writeq ( linda, &rcvctrl, QIB_7220_RcvCtrl_offset ); + + /* Configure receive header buffer sizes */ + memset ( &rcvhdrcnt, 0, sizeof ( rcvhdrcnt ) ); + BIT_FILL_1 ( &rcvhdrcnt, Value, LINDA_RECV_HEADER_COUNT ); + linda_writeq ( linda, &rcvhdrcnt, QIB_7220_RcvHdrCnt_offset ); + memset ( &rcvhdrentsize, 0, sizeof ( rcvhdrentsize ) ); + BIT_FILL_1 ( &rcvhdrentsize, Value, ( LINDA_RECV_HEADER_SIZE >> 2 ) ); + linda_writeq ( linda, &rcvhdrentsize, QIB_7220_RcvHdrEntSize_offset ); + + /* Calculate eager array start addresses for each context */ + linda_readq ( linda, &rcvegrbase, QIB_7220_RcvEgrBase_offset ); + egrbase = BIT_GET ( &rcvegrbase, Value ); + linda->recv_wq[0].eager_array = egrbase; + linda->recv_wq[0].eager_entries = eager_array_size_0; + egrbase += ( eager_array_size_0 * sizeof ( struct QIB_7220_RcvEgr ) ); + for ( ctx = 1 ; ctx < LINDA_NUM_CONTEXTS ; ctx++ ) { + linda->recv_wq[ctx].eager_array = egrbase; + linda->recv_wq[ctx].eager_entries = eager_array_size_other; + egrbase += ( eager_array_size_other * + sizeof ( struct QIB_7220_RcvEgr ) ); + } + for ( ctx = 0 ; ctx < LINDA_NUM_CONTEXTS ; ctx++ ) { + DBGC ( linda, "Linda %p CTX %d eager array at %lx (%d " + "entries)\n", linda, ctx, + linda->recv_wq[ctx].eager_array, + linda->recv_wq[ctx].eager_entries ); + } + + /* Set the BTH QP for Infinipath packets to an unused value */ + memset ( &rcvbthqp, 0, sizeof ( rcvbthqp ) ); + BIT_FILL_1 ( &rcvbthqp, RcvBTHQP, LINDA_QP_IDETH ); + linda_writeq ( linda, &rcvbthqp, QIB_7220_RcvBTHQP_offset ); + + return 0; +} + +/** + * Shut down receive datapath + * + * @v linda Linda device + */ +static void linda_fini_recv ( struct linda *linda __unused ) { + /* Nothing to do; all contexts were already disabled when the + * queue pairs were destroyed + */ +} + +/*************************************************************************** + * + * Completion queue operations + * + *************************************************************************** + */ + +/** + * Create completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + * @ret rc Return status code + */ +static int linda_create_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + static int cqn; + + /* The hardware has no concept of completion queues. We + * simply use the association between CQs and WQs (already + * handled by the IB core) to decide which WQs to poll. + * + * We do set a CQN, just to avoid confusing debug messages + * from the IB core. + */ + cq->cqn = ++cqn; + DBGC ( linda, "Linda %p CQN %ld created\n", linda, cq->cqn ); + + return 0; +} + +/** + * Destroy completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + */ +static void linda_destroy_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + + /* Nothing to do */ + DBGC ( linda, "Linda %p CQN %ld destroyed\n", linda, cq->cqn ); +} + +/*************************************************************************** + * + * Queue pair operations + * + *************************************************************************** + */ + +/** + * Create queue pair + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @ret rc Return status code + */ +static int linda_create_qp ( struct ib_device *ibdev, + struct ib_queue_pair *qp ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + int ctx; + int rc; + + /* Locate an available context */ + ctx = linda_alloc_ctx ( linda ); + if ( ctx < 0 ) { + rc = ctx; + goto err_alloc_ctx; + } + + /* Set queue pair number based on context index */ + qp->qpn = linda_ctx_to_qpn ( ctx ); + + /* Set work-queue private data pointers */ + ib_wq_set_drvdata ( &qp->send, &linda->send_wq[ctx] ); + ib_wq_set_drvdata ( &qp->recv, &linda->recv_wq[ctx] ); + + /* Create receive work queue */ + if ( ( rc = linda_create_recv_wq ( linda, qp ) ) != 0 ) + goto err_create_recv_wq; + + /* Create send work queue */ + if ( ( rc = linda_create_send_wq ( linda, qp ) ) != 0 ) + goto err_create_send_wq; + + return 0; + + linda_destroy_send_wq ( linda, qp ); + err_create_send_wq: + linda_destroy_recv_wq ( linda, qp ); + err_create_recv_wq: + linda_free_ctx ( linda, ctx ); + err_alloc_ctx: + return rc; +} + +/** + * Modify queue pair + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @ret rc Return status code + */ +static int linda_modify_qp ( struct ib_device *ibdev, + struct ib_queue_pair *qp ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + + /* Nothing to do; the hardware doesn't have a notion of queue + * keys + */ + DBGC ( linda, "Linda %p QPN %ld modified\n", linda, qp->qpn ); + return 0; +} + +/** + * Destroy queue pair + * + * @v ibdev Infiniband device + * @v qp Queue pair + */ +static void linda_destroy_qp ( struct ib_device *ibdev, + struct ib_queue_pair *qp ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + + linda_destroy_send_wq ( linda, qp ); + linda_destroy_recv_wq ( linda, qp ); +} + +/*************************************************************************** + * + * Work request operations + * + *************************************************************************** + */ + +/** + * Post send work queue entry + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v av Address vector + * @v iobuf I/O buffer + * @ret rc Return status code + */ +static int linda_post_send ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_address_vector *av, + struct io_buffer *iobuf ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_work_queue *wq = &qp->send; + struct linda_send_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct QIB_7220_SendPbc sendpbc; + uint8_t header_buf[IB_MAX_HEADER_SIZE]; + struct io_buffer headers; + unsigned int send_buf; + unsigned long start_offset; + unsigned long offset; + size_t len; + ssize_t frag_len; + uint32_t *data; + + /* Allocate send buffer and calculate offset */ + send_buf = linda_alloc_send_buf ( linda ); + start_offset = offset = linda_send_buffer_offset ( linda, send_buf ); + + /* Store I/O buffer and send buffer index */ + assert ( wq->iobufs[linda_wq->prod] == NULL ); + wq->iobufs[linda_wq->prod] = iobuf; + linda_wq->send_buf[linda_wq->prod] = send_buf; + + /* Construct headers */ + iob_populate ( &headers, header_buf, 0, sizeof ( header_buf ) ); + iob_reserve ( &headers, sizeof ( header_buf ) ); + ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), av ); + + /* Calculate packet length */ + len = ( ( sizeof ( sendpbc ) + iob_len ( &headers ) + + iob_len ( iobuf ) + 3 ) & ~3 ); + + /* Construct send per-buffer control word */ + memset ( &sendpbc, 0, sizeof ( sendpbc ) ); + BIT_FILL_2 ( &sendpbc, + LengthP1_toibc, ( ( len >> 2 ) - 1 ), + VL15, 1 ); + + /* Write SendPbc */ + DBG_DISABLE ( DBGLVL_IO ); + linda_writeq ( linda, &sendpbc, offset ); + offset += sizeof ( sendpbc ); + + /* Write headers */ + for ( data = headers.data, frag_len = iob_len ( &headers ) ; + frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) { + linda_writel ( linda, *data, offset ); + } + + /* Write data */ + for ( data = iobuf->data, frag_len = iob_len ( iobuf ) ; + frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) { + linda_writel ( linda, *data, offset ); + } + DBG_ENABLE ( DBGLVL_IO ); + + assert ( ( start_offset + len ) == offset ); + DBGC2 ( linda, "Linda %p QPN %ld TX %d(%d) posted [%lx,%lx)\n", + linda, qp->qpn, send_buf, linda_wq->prod, + start_offset, offset ); + + /* Increment producer counter */ + linda_wq->prod = ( ( linda_wq->prod + 1 ) & ( wq->num_wqes - 1 ) ); + + return 0; +} + +/** + * Complete send work queue entry + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v wqe_idx Work queue entry index + */ +static void linda_complete_send ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + unsigned int wqe_idx ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_work_queue *wq = &qp->send; + struct linda_send_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct io_buffer *iobuf; + unsigned int send_buf; + + /* Parse completion */ + send_buf = linda_wq->send_buf[wqe_idx]; + DBGC2 ( linda, "Linda %p QPN %ld TX %d(%d) complete\n", + linda, qp->qpn, send_buf, wqe_idx ); + + /* Complete work queue entry */ + iobuf = wq->iobufs[wqe_idx]; + assert ( iobuf != NULL ); + ib_complete_send ( ibdev, qp, iobuf, 0 ); + wq->iobufs[wqe_idx] = NULL; + + /* Free send buffer */ + linda_free_send_buf ( linda, send_buf ); +} + +/** + * Poll send work queue + * + * @v ibdev Infiniband device + * @v qp Queue pair + */ +static void linda_poll_send_wq ( struct ib_device *ibdev, + struct ib_queue_pair *qp ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_work_queue *wq = &qp->send; + struct linda_send_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + unsigned int send_buf; + + /* Look for completions */ + while ( wq->fill ) { + + /* Check to see if send buffer has completed */ + send_buf = linda_wq->send_buf[linda_wq->cons]; + if ( linda_send_buf_in_use ( linda, send_buf ) ) + break; + + /* Complete this buffer */ + linda_complete_send ( ibdev, qp, linda_wq->cons ); + + /* Increment consumer counter */ + linda_wq->cons = ( ( linda_wq->cons + 1 ) & + ( wq->num_wqes - 1 ) ); + } +} + +/** + * Post receive work queue entry + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v iobuf I/O buffer + * @ret rc Return status code + */ +static int linda_post_recv ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct io_buffer *iobuf ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_work_queue *wq = &qp->recv; + struct linda_recv_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct QIB_7220_RcvEgr rcvegr; + struct QIB_7220_scalar rcvegrindexhead; + unsigned int ctx = linda_qpn_to_ctx ( qp->qpn ); + physaddr_t addr; + size_t len; + unsigned int wqe_idx; + unsigned int bufsize; + + /* Sanity checks */ + addr = virt_to_bus ( iobuf->data ); + len = iob_tailroom ( iobuf ); + if ( addr & ( LINDA_EAGER_BUFFER_ALIGN - 1 ) ) { + DBGC ( linda, "Linda %p QPN %ld misaligned RX buffer " + "(%08lx)\n", linda, qp->qpn, addr ); + return -EINVAL; + } + if ( len != LINDA_RECV_PAYLOAD_SIZE ) { + DBGC ( linda, "Linda %p QPN %ld wrong RX buffer size (%zd)\n", + linda, qp->qpn, len ); + return -EINVAL; + } + + /* Calculate eager producer index and WQE index */ + wqe_idx = ( linda_wq->eager_prod & ( wq->num_wqes - 1 ) ); + assert ( wq->iobufs[wqe_idx] == NULL ); + + /* Store I/O buffer */ + wq->iobufs[wqe_idx] = iobuf; + + /* Calculate buffer size */ + switch ( LINDA_RECV_PAYLOAD_SIZE ) { + case 2048: bufsize = LINDA_EAGER_BUFFER_2K; break; + case 4096: bufsize = LINDA_EAGER_BUFFER_4K; break; + case 8192: bufsize = LINDA_EAGER_BUFFER_8K; break; + case 16384: bufsize = LINDA_EAGER_BUFFER_16K; break; + case 32768: bufsize = LINDA_EAGER_BUFFER_32K; break; + case 65536: bufsize = LINDA_EAGER_BUFFER_64K; break; + default: linker_assert ( 0, invalid_rx_payload_size ); + bufsize = LINDA_EAGER_BUFFER_NONE; + } + + /* Post eager buffer */ + memset ( &rcvegr, 0, sizeof ( rcvegr ) ); + BIT_FILL_2 ( &rcvegr, + Addr, ( addr >> 11 ), + BufSize, bufsize ); + linda_writeq_array8b ( linda, &rcvegr, + linda_wq->eager_array, linda_wq->eager_prod ); + DBGC2 ( linda, "Linda %p QPN %ld RX egr %d(%d) posted [%lx,%lx)\n", + linda, qp->qpn, linda_wq->eager_prod, wqe_idx, + addr, ( addr + len ) ); + + /* Increment producer index */ + linda_wq->eager_prod = ( ( linda_wq->eager_prod + 1 ) & + ( linda_wq->eager_entries - 1 ) ); + + /* Update head index */ + memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) ); + BIT_FILL_1 ( &rcvegrindexhead, + Value, ( ( linda_wq->eager_prod + 1 ) & + ( linda_wq->eager_entries - 1 ) ) ); + linda_writeq_array64k ( linda, &rcvegrindexhead, + QIB_7220_RcvEgrIndexHead0_offset, ctx ); + + return 0; +} + +/** + * Complete receive work queue entry + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v header_offs Header offset + */ +static void linda_complete_recv ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + unsigned int header_offs ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_work_queue *wq = &qp->recv; + struct linda_recv_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct QIB_7220_RcvHdrFlags *rcvhdrflags; + struct QIB_7220_RcvEgr rcvegr; + struct io_buffer headers; + struct io_buffer *iobuf; + struct ib_queue_pair *intended_qp; + struct ib_address_vector av; + unsigned int rcvtype; + unsigned int pktlen; + unsigned int egrindex; + unsigned int useegrbfr; + unsigned int iberr, mkerr, tiderr, khdrerr, mtuerr; + unsigned int lenerr, parityerr, vcrcerr, icrcerr; + unsigned int err; + unsigned int hdrqoffset; + unsigned int header_len; + unsigned int padded_payload_len; + unsigned int wqe_idx; + size_t payload_len; + int qp0; + int rc; + + /* RcvHdrFlags are at the end of the header entry */ + rcvhdrflags = ( linda_wq->header + header_offs + + LINDA_RECV_HEADER_SIZE - sizeof ( *rcvhdrflags ) ); + rcvtype = BIT_GET ( rcvhdrflags, RcvType ); + pktlen = ( BIT_GET ( rcvhdrflags, PktLen ) << 2 ); + egrindex = BIT_GET ( rcvhdrflags, EgrIndex ); + useegrbfr = BIT_GET ( rcvhdrflags, UseEgrBfr ); + hdrqoffset = ( BIT_GET ( rcvhdrflags, HdrqOffset ) << 2 ); + iberr = BIT_GET ( rcvhdrflags, IBErr ); + mkerr = BIT_GET ( rcvhdrflags, MKErr ); + tiderr = BIT_GET ( rcvhdrflags, TIDErr ); + khdrerr = BIT_GET ( rcvhdrflags, KHdrErr ); + mtuerr = BIT_GET ( rcvhdrflags, MTUErr ); + lenerr = BIT_GET ( rcvhdrflags, LenErr ); + parityerr = BIT_GET ( rcvhdrflags, ParityErr ); + vcrcerr = BIT_GET ( rcvhdrflags, VCRCErr ); + icrcerr = BIT_GET ( rcvhdrflags, ICRCErr ); + header_len = ( LINDA_RECV_HEADER_SIZE - hdrqoffset - + sizeof ( *rcvhdrflags ) ); + padded_payload_len = ( pktlen - header_len - 4 /* ICRC */ ); + err = ( iberr | mkerr | tiderr | khdrerr | mtuerr | + lenerr | parityerr | vcrcerr | icrcerr ); + /* IB header is placed immediately before RcvHdrFlags */ + iob_populate ( &headers, ( ( ( void * ) rcvhdrflags ) - header_len ), + header_len, header_len ); + + /* Dump diagnostic information */ + if ( err || ( ! useegrbfr ) ) { + DBGC ( linda, "Linda %p QPN %ld RX egr %d%s hdr %d type %d " + "len %d(%d+%d+4)%s%s%s%s%s%s%s%s%s%s%s\n", linda, + qp->qpn, egrindex, ( useegrbfr ? "" : "(unused)" ), + ( header_offs / LINDA_RECV_HEADER_SIZE ), rcvtype, + pktlen, header_len, padded_payload_len, + ( err ? " [Err" : "" ), ( iberr ? " IB" : "" ), + ( mkerr ? " MK" : "" ), ( tiderr ? " TID" : "" ), + ( khdrerr ? " KHdr" : "" ), ( mtuerr ? " MTU" : "" ), + ( lenerr ? " Len" : "" ), ( parityerr ? " Parity" : ""), + ( vcrcerr ? " VCRC" : "" ), ( icrcerr ? " ICRC" : "" ), + ( err ? "]" : "" ) ); + } else { + DBGC2 ( linda, "Linda %p QPN %ld RX egr %d hdr %d type %d " + "len %d(%d+%d+4)\n", linda, qp->qpn, egrindex, + ( header_offs / LINDA_RECV_HEADER_SIZE ), rcvtype, + pktlen, header_len, padded_payload_len ); + } + DBGCP_HDA ( linda, hdrqoffset, headers.data, + ( header_len + sizeof ( *rcvhdrflags ) ) ); + + /* Parse header to generate address vector */ + qp0 = ( qp->qpn == 0 ); + intended_qp = NULL; + if ( ( rc = ib_pull ( ibdev, &headers, ( qp0 ? &intended_qp : NULL ), + &payload_len, &av ) ) != 0 ) { + DBGC ( linda, "Linda %p could not parse headers: %s\n", + linda, strerror ( rc ) ); + err = 1; + } + if ( ! intended_qp ) + intended_qp = qp; + + /* Complete this buffer and any skipped buffers. Note that + * when the hardware runs out of buffers, it will repeatedly + * report the same buffer (the tail) as a TID error, and that + * it also has a habit of sometimes skipping over several + * buffers at once. + */ + while ( 1 ) { + + /* If we have caught up to the producer counter, stop. + * This will happen when the hardware first runs out + * of buffers and starts reporting TID errors against + * the eager buffer it wants to use next. + */ + if ( linda_wq->eager_cons == linda_wq->eager_prod ) + break; + + /* If we have caught up to where we should be after + * completing this egrindex, stop. We phrase the test + * this way to avoid completing the entire ring when + * we receive the same egrindex twice in a row. + */ + if ( ( linda_wq->eager_cons == + ( ( egrindex + 1 ) & ( linda_wq->eager_entries - 1 ) ))) + break; + + /* Identify work queue entry and corresponding I/O + * buffer. + */ + wqe_idx = ( linda_wq->eager_cons & ( wq->num_wqes - 1 ) ); + iobuf = wq->iobufs[wqe_idx]; + assert ( iobuf != NULL ); + wq->iobufs[wqe_idx] = NULL; + + /* Complete the eager buffer */ + if ( linda_wq->eager_cons == egrindex ) { + /* Completing the eager buffer described in + * this header entry. + */ + iob_put ( iobuf, payload_len ); + rc = ( err ? -EIO : ( useegrbfr ? 0 : -ECANCELED ) ); + /* Redirect to target QP if necessary */ + if ( qp != intended_qp ) { + DBGC ( linda, "Linda %p redirecting QPN %ld " + "=> %ld\n", + linda, qp->qpn, intended_qp->qpn ); + /* Compensate for incorrect fill levels */ + qp->recv.fill--; + intended_qp->recv.fill++; + } + ib_complete_recv ( ibdev, intended_qp, &av, iobuf, rc); + } else { + /* Completing on a skipped-over eager buffer */ + ib_complete_recv ( ibdev, qp, &av, iobuf, -ECANCELED ); + } + + /* Clear eager buffer */ + memset ( &rcvegr, 0, sizeof ( rcvegr ) ); + linda_writeq_array8b ( linda, &rcvegr, linda_wq->eager_array, + linda_wq->eager_cons ); + + /* Increment consumer index */ + linda_wq->eager_cons = ( ( linda_wq->eager_cons + 1 ) & + ( linda_wq->eager_entries - 1 ) ); + } +} + +/** + * Poll receive work queue + * + * @v ibdev Infiniband device + * @v qp Queue pair + */ +static void linda_poll_recv_wq ( struct ib_device *ibdev, + struct ib_queue_pair *qp ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct ib_work_queue *wq = &qp->recv; + struct linda_recv_work_queue *linda_wq = ib_wq_get_drvdata ( wq ); + struct QIB_7220_RcvHdrHead0 rcvhdrhead; + unsigned int ctx = linda_qpn_to_ctx ( qp->qpn ); + unsigned int header_prod; + + /* Check for received packets */ + header_prod = ( BIT_GET ( &linda_wq->header_prod, Value ) << 2 ); + if ( header_prod == linda_wq->header_cons ) + return; + + /* Process all received packets */ + while ( linda_wq->header_cons != header_prod ) { + + /* Complete the receive */ + linda_complete_recv ( ibdev, qp, linda_wq->header_cons ); + + /* Increment the consumer offset */ + linda_wq->header_cons += LINDA_RECV_HEADER_SIZE; + linda_wq->header_cons %= LINDA_RECV_HEADERS_SIZE; + } + + /* Update consumer offset */ + memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) ); + BIT_FILL_2 ( &rcvhdrhead, + RcvHeadPointer, ( linda_wq->header_cons >> 2 ), + counter, 1 ); + linda_writeq_array64k ( linda, &rcvhdrhead, + QIB_7220_RcvHdrHead0_offset, ctx ); +} + +/** + * Poll completion queue + * + * @v ibdev Infiniband device + * @v cq Completion queue + */ +static void linda_poll_cq ( struct ib_device *ibdev, + struct ib_completion_queue *cq ) { + struct ib_work_queue *wq; + + /* Poll associated send and receive queues */ + list_for_each_entry ( wq, &cq->work_queues, list ) { + if ( wq->is_send ) { + linda_poll_send_wq ( ibdev, wq->qp ); + } else { + linda_poll_recv_wq ( ibdev, wq->qp ); + } + } +} + +/*************************************************************************** + * + * Event queues + * + *************************************************************************** + */ + +/** + * Poll event queue + * + * @v ibdev Infiniband device + */ +static void linda_poll_eq ( struct ib_device *ibdev ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct QIB_7220_ErrStatus errstatus; + struct QIB_7220_ErrClear errclear; + + /* Check for link status changes */ + DBG_DISABLE ( DBGLVL_IO ); + linda_readq ( linda, &errstatus, QIB_7220_ErrStatus_offset ); + DBG_ENABLE ( DBGLVL_IO ); + if ( BIT_GET ( &errstatus, IBStatusChanged ) ) { + linda_link_state_changed ( ibdev ); + memset ( &errclear, 0, sizeof ( errclear ) ); + BIT_FILL_1 ( &errclear, IBStatusChangedClear, 1 ); + linda_writeq ( linda, &errclear, QIB_7220_ErrClear_offset ); + } +} + +/*************************************************************************** + * + * Infiniband link-layer operations + * + *************************************************************************** + */ + +/** + * Initialise Infiniband link + * + * @v ibdev Infiniband device + * @ret rc Return status code + */ +static int linda_open ( struct ib_device *ibdev ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct QIB_7220_Control control; + + /* Disable link */ + linda_readq ( linda, &control, QIB_7220_Control_offset ); + BIT_SET ( &control, LinkEn, 1 ); + linda_writeq ( linda, &control, QIB_7220_Control_offset ); + return 0; +} + +/** + * Close Infiniband link + * + * @v ibdev Infiniband device + */ +static void linda_close ( struct ib_device *ibdev ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + struct QIB_7220_Control control; + + /* Disable link */ + linda_readq ( linda, &control, QIB_7220_Control_offset ); + BIT_SET ( &control, LinkEn, 0 ); + linda_writeq ( linda, &control, QIB_7220_Control_offset ); +} + +/*************************************************************************** + * + * Multicast group operations + * + *************************************************************************** + */ + +/** + * Attach to multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + * @ret rc Return status code + */ +static int linda_mcast_attach ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_gid *gid ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + + ( void ) linda; + ( void ) qp; + ( void ) gid; + return 0; +} + +/** + * Detach from multicast group + * + * @v ibdev Infiniband device + * @v qp Queue pair + * @v gid Multicast GID + */ +static void linda_mcast_detach ( struct ib_device *ibdev, + struct ib_queue_pair *qp, + struct ib_gid *gid ) { + struct linda *linda = ib_get_drvdata ( ibdev ); + + ( void ) linda; + ( void ) qp; + ( void ) gid; +} + +/** Linda Infiniband operations */ +static struct ib_device_operations linda_ib_operations = { + .create_cq = linda_create_cq, + .destroy_cq = linda_destroy_cq, + .create_qp = linda_create_qp, + .modify_qp = linda_modify_qp, + .destroy_qp = linda_destroy_qp, + .post_send = linda_post_send, + .post_recv = linda_post_recv, + .poll_cq = linda_poll_cq, + .poll_eq = linda_poll_eq, + .open = linda_open, + .close = linda_close, + .mcast_attach = linda_mcast_attach, + .mcast_detach = linda_mcast_detach, + .set_port_info = linda_set_port_info, + .set_pkey_table = linda_set_pkey_table, +}; + +/*************************************************************************** + * + * I2C bus operations + * + *************************************************************************** + */ + +/** Linda I2C bit to GPIO mappings */ +static unsigned int linda_i2c_bits[] = { + [I2C_BIT_SCL] = ( 1 << LINDA_GPIO_SCL ), + [I2C_BIT_SDA] = ( 1 << LINDA_GPIO_SDA ), +}; + +/** + * Read Linda I2C line status + * + * @v basher Bit-bashing interface + * @v bit_id Bit number + * @ret zero Input is a logic 0 + * @ret non-zero Input is a logic 1 + */ +static int linda_i2c_read_bit ( struct bit_basher *basher, + unsigned int bit_id ) { + struct linda *linda = + container_of ( basher, struct linda, i2c.basher ); + struct QIB_7220_EXTStatus extstatus; + unsigned int status; + + DBG_DISABLE ( DBGLVL_IO ); + + linda_readq ( linda, &extstatus, QIB_7220_EXTStatus_offset ); + status = ( BIT_GET ( &extstatus, GPIOIn ) & linda_i2c_bits[bit_id] ); + + DBG_ENABLE ( DBGLVL_IO ); + + return status; +} + +/** + * Write Linda I2C line status + * + * @v basher Bit-bashing interface + * @v bit_id Bit number + * @v data Value to write + */ +static void linda_i2c_write_bit ( struct bit_basher *basher, + unsigned int bit_id, unsigned long data ) { + struct linda *linda = + container_of ( basher, struct linda, i2c.basher ); + struct QIB_7220_EXTCtrl extctrl; + struct QIB_7220_GPIO gpioout; + unsigned int bit = linda_i2c_bits[bit_id]; + unsigned int outputs = 0; + unsigned int output_enables = 0; + + DBG_DISABLE ( DBGLVL_IO ); + + /* Read current GPIO mask and outputs */ + linda_readq ( linda, &extctrl, QIB_7220_EXTCtrl_offset ); + linda_readq ( linda, &gpioout, QIB_7220_GPIOOut_offset ); + + /* Update outputs and output enables. I2C lines are tied + * high, so we always set the output to 0 and use the output + * enable to control the line. + */ + output_enables = BIT_GET ( &extctrl, GPIOOe ); + output_enables = ( ( output_enables & ~bit ) | ( ~data & bit ) ); + outputs = BIT_GET ( &gpioout, GPIO ); + outputs = ( outputs & ~bit ); + BIT_SET ( &extctrl, GPIOOe, output_enables ); + BIT_SET ( &gpioout, GPIO, outputs ); + + /* Write the output enable first; that way we avoid logic + * hazards. + */ + linda_writeq ( linda, &extctrl, QIB_7220_EXTCtrl_offset ); + linda_writeq ( linda, &gpioout, QIB_7220_GPIOOut_offset ); + mb(); + + DBG_ENABLE ( DBGLVL_IO ); +} + +/** Linda I2C bit-bashing interface operations */ +static struct bit_basher_operations linda_i2c_basher_ops = { + .read = linda_i2c_read_bit, + .write = linda_i2c_write_bit, +}; + +/** + * Initialise Linda I2C subsystem + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_init_i2c ( struct linda *linda ) { + static int try_eeprom_address[] = { 0x51, 0x50 }; + unsigned int i; + int rc; + + /* Initialise bus */ + if ( ( rc = init_i2c_bit_basher ( &linda->i2c, + &linda_i2c_basher_ops ) ) != 0 ) { + DBGC ( linda, "Linda %p could not initialise I2C bus: %s\n", + linda, strerror ( rc ) ); + return rc; + } + + /* Probe for devices */ + for ( i = 0 ; i < ( sizeof ( try_eeprom_address ) / + sizeof ( try_eeprom_address[0] ) ) ; i++ ) { + init_i2c_eeprom ( &linda->eeprom, try_eeprom_address[i] ); + if ( ( rc = i2c_check_presence ( &linda->i2c.i2c, + &linda->eeprom ) ) == 0 ) { + DBGC2 ( linda, "Linda %p found EEPROM at %02x\n", + linda, try_eeprom_address[i] ); + return 0; + } + } + + DBGC ( linda, "Linda %p could not find EEPROM\n", linda ); + return -ENODEV; +} + +/** + * Read EEPROM parameters + * + * @v linda Linda device + * @v guid GUID to fill in + * @ret rc Return status code + */ +static int linda_read_eeprom ( struct linda *linda, + struct ib_gid_half *guid ) { + struct i2c_interface *i2c = &linda->i2c.i2c; + int rc; + + /* Read GUID */ + if ( ( rc = i2c->read ( i2c, &linda->eeprom, LINDA_EEPROM_GUID_OFFSET, + guid->u.bytes, sizeof ( *guid ) ) ) != 0 ) { + DBGC ( linda, "Linda %p could not read GUID: %s\n", + linda, strerror ( rc ) ); + return rc; + } + DBGC2 ( linda, "Linda %p has GUID %02x:%02x:%02x:%02x:%02x:%02x:" + "%02x:%02x\n", linda, guid->u.bytes[0], guid->u.bytes[1], + guid->u.bytes[2], guid->u.bytes[3], guid->u.bytes[4], + guid->u.bytes[5], guid->u.bytes[6], guid->u.bytes[7] ); + + /* Read serial number (debug only) */ + if ( DBG_LOG ) { + uint8_t serial[LINDA_EEPROM_SERIAL_SIZE + 1]; + + serial[ sizeof ( serial ) - 1 ] = '\0'; + if ( ( rc = i2c->read ( i2c, &linda->eeprom, + LINDA_EEPROM_SERIAL_OFFSET, serial, + ( sizeof ( serial ) - 1 ) ) ) != 0 ) { + DBGC ( linda, "Linda %p could not read serial: %s\n", + linda, strerror ( rc ) ); + return rc; + } + DBGC2 ( linda, "Linda %p has serial number \"%s\"\n", + linda, serial ); + } + + return 0; +} + +/*************************************************************************** + * + * External parallel bus access + * + *************************************************************************** + */ + +/** + * Request ownership of the IB external parallel bus + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_ib_epb_request ( struct linda *linda ) { + struct QIB_7220_ibsd_epb_access_ctrl access; + unsigned int i; + + /* Request ownership */ + memset ( &access, 0, sizeof ( access ) ); + BIT_FILL_1 ( &access, sw_ib_epb_req, 1 ); + linda_writeq ( linda, &access, QIB_7220_ibsd_epb_access_ctrl_offset ); + + /* Wait for ownership to be granted */ + for ( i = 0 ; i < LINDA_EPB_REQUEST_MAX_WAIT_US ; i++ ) { + linda_readq ( linda, &access, + QIB_7220_ibsd_epb_access_ctrl_offset ); + if ( BIT_GET ( &access, sw_ib_epb_req_granted ) ) + return 0; + udelay ( 1 ); + } + + DBGC ( linda, "Linda %p timed out waiting for IB EPB request\n", + linda ); + return -ETIMEDOUT; +} + +/** + * Wait for IB external parallel bus transaction to complete + * + * @v linda Linda device + * @v xact Buffer to hold transaction result + * @ret rc Return status code + */ +static int linda_ib_epb_wait ( struct linda *linda, + struct QIB_7220_ibsd_epb_transaction_reg *xact ) { + unsigned int i; + + /* Discard first read to allow for signals crossing clock domains */ + linda_readq ( linda, xact, QIB_7220_ibsd_epb_transaction_reg_offset ); + + for ( i = 0 ; i < LINDA_EPB_XACT_MAX_WAIT_US ; i++ ) { + linda_readq ( linda, xact, + QIB_7220_ibsd_epb_transaction_reg_offset ); + if ( BIT_GET ( xact, ib_epb_rdy ) ) { + if ( BIT_GET ( xact, ib_epb_req_error ) ) { + DBGC ( linda, "Linda %p EPB transaction " + "failed\n", linda ); + return -EIO; + } else { + return 0; + } + } + udelay ( 1 ); + } + + DBGC ( linda, "Linda %p timed out waiting for IB EPB transaction\n", + linda ); + return -ETIMEDOUT; +} + +/** + * Release ownership of the IB external parallel bus + * + * @v linda Linda device + */ +static void linda_ib_epb_release ( struct linda *linda ) { + struct QIB_7220_ibsd_epb_access_ctrl access; + + memset ( &access, 0, sizeof ( access ) ); + BIT_FILL_1 ( &access, sw_ib_epb_req, 0 ); + linda_writeq ( linda, &access, QIB_7220_ibsd_epb_access_ctrl_offset ); +} + +/** + * Read data via IB external parallel bus + * + * @v linda Linda device + * @v location EPB location + * @ret data Data read, or negative error + * + * You must have already acquired ownership of the IB external + * parallel bus. + */ +static int linda_ib_epb_read ( struct linda *linda, unsigned int location ) { + struct QIB_7220_ibsd_epb_transaction_reg xact; + unsigned int data; + int rc; + + /* Ensure no transaction is currently in progress */ + if ( ( rc = linda_ib_epb_wait ( linda, &xact ) ) != 0 ) + return rc; + + /* Process data */ + memset ( &xact, 0, sizeof ( xact ) ); + BIT_FILL_3 ( &xact, + ib_epb_address, LINDA_EPB_LOC_ADDRESS ( location ), + ib_epb_read_write, LINDA_EPB_READ, + ib_epb_cs, LINDA_EPB_LOC_CS ( location ) ); + linda_writeq ( linda, &xact, + QIB_7220_ibsd_epb_transaction_reg_offset ); + + /* Wait for transaction to complete */ + if ( ( rc = linda_ib_epb_wait ( linda, &xact ) ) != 0 ) + return rc; + + data = BIT_GET ( &xact, ib_epb_data ); + return data; +} + +/** + * Write data via IB external parallel bus + * + * @v linda Linda device + * @v location EPB location + * @v data Data to write + * @ret rc Return status code + * + * You must have already acquired ownership of the IB external + * parallel bus. + */ +static int linda_ib_epb_write ( struct linda *linda, unsigned int location, + unsigned int data ) { + struct QIB_7220_ibsd_epb_transaction_reg xact; + int rc; + + /* Ensure no transaction is currently in progress */ + if ( ( rc = linda_ib_epb_wait ( linda, &xact ) ) != 0 ) + return rc; + + /* Process data */ + memset ( &xact, 0, sizeof ( xact ) ); + BIT_FILL_4 ( &xact, + ib_epb_data, data, + ib_epb_address, LINDA_EPB_LOC_ADDRESS ( location ), + ib_epb_read_write, LINDA_EPB_WRITE, + ib_epb_cs, LINDA_EPB_LOC_CS ( location ) ); + linda_writeq ( linda, &xact, + QIB_7220_ibsd_epb_transaction_reg_offset ); + + /* Wait for transaction to complete */ + if ( ( rc = linda_ib_epb_wait ( linda, &xact ) ) != 0 ) + return rc; + + return 0; +} + +/** + * Read/modify/write EPB register + * + * @v linda Linda device + * @v cs Chip select + * @v channel Channel + * @v element Element + * @v reg Register + * @v value Value to set + * @v mask Mask to apply to old value + * @ret rc Return status code + */ +static int linda_ib_epb_mod_reg ( struct linda *linda, unsigned int cs, + unsigned int channel, unsigned int element, + unsigned int reg, unsigned int value, + unsigned int mask ) { + unsigned int location; + int old_value; + int rc; + + DBG_DISABLE ( DBGLVL_IO ); + + /* Sanity check */ + assert ( ( value & mask ) == value ); + + /* Acquire bus ownership */ + if ( ( rc = linda_ib_epb_request ( linda ) ) != 0 ) + goto out; + + /* Read existing value, if necessary */ + location = LINDA_EPB_LOC ( cs, channel, element, reg ); + if ( (~mask) & 0xff ) { + old_value = linda_ib_epb_read ( linda, location ); + if ( old_value < 0 ) { + rc = old_value; + goto out_release; + } + } else { + old_value = 0; + } + + /* Update value */ + value = ( ( old_value & ~mask ) | value ); + DBGCP ( linda, "Linda %p CS %d EPB(%d,%d,%#02x) %#02x => %#02x\n", + linda, cs, channel, element, reg, old_value, value ); + if ( ( rc = linda_ib_epb_write ( linda, location, value ) ) != 0 ) + goto out_release; + + out_release: + /* Release bus */ + linda_ib_epb_release ( linda ); + out: + DBG_ENABLE ( DBGLVL_IO ); + return rc; +} + +/** + * Transfer data to/from microcontroller RAM + * + * @v linda Linda device + * @v address Starting address + * @v write Data to write, or NULL + * @v read Data to read, or NULL + * @v len Length of data + * @ret rc Return status code + */ +static int linda_ib_epb_ram_xfer ( struct linda *linda, unsigned int address, + const void *write, void *read, + size_t len ) { + unsigned int control; + unsigned int address_hi; + unsigned int address_lo; + int data; + int rc; + + DBG_DISABLE ( DBGLVL_IO ); + + assert ( ! ( write && read ) ); + assert ( ( address % LINDA_EPB_UC_CHUNK_SIZE ) == 0 ); + assert ( ( len % LINDA_EPB_UC_CHUNK_SIZE ) == 0 ); + + /* Acquire bus ownership */ + if ( ( rc = linda_ib_epb_request ( linda ) ) != 0 ) + goto out; + + /* Process data */ + while ( len ) { + + /* Reset the address for each new chunk */ + if ( ( address % LINDA_EPB_UC_CHUNK_SIZE ) == 0 ) { + + /* Write the control register */ + control = ( read ? LINDA_EPB_UC_CTL_READ : + LINDA_EPB_UC_CTL_WRITE ); + if ( ( rc = linda_ib_epb_write ( linda, + LINDA_EPB_UC_CTL, + control ) ) != 0 ) + break; + + /* Write the address registers */ + address_hi = ( address >> 8 ); + if ( ( rc = linda_ib_epb_write ( linda, + LINDA_EPB_UC_ADDR_HI, + address_hi ) ) != 0 ) + break; + address_lo = ( address & 0xff ); + if ( ( rc = linda_ib_epb_write ( linda, + LINDA_EPB_UC_ADDR_LO, + address_lo ) ) != 0 ) + break; + } + + /* Read or write the data */ + if ( read ) { + data = linda_ib_epb_read ( linda, LINDA_EPB_UC_DATA ); + if ( data < 0 ) { + rc = data; + break; + } + *( ( uint8_t * ) read++ ) = data; + } else { + data = *( ( uint8_t * ) write++ ); + if ( ( rc = linda_ib_epb_write ( linda, + LINDA_EPB_UC_DATA, + data ) ) != 0 ) + break; + } + address++; + len--; + + /* Reset the control byte after each chunk */ + if ( ( address % LINDA_EPB_UC_CHUNK_SIZE ) == 0 ) { + if ( ( rc = linda_ib_epb_write ( linda, + LINDA_EPB_UC_CTL, + 0 ) ) != 0 ) + break; + } + } + + /* Release bus */ + linda_ib_epb_release ( linda ); + + out: + DBG_ENABLE ( DBGLVL_IO ); + return rc; +} + +/*************************************************************************** + * + * Infiniband SerDes initialisation + * + *************************************************************************** + */ + +/** A Linda SerDes parameter */ +struct linda_serdes_param { + /** EPB address as constructed by LINDA_EPB_ADDRESS() */ + uint16_t address; + /** Value to set */ + uint8_t value; + /** Mask to apply to old value */ + uint8_t mask; +} __packed; + +/** Magic "all channels" channel number */ +#define LINDA_EPB_ALL_CHANNELS 31 + +/** End of SerDes parameter list marker */ +#define LINDA_SERDES_PARAM_END { 0, 0, 0 } + +/** + * Program IB SerDes register(s) + * + * @v linda Linda device + * @v param SerDes parameter + * @ret rc Return status code + */ +static int linda_set_serdes_param ( struct linda *linda, + struct linda_serdes_param *param ) { + unsigned int channel; + unsigned int channel_start; + unsigned int channel_end; + unsigned int element; + unsigned int reg; + int rc; + + /* Break down the EPB address and determine channels */ + channel = LINDA_EPB_ADDRESS_CHANNEL ( param->address ); + element = LINDA_EPB_ADDRESS_ELEMENT ( param->address ); + reg = LINDA_EPB_ADDRESS_REG ( param->address ); + if ( channel == LINDA_EPB_ALL_CHANNELS ) { + channel_start = 0; + channel_end = 3; + } else { + channel_start = channel_end = channel; + } + + /* Modify register for each specified channel */ + for ( channel = channel_start ; channel <= channel_end ; channel++ ) { + if ( ( rc = linda_ib_epb_mod_reg ( linda, LINDA_EPB_CS_SERDES, + channel, element, reg, + param->value, + param->mask ) ) != 0 ) + return rc; + } + + return 0; +} + +/** + * Program IB SerDes registers + * + * @v linda Linda device + * @v param SerDes parameters + * @v count Number of parameters + * @ret rc Return status code + */ +static int linda_set_serdes_params ( struct linda *linda, + struct linda_serdes_param *params ) { + int rc; + + for ( ; params->mask != 0 ; params++ ){ + if ( ( rc = linda_set_serdes_param ( linda, + params ) ) != 0 ) + return rc; + } + + return 0; +} + +#define LINDA_DDS_VAL( amp_d, main_d, ipst_d, ipre_d, \ + amp_s, main_s, ipst_s, ipre_s ) \ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 9, 0x00 ), \ + ( ( ( amp_d & 0x1f ) << 1 ) | 1 ), 0xff }, \ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 9, 0x01 ), \ + ( ( ( amp_s & 0x1f ) << 1 ) | 1 ), 0xff }, \ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 9, 0x09 ), \ + ( ( main_d << 3 ) | 4 | ( ipre_d >> 2 ) ), 0xff }, \ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 9, 0x0a ), \ + ( ( main_s << 3 ) | 4 | ( ipre_s >> 2 ) ), 0xff }, \ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 9, 0x06 ), \ + ( ( ( ipst_d & 0xf ) << 1 ) | \ + ( ( ipre_d & 3 ) << 6 ) | 0x21 ), 0xff }, \ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 9, 0x07 ), \ + ( ( ( ipst_s & 0xf ) << 1 ) | \ + ( ( ipre_s & 3 ) << 6) | 0x21 ), 0xff } + +/** + * Linda SerDes default parameters + * + * These magic start-of-day values are taken from the Linux driver. + */ +static struct linda_serdes_param linda_serdes_defaults1[] = { + /* RXHSCTRL0 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x00 ), 0xd4, 0xff }, + /* VCDL_DAC2 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x05 ), 0x2d, 0xff }, + /* VCDL_CTRL2 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x08 ), 0x03, 0x0f }, + /* START_EQ1 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x27 ), 0x10, 0xff }, + /* START_EQ2 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x28 ), 0x30, 0xff }, + /* BACTRL */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x0e ), 0x40, 0xff }, + /* LDOUTCTRL1 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x06 ), 0x04, 0xff }, + /* RXHSSTATUS */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x0f ), 0x04, 0xff }, + /* End of this block */ + LINDA_SERDES_PARAM_END +}; +static struct linda_serdes_param linda_serdes_defaults2[] = { + /* LDOUTCTRL1 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x06 ), 0x00, 0xff }, + /* DDS values */ + LINDA_DDS_VAL ( 31, 19, 12, 0, 29, 22, 9, 0 ), + /* Set Rcv Eq. to Preset node */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x27 ), 0x10, 0xff }, + /* DFELTHFDR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x08 ), 0x00, 0xff }, + /* DFELTHHDR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x21 ), 0x00, 0xff }, + /* TLTHFDR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x09 ), 0x02, 0xff }, + /* TLTHHDR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x23 ), 0x02, 0xff }, + /* ZFR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x1b ), 0x0c, 0xff }, + /* ZCNT) */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x1c ), 0x0c, 0xff }, + /* GFR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x1e ), 0x10, 0xff }, + /* GHR */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x1f ), 0x10, 0xff }, + /* VCDL_CTRL0 toggle */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x06 ), 0x20, 0xff }, + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 6, 0x06 ), 0x00, 0xff }, + /* CMUCTRL5 */ + { LINDA_EPB_ADDRESS ( 7, 0, 0x15 ), 0x80, 0xff }, + /* End of this block */ + LINDA_SERDES_PARAM_END +}; +static struct linda_serdes_param linda_serdes_defaults3[] = { + /* START_EQ1 */ + { LINDA_EPB_ADDRESS ( LINDA_EPB_ALL_CHANNELS, 7, 0x27 ), 0x00, 0x38 }, + /* End of this block */ + LINDA_SERDES_PARAM_END +}; + +/** + * Program the microcontroller RAM + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_program_uc_ram ( struct linda *linda ) { + int rc; + + if ( ( rc = linda_ib_epb_ram_xfer ( linda, 0, linda_ib_fw, NULL, + sizeof ( linda_ib_fw ) ) ) != 0 ){ + DBGC ( linda, "Linda %p could not load IB firmware: %s\n", + linda, strerror ( rc ) ); + return rc; + } + + return 0; +} + +/** + * Verify the microcontroller RAM + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_verify_uc_ram ( struct linda *linda ) { + uint8_t verify[LINDA_EPB_UC_CHUNK_SIZE]; + unsigned int offset; + int rc; + + for ( offset = 0 ; offset < sizeof ( linda_ib_fw ); + offset += sizeof ( verify ) ) { + if ( ( rc = linda_ib_epb_ram_xfer ( linda, offset, + NULL, verify, + sizeof (verify) )) != 0 ){ + DBGC ( linda, "Linda %p could not read back IB " + "firmware: %s\n", linda, strerror ( rc ) ); + return rc; + } + if ( memcmp ( ( linda_ib_fw + offset ), verify, + sizeof ( verify ) ) != 0 ) { + DBGC ( linda, "Linda %p firmware verification failed " + "at offset %#x\n", linda, offset ); + DBGC_HDA ( linda, offset, ( linda_ib_fw + offset ), + sizeof ( verify ) ); + DBGC_HDA ( linda, offset, verify, sizeof ( verify ) ); + return -EIO; + } + } + + DBGC2 ( linda, "Linda %p firmware verified ok\n", linda ); + return 0; +} + +/** + * Use the microcontroller to trim the IB link + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_trim_ib ( struct linda *linda ) { + struct QIB_7220_IBSerDesCtrl ctrl; + struct QIB_7220_IntStatus intstatus; + unsigned int i; + int rc; + + /* Bring the microcontroller out of reset */ + linda_readq ( linda, &ctrl, QIB_7220_IBSerDesCtrl_offset ); + BIT_SET ( &ctrl, ResetIB_uC_Core, 0 ); + linda_writeq ( linda, &ctrl, QIB_7220_IBSerDesCtrl_offset ); + + /* Wait for the "trim done" signal */ + for ( i = 0 ; i < LINDA_TRIM_DONE_MAX_WAIT_MS ; i++ ) { + linda_readq ( linda, &intstatus, QIB_7220_IntStatus_offset ); + if ( BIT_GET ( &intstatus, IBSerdesTrimDone ) ) { + rc = 0; + goto out_reset; + } + mdelay ( 1 ); + } + + DBGC ( linda, "Linda %p timed out waiting for trim done\n", linda ); + rc = -ETIMEDOUT; + out_reset: + /* Put the microcontroller back into reset */ + BIT_SET ( &ctrl, ResetIB_uC_Core, 1 ); + linda_writeq ( linda, &ctrl, QIB_7220_IBSerDesCtrl_offset ); + + return rc; +} + +/** + * Initialise the IB SerDes + * + * @v linda Linda device + * @ret rc Return status code + */ +static int linda_init_ib_serdes ( struct linda *linda ) { + struct QIB_7220_Control control; + struct QIB_7220_IBCCtrl ibcctrl; + struct QIB_7220_IBCDDRCtrl ibcddrctrl; + struct QIB_7220_XGXSCfg xgxscfg; + int rc; + + /* Disable link */ + linda_readq ( linda, &control, QIB_7220_Control_offset ); + BIT_SET ( &control, LinkEn, 0 ); + linda_writeq ( linda, &control, QIB_7220_Control_offset ); + + /* Configure sensible defaults for IBC */ + memset ( &ibcctrl, 0, sizeof ( ibcctrl ) ); + BIT_FILL_6 ( &ibcctrl, /* Tuning values taken from Linux driver */ + FlowCtrlPeriod, 0x03, + FlowCtrlWaterMark, 0x05, + MaxPktLen, ( ( LINDA_RECV_HEADER_SIZE + + LINDA_RECV_PAYLOAD_SIZE + + 4 /* ICRC */ ) >> 2 ), + PhyerrThreshold, 0xf, + OverrunThreshold, 0xf, + CreditScale, 0x4 ); + linda_writeq ( linda, &ibcctrl, QIB_7220_IBCCtrl_offset ); + + /* Force SDR only to avoid needing all the DDR tuning, + * Mellanox compatibility hacks etc. SDR is plenty for + * boot-time operation. + */ + linda_readq ( linda, &ibcddrctrl, QIB_7220_IBCDDRCtrl_offset ); + BIT_SET ( &ibcddrctrl, IB_ENHANCED_MODE, 0 ); + BIT_SET ( &ibcddrctrl, SD_SPEED_SDR, 1 ); + BIT_SET ( &ibcddrctrl, SD_SPEED_DDR, 0 ); + BIT_SET ( &ibcddrctrl, SD_SPEED_QDR, 0 ); + BIT_SET ( &ibcddrctrl, HRTBT_ENB, 0 ); + BIT_SET ( &ibcddrctrl, HRTBT_AUTO, 0 ); + linda_writeq ( linda, &ibcddrctrl, QIB_7220_IBCDDRCtrl_offset ); + + /* Set default SerDes parameters */ + if ( ( rc = linda_set_serdes_params ( linda, + linda_serdes_defaults1 ) ) != 0 ) + return rc; + udelay ( 415 ); /* Magic delay while SerDes sorts itself out */ + if ( ( rc = linda_set_serdes_params ( linda, + linda_serdes_defaults2 ) ) != 0 ) + return rc; + + /* Program the microcontroller RAM */ + if ( ( rc = linda_program_uc_ram ( linda ) ) != 0 ) + return rc; + + /* Verify the microcontroller RAM contents */ + if ( DBGLVL_LOG ) { + if ( ( rc = linda_verify_uc_ram ( linda ) ) != 0 ) + return rc; + } + + /* More SerDes tuning */ + if ( ( rc = linda_set_serdes_params ( linda, + linda_serdes_defaults3 ) ) != 0 ) + return rc; + + /* Use the microcontroller to trim the IB link */ + if ( ( rc = linda_trim_ib ( linda ) ) != 0 ) + return rc; + + /* Bring XGXS out of reset */ + linda_readq ( linda, &xgxscfg, QIB_7220_XGXSCfg_offset ); + BIT_SET ( &xgxscfg, tx_rx_reset, 0 ); + BIT_SET ( &xgxscfg, xcv_reset, 0 ); + linda_writeq ( linda, &xgxscfg, QIB_7220_XGXSCfg_offset ); + + return rc; +} + +/*************************************************************************** + * + * PCI layer interface + * + *************************************************************************** + */ + +/** + * Probe PCI device + * + * @v pci PCI device + * @v id PCI ID + * @ret rc Return status code + */ +static int linda_probe ( struct pci_device *pci, + const struct pci_device_id *id __unused ) { + struct ib_device *ibdev; + struct linda *linda; + struct QIB_7220_Revision revision; + int rc; + + /* Allocate Infiniband device */ + ibdev = alloc_ibdev ( sizeof ( *linda ) ); + if ( ! ibdev ) { + rc = -ENOMEM; + goto err_alloc_ibdev; + } + pci_set_drvdata ( pci, ibdev ); + linda = ib_get_drvdata ( ibdev ); + ibdev->op = &linda_ib_operations; + ibdev->dev = &pci->dev; + ibdev->port = 1; + + /* Fix up PCI device */ + adjust_pci_device ( pci ); + + /* Get PCI BARs */ + linda->regs = ioremap ( pci->membase, LINDA_BAR0_SIZE ); + DBGC2 ( linda, "Linda %p has BAR at %08lx\n", linda, pci->membase ); + + /* Print some general data */ + linda_readq ( linda, &revision, QIB_7220_Revision_offset ); + DBGC2 ( linda, "Linda %p board %02lx v%ld.%ld.%ld.%ld\n", linda, + BIT_GET ( &revision, BoardID ), + BIT_GET ( &revision, R_SW ), + BIT_GET ( &revision, R_Arch ), + BIT_GET ( &revision, R_ChipRevMajor ), + BIT_GET ( &revision, R_ChipRevMinor ) ); + + /* Record link capabilities. Note that we force SDR only to + * avoid having to carry extra code for DDR tuning etc. + */ + ibdev->link_width_enabled = ibdev->link_width_supported = + ( IB_LINK_WIDTH_4X | IB_LINK_WIDTH_1X ); + ibdev->link_speed_enabled = ibdev->link_speed_supported = + IB_LINK_SPEED_SDR; + + /* Initialise I2C subsystem */ + if ( ( rc = linda_init_i2c ( linda ) ) != 0 ) + goto err_init_i2c; + + /* Read EEPROM parameters */ + if ( ( rc = linda_read_eeprom ( linda, &ibdev->gid.u.half[1] ) ) != 0 ) + goto err_read_eeprom; + + /* Initialise send datapath */ + if ( ( rc = linda_init_send ( linda ) ) != 0 ) + goto err_init_send; + + /* Initialise receive datapath */ + if ( ( rc = linda_init_recv ( linda ) ) != 0 ) + goto err_init_recv; + + /* Initialise the IB SerDes */ + if ( ( rc = linda_init_ib_serdes ( linda ) ) != 0 ) + goto err_init_ib_serdes; + + /* Register Infiniband device */ + if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) { + DBGC ( linda, "Linda %p could not register IB " + "device: %s\n", linda, strerror ( rc ) ); + goto err_register_ibdev; + } + + return 0; + + unregister_ibdev ( ibdev ); + err_register_ibdev: + linda_fini_recv ( linda ); + err_init_recv: + linda_fini_send ( linda ); + err_init_send: + err_init_ib_serdes: + err_read_eeprom: + err_init_i2c: + ibdev_put ( ibdev ); + err_alloc_ibdev: + return rc; +} + +/** + * Remove PCI device + * + * @v pci PCI device + */ +static void linda_remove ( struct pci_device *pci ) { + struct ib_device *ibdev = pci_get_drvdata ( pci ); + struct linda *linda = ib_get_drvdata ( ibdev ); + + unregister_ibdev ( ibdev ); + linda_fini_recv ( linda ); + linda_fini_send ( linda ); + ibdev_put ( ibdev ); +} + +static struct pci_device_id linda_nics[] = { + PCI_ROM ( 0x1077, 0x7220, "iba7220", "QLE7240/7280 HCA driver", 0 ), +}; + +struct pci_driver linda_driver __pci_driver = { + .ids = linda_nics, + .id_count = ( sizeof ( linda_nics ) / sizeof ( linda_nics[0] ) ), + .probe = linda_probe, + .remove = linda_remove, +}; |