diff options
Diffstat (limited to 'src/drivers/net/mlx_ipoib/ib_mt23108.c')
| -rw-r--r-- | src/drivers/net/mlx_ipoib/ib_mt23108.c | 1701 |
1 files changed, 0 insertions, 1701 deletions
diff --git a/src/drivers/net/mlx_ipoib/ib_mt23108.c b/src/drivers/net/mlx_ipoib/ib_mt23108.c deleted file mode 100644 index ca3abb10..00000000 --- a/src/drivers/net/mlx_ipoib/ib_mt23108.c +++ /dev/null @@ -1,1701 +0,0 @@ -/* - This software is available to you under a choice of one of two - licenses. You may choose to be licensed under the terms of the GNU - General Public License (GPL) Version 2, available at - <http://www.fsf.org/copyleft/gpl.html>, or the OpenIB.org BSD - license, available in the LICENSE.TXT file accompanying this - software. These details are also available at - <http://openib.org/license.html>. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - - Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. -*/ - -#include "mt23108.h" -#include "ib_driver.h" -#include <gpxe/pci.h> - -struct device_buffers_st { - union recv_wqe_u mads_qp_rcv_queue[NUM_MADS_RCV_WQES] - __attribute__ ((aligned(RECV_WQE_U_ALIGN))); - union recv_wqe_u ipoib_qp_rcv_queue[NUM_IPOIB_RCV_WQES] - __attribute__ ((aligned(RECV_WQE_U_ALIGN))); - union ud_send_wqe_u mads_qp_snd_queue[NUM_MADS_SND_WQES] - __attribute__ ((aligned(UD_SEND_WQE_U_ALIGN))); - union ud_send_wqe_u ipoib_qp_snd_queue[NUM_IPOIB_SND_WQES] - __attribute__ ((aligned(UD_SEND_WQE_U_ALIGN))); - u8 inprm_buf[INPRM_BUF_SZ] __attribute__ ((aligned(INPRM_BUF_ALIGN))); - u8 outprm_buf[OUTPRM_BUF_SZ] - __attribute__ ((aligned(OUTPRM_BUF_ALIGN))); - struct eqe_t eq_buf[1 << LOG2_EQ_SZ] - __attribute__ ((aligned(sizeof(struct eqe_t)))); - union cqe_st mads_snd_cq_buf[NUM_MADS_SND_CQES] - __attribute__ ((aligned(sizeof(union cqe_st)))); - union cqe_st ipoib_snd_cq_buf[NUM_IPOIB_SND_CQES] - __attribute__ ((aligned(sizeof(union cqe_st)))); - union cqe_st mads_rcv_cq_buf[NUM_MADS_RCV_CQES] - __attribute__ ((aligned(sizeof(union cqe_st)))); - union cqe_st ipoib_rcv_cq_buf[NUM_IPOIB_RCV_CQES] - __attribute__ ((aligned(sizeof(union cqe_st)))); - union ud_av_u av_array[NUM_AVS] - __attribute__ ((aligned(ADDRESS_VECTOR_ST_ALIGN))); -} __attribute__ ((packed)); - -#define STRUCT_ALIGN_SZ 4096 -#define SRC_BUF_SZ (sizeof(struct device_buffers_st) + STRUCT_ALIGN_SZ - 1) - -/* the following must be kept in this order - for the memory region to cover the buffers */ -static u8 src_buf[SRC_BUF_SZ]; -static struct ib_buffers_st ib_buffers; -static __u32 memreg_size; -/* end of order constraint */ - -static struct dev_pci_struct tavor_pci_dev; -static struct device_buffers_st *dev_buffers_p; -static struct device_ib_data_st dev_ib_data; - -static int gw_write_cr(__u32 addr, __u32 data) -{ - writel(htonl(data), tavor_pci_dev.cr_space + addr); - return 0; -} - -static int gw_read_cr(__u32 addr, __u32 * result) -{ - *result = ntohl(readl(tavor_pci_dev.cr_space + addr)); - return 0; -} - -static int reset_hca(void) -{ - return gw_write_cr(TAVOR_RESET_OFFSET, 1); -} - -static int find_mlx_bridge(__u8 hca_bus, __u8 * br_bus_p, __u8 * br_devfn_p) -{ - int bus; - int dev; - int devfn; - int rc; - __u16 vendor, dev_id; - __u8 sec_bus; - - for (bus = 0; bus < 256; ++bus) { - for (dev = 0; dev < 32; ++dev) { - devfn = (dev << 3); - rc = pcibios_read_config_word(bus, devfn, PCI_VENDOR_ID, - &vendor); - if (rc) - return rc; - - if (vendor != MELLANOX_VENDOR_ID) - continue; - - rc = pcibios_read_config_word(bus, devfn, PCI_DEVICE_ID, - &dev_id); - if (rc) - return rc; - - if (dev_id != TAVOR_BRIDGE_DEVICE_ID) - continue; - - rc = pcibios_read_config_byte(bus, devfn, - PCI_SECONDARY_BUS, - &sec_bus); - if (rc) - return rc; - - if (sec_bus == hca_bus) { - *br_bus_p = bus; - *br_devfn_p = devfn; - return 0; - } - } - } - - return -1; -} - -static int ib_device_init(struct pci_device *dev) -{ - int i; - int rc; - __u8 br_bus, br_devfn; - - tprintf(""); - - memset(&dev_ib_data, 0, sizeof dev_ib_data); - - /* save bars */ - tprintf("bus=%d devfn=0x%x", dev->bus, dev->devfn); - for (i = 0; i < 6; ++i) { - tavor_pci_dev.dev.bar[i] = - pci_bar_start(dev, PCI_BASE_ADDRESS_0 + (i << 2)); - tprintf("bar[%d]= 0x%08lx", i, tavor_pci_dev.dev.bar[i]); - } - - tprintf(""); - /* save config space */ - for (i = 0; i < 64; ++i) { - rc = pci_read_config_dword(dev, i << 2, - &tavor_pci_dev.dev. - dev_config_space[i]); - if (rc) { - eprintf(""); - return rc; - } - tprintf("config[%d]= 0x%08lx", i << 2, - tavor_pci_dev.dev.dev_config_space[i]); - } - - tprintf(""); - tavor_pci_dev.dev.dev = dev; - - tprintf(""); - if (dev->dev_id == TAVOR_DEVICE_ID) { - - rc = find_mlx_bridge(dev->bus, &br_bus, &br_devfn); - if (rc) { - eprintf(""); - return rc; - } - - tavor_pci_dev.br.bus = br_bus; - tavor_pci_dev.br.devfn = br_devfn; - - tprintf("bus=%d devfn=0x%x", br_bus, br_devfn); - /* save config space */ - for (i = 0; i < 64; ++i) { - rc = pcibios_read_config_dword(br_bus, br_devfn, i << 2, - &tavor_pci_dev.br. - dev_config_space[i]); - if (rc) { - eprintf(""); - return rc; - } - tprintf("config[%d]= 0x%08lx", i << 2, - tavor_pci_dev.br.dev_config_space[i]); - } - } - - tprintf(""); - - /* map cr-space */ - tavor_pci_dev.cr_space = ioremap(tavor_pci_dev.dev.bar[0], 0x100000); - if (!tavor_pci_dev.cr_space) { - eprintf(""); - return -1; - } - - /* map uar */ - tavor_pci_dev.uar = - ioremap(tavor_pci_dev.dev.bar[2] + UAR_IDX * 0x1000, 0x1000); - if (!tavor_pci_dev.uar) { - eprintf(""); - return -1; - } - tprintf("uar_base (pa:va) = 0x%lx 0x%lx", - tavor_pci_dev.dev.bar[2] + UAR_IDX * 0x1000, tavor_pci_dev.uar); - - tprintf(""); - - return 0; -} - -static inline unsigned long lalign(unsigned long buf, unsigned long align) -{ - return (unsigned long)((buf + align - 1) & - (~(((unsigned long)align) - 1))); -} - -static int init_dev_data(void) -{ - unsigned long tmp; - - tmp = lalign(virt_to_bus(src_buf), STRUCT_ALIGN_SZ); - - dev_buffers_p = bus_to_virt(tmp); - memreg_size = (__u32) (&memreg_size) - (__u32) dev_buffers_p; - tprintf("src_buf=0x%lx, dev_buffers_p=0x%lx, memreg_size=0x%x", src_buf, - dev_buffers_p, memreg_size); - - return 0; -} - -static int restore_config(void) -{ - int i; - int rc; - - if (tavor_pci_dev.dev.dev->dev_id == TAVOR_DEVICE_ID) { - for (i = 0; i < 64; ++i) { - rc = pcibios_write_config_dword(tavor_pci_dev.br.bus, - tavor_pci_dev.br.devfn, - i << 2, - tavor_pci_dev.br. - dev_config_space[i]); - if (rc) { - return rc; - } - } - } - - for (i = 0; i < 64; ++i) { - if (i != 22 && i != 23) { - rc = pci_write_config_dword(tavor_pci_dev.dev.dev, - i << 2, - tavor_pci_dev.dev. - dev_config_space[i]); - if (rc) { - return rc; - } - } - } - return 0; -} - -static void prep_init_hca_buf(const struct init_hca_st *init_hca_p, void *buf) -{ - /*struct init_hca_param_st */ void *p = buf; - void *tmp; - - memset(buf, 0, MT_STRUCT_SIZE(tavorprm_init_hca_st)); - - tmp = - p + MT_BYTE_OFFSET(tavorprm_init_hca_st, - qpc_eec_cqc_eqc_rdb_parameters); - - INS_FLD(init_hca_p->qpc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, - qpc_base_addr_h); - INS_FLD(init_hca_p-> - qpc_base_addr_l >> (32 - - (MT_BIT_SIZE - (tavorprm_qpcbaseaddr_st, - qpc_base_addr_l))), tmp, - tavorprm_qpcbaseaddr_st, qpc_base_addr_l); - INS_FLD(init_hca_p->log_num_of_qp, tmp, tavorprm_qpcbaseaddr_st, - log_num_of_qp); - - INS_FLD(init_hca_p->cqc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, - cqc_base_addr_h); - INS_FLD(init_hca_p-> - cqc_base_addr_l >> (32 - - (MT_BIT_SIZE - (tavorprm_qpcbaseaddr_st, - cqc_base_addr_l))), tmp, - tavorprm_qpcbaseaddr_st, cqc_base_addr_l); - INS_FLD(init_hca_p->log_num_of_cq, tmp, tavorprm_qpcbaseaddr_st, - log_num_of_cq); - - INS_FLD(init_hca_p->eqc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, - eqc_base_addr_h); - INS_FLD(init_hca_p-> - eqc_base_addr_l >> (32 - - (MT_BIT_SIZE - (tavorprm_qpcbaseaddr_st, - eqc_base_addr_l))), tmp, - tavorprm_qpcbaseaddr_st, eqc_base_addr_l); - INS_FLD(LOG2_EQS, tmp, tavorprm_qpcbaseaddr_st, log_num_eq); - - INS_FLD(init_hca_p->srqc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, - srqc_base_addr_h); - INS_FLD(init_hca_p-> - srqc_base_addr_l >> (32 - - (MT_BIT_SIZE - (tavorprm_qpcbaseaddr_st, - srqc_base_addr_l))), tmp, - tavorprm_qpcbaseaddr_st, srqc_base_addr_l); - INS_FLD(init_hca_p->log_num_of_srq, tmp, tavorprm_qpcbaseaddr_st, - log_num_of_srq); - - INS_FLD(init_hca_p->eqpc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, - eqpc_base_addr_h); - INS_FLD(init_hca_p->eqpc_base_addr_l, tmp, tavorprm_qpcbaseaddr_st, - eqpc_base_addr_l); - - INS_FLD(init_hca_p->eeec_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, - eeec_base_addr_h); - INS_FLD(init_hca_p->eeec_base_addr_l, tmp, tavorprm_qpcbaseaddr_st, - eeec_base_addr_l); - - tmp = p + MT_BYTE_OFFSET(tavorprm_init_hca_st, multicast_parameters); - - INS_FLD(init_hca_p->mc_base_addr_h, tmp, tavorprm_multicastparam_st, - mc_base_addr_h); - INS_FLD(init_hca_p->mc_base_addr_l, tmp, tavorprm_multicastparam_st, - mc_base_addr_l); - - INS_FLD(init_hca_p->log_mc_table_entry_sz, tmp, - tavorprm_multicastparam_st, log_mc_table_entry_sz); - INS_FLD(init_hca_p->log_mc_table_sz, tmp, tavorprm_multicastparam_st, - log_mc_table_sz); - INS_FLD(init_hca_p->mc_table_hash_sz, tmp, tavorprm_multicastparam_st, - mc_table_hash_sz); - - tmp = p + MT_BYTE_OFFSET(tavorprm_init_hca_st, tpt_parameters); - - INS_FLD(init_hca_p->mpt_base_addr_h, tmp, tavorprm_tptparams_st, - mpt_base_adr_h); - INS_FLD(init_hca_p->mpt_base_addr_l, tmp, tavorprm_tptparams_st, - mpt_base_adr_l); - INS_FLD(init_hca_p->log_mpt_sz, tmp, tavorprm_tptparams_st, log_mpt_sz); - - INS_FLD(init_hca_p->mtt_base_addr_h, tmp, tavorprm_tptparams_st, - mtt_base_addr_h); - INS_FLD(init_hca_p->mtt_base_addr_l, tmp, tavorprm_tptparams_st, - mtt_base_addr_l); - - tmp = p + MT_BYTE_OFFSET(tavorprm_init_hca_st, uar_parameters); - INS_FLD(tavor_pci_dev.dev.bar[3], tmp, tavorprm_uar_params_st, - uar_base_addr_h); - INS_FLD(tavor_pci_dev.dev.bar[2] & 0xfff00000, tmp, - tavorprm_uar_params_st, uar_base_addr_l); - -} - -static void prep_sw2hw_mpt_buf(void *buf, __u32 mkey) -{ - INS_FLD(1, buf, tavorprm_mpt_st, m_io); - INS_FLD(1, buf, tavorprm_mpt_st, lw); - INS_FLD(1, buf, tavorprm_mpt_st, lr); - INS_FLD(1, buf, tavorprm_mpt_st, pa); - INS_FLD(1, buf, tavorprm_mpt_st, r_w); - - INS_FLD(mkey, buf, tavorprm_mpt_st, mem_key); - INS_FLD(GLOBAL_PD, buf, tavorprm_mpt_st, pd); - - INS_FLD(virt_to_bus(dev_buffers_p), buf, tavorprm_mpt_st, - start_address_l); - INS_FLD(memreg_size, buf, tavorprm_mpt_st, reg_wnd_len_l); -} - -static void prep_sw2hw_eq_buf(void *buf, struct eqe_t *eq) -{ - memset(buf, 0, MT_STRUCT_SIZE(tavorprm_eqc_st)); - - INS_FLD(2, buf, tavorprm_eqc_st, st); /* fired */ - INS_FLD(virt_to_bus(eq), buf, tavorprm_eqc_st, start_address_l); - INS_FLD(LOG2_EQ_SZ, buf, tavorprm_eqc_st, log_eq_size); - INS_FLD(UAR_IDX, buf, tavorprm_eqc_st, usr_page); - INS_FLD(GLOBAL_PD, buf, tavorprm_eqc_st, pd); - INS_FLD(dev_ib_data.mkey, buf, tavorprm_eqc_st, lkey); -} - -static void init_eq_buf(void *eq_buf) -{ - int num_eqes = 1 << LOG2_EQ_SZ; - - memset(eq_buf, 0xff, num_eqes * sizeof(struct eqe_t)); -} - -static void prep_init_ib_buf(void *buf) -{ - __u32 *ptr = (__u32 *) buf; - - ptr[0] = 0x4310; - ptr[1] = 1; - ptr[2] = 64; -} - -static void prep_sw2hw_cq_buf(void *buf, __u8 eqn, __u32 cqn, - union cqe_st *cq_buf) -{ - __u32 *ptr = (__u32 *) buf; - - ptr[2] = virt_to_bus(cq_buf); - ptr[3] = (LOG2_CQ_SZ << 24) | UAR_IDX; - ptr[4] = eqn; - ptr[5] = eqn; - ptr[6] = dev_ib_data.pd; - ptr[7] = dev_ib_data.mkey; - ptr[12] = cqn; -} - -static void prep_rst2init_qpee_buf(void *buf, __u32 snd_cqn, __u32 rcv_cqn, - __u32 qkey) -{ - struct qp_ee_state_tarnisition_st *prm; - void *tmp; - - prm = (struct qp_ee_state_tarnisition_st *)buf; - - INS_FLD(3, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, st); /* service type = UD */ - INS_FLD(3, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, pm_state); /* required for UD QP */ - INS_FLD(UAR_IDX, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, - usr_page); - INS_FLD(dev_ib_data.pd, &prm->ctx, - tavorprm_queue_pair_ee_context_entry_st, pd); - INS_FLD(dev_ib_data.mkey, &prm->ctx, - tavorprm_queue_pair_ee_context_entry_st, wqe_lkey); - INS_FLD(1, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, ssc); /* generate send CQE */ - INS_FLD(1, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, rsc); /* generate receive CQE */ - INS_FLD(snd_cqn, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, - cqn_snd); - INS_FLD(rcv_cqn, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, - cqn_rcv); - INS_FLD(qkey, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, - q_key); - - tmp = - (void *)(&prm->ctx) + - MT_BYTE_OFFSET(tavorprm_queue_pair_ee_context_entry_st, - primary_address_path); - INS_FLD(dev_ib_data.port, tmp, tavorprm_address_path_st, port_number); - - INS_FLD(4, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, mtu); - INS_FLD(0xb, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, - msg_max); -} - -static void prep_init2rtr_qpee_buf(void *buf) -{ - struct qp_ee_state_tarnisition_st *prm; - - prm = (struct qp_ee_state_tarnisition_st *)buf; - - INS_FLD(4, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, mtu); - INS_FLD(0xb, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, - msg_max); -} - -static void init_av_array() -{ - int i; - - dev_ib_data.udav.av_array = dev_buffers_p->av_array; - dev_ib_data.udav.udav_next_free = FL_EOL; - for (i = 0; i < NUM_AVS; ++i) { - dev_ib_data.udav.av_array[i].ud_av.next_free = - dev_ib_data.udav.udav_next_free; - dev_ib_data.udav.udav_next_free = i; - } - tprintf("dev_ib_data.udav.udav_next_free=%d", i); -} - -static int setup_hca(__u8 port, void **eq_p) -{ - int rc; - __u32 key, in_key; - __u32 *inprm; - struct eqe_t *eq_buf; - __u32 event_mask; - void *cfg; - int ret = 0; - __u8 eqn; - struct dev_lim_st dev_lim; - struct init_hca_st init_hca; - __u32 offset, base_h, base_l; - const __u32 delta = 0x400000; - struct query_fw_st qfw; - - tprintf("called"); - - init_dev_data(); - - rc = reset_hca(); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } else { - tprintf("reset_hca() success"); - } - - mdelay(1000); /* wait for 1 sec */ - - rc = restore_config(); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } else { - tprintf("restore_config() success"); - } - - dev_ib_data.pd = GLOBAL_PD; - dev_ib_data.port = port; - - /* execute system enable command */ - rc = cmd_sys_en(); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } else { - tprintf("cmd_sys_en() success"); - } - - rc= cmd_query_fw(&qfw); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } else { - tprintf("cmd_query_fw() success"); - - if (print_info) { - printf("FW ver = %d.%d.%d\n", - qfw.fw_rev_major, - qfw.fw_rev_minor, - qfw.fw_rev_subminor); - } - tprintf("fw_rev_major=%d", qfw.fw_rev_major); - tprintf("fw_rev_minor=%d", qfw.fw_rev_minor); - tprintf("fw_rev_subminor=%d", qfw.fw_rev_subminor); - tprintf("error_buf_start_h=0x%x", qfw.error_buf_start_h); - tprintf("error_buf_start_l=0x%x", qfw.error_buf_start_l); - tprintf("error_buf_size=%d", qfw.error_buf_size); - } - - if (qfw.error_buf_start_h) { - eprintf("too high physical address"); - ret = -1; - goto exit; - } - - dev_ib_data.error_buf_addr= ioremap(qfw.error_buf_start_l, - qfw.error_buf_size*4); - dev_ib_data.error_buf_size= qfw.error_buf_size; - if (!dev_ib_data.error_buf_addr) { - eprintf(""); - ret = -1; - goto exit; - } - - - rc = cmd_query_dev_lim(&dev_lim); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } else { - tprintf("cmd_query_dev_lim() success"); - tprintf("log2_rsvd_qps=%x", dev_lim.log2_rsvd_qps); - tprintf("qpc_entry_sz=%x", dev_lim.qpc_entry_sz); - tprintf("log2_rsvd_srqs=%x", dev_lim.log2_rsvd_srqs); - tprintf("srq_entry_sz=%x", dev_lim.srq_entry_sz); - tprintf("log2_rsvd_ees=%x", dev_lim.log2_rsvd_ees); - tprintf("eec_entry_sz=%x", dev_lim.eec_entry_sz); - tprintf("log2_rsvd_cqs=%x", dev_lim.log2_rsvd_cqs); - tprintf("cqc_entry_sz=%x", dev_lim.cqc_entry_sz); - tprintf("log2_rsvd_mtts=%x", dev_lim.log2_rsvd_mtts); - tprintf("mtt_entry_sz=%x", dev_lim.mtt_entry_sz); - tprintf("log2_rsvd_mrws=%x", dev_lim.log2_rsvd_mrws); - tprintf("mpt_entry_sz=%x", dev_lim.mpt_entry_sz); - tprintf("eqc_entry_sz=%x", dev_lim.eqc_entry_sz); - } - - /* set the qp and cq numbers according - to the results of query_dev_lim */ - dev_ib_data.mads_qp.qpn = (1 << dev_lim.log2_rsvd_qps) + - +QPN_BASE + MADS_QPN_SN; - dev_ib_data.ipoib_qp.qpn = (1 << dev_lim.log2_rsvd_qps) + - +QPN_BASE + IPOIB_QPN_SN; - - dev_ib_data.mads_qp.snd_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + - MADS_SND_CQN_SN; - dev_ib_data.mads_qp.rcv_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + - MADS_RCV_CQN_SN; - - dev_ib_data.ipoib_qp.snd_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + - IPOIB_SND_CQN_SN; - dev_ib_data.ipoib_qp.rcv_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + - IPOIB_RCV_CQN_SN; - - /* disable SRQ */ - cfg = (void *)dev_buffers_p->inprm_buf; - memset(cfg, 0, MT_STRUCT_SIZE(tavorprm_mod_stat_cfg_st)); - INS_FLD(1, cfg, tavorprm_mod_stat_cfg_st, srq_m); //cfg->srq_m = 1; - rc = cmd_mod_stat_cfg(cfg); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } else { - tprintf("cmd_mod_stat_cfg() success"); - } - - /* prepare the init_hca params to pass - to prep_init_hca_buf */ - memset(&init_hca, 0, sizeof init_hca); - offset = 0; - base_h = tavor_pci_dev.dev.bar[5] & 0xfffffff0; - base_l = tavor_pci_dev.dev.bar[4] & 0xfffffff0; - - tprintf("base_h=0x%lx, base_l=0x%lx", base_h, base_l); - - init_hca.qpc_base_addr_h = base_h; - init_hca.qpc_base_addr_l = base_l + offset; - init_hca.log_num_of_qp = dev_lim.log2_rsvd_qps + 1; - offset += delta; - - init_hca.eec_base_addr_h = base_h; - init_hca.eec_base_addr_l = base_l + offset; - init_hca.log_num_of_ee = dev_lim.log2_rsvd_ees; - offset += delta; - - init_hca.srqc_base_addr_h = base_h; - init_hca.srqc_base_addr_l = base_l + offset; - init_hca.log_num_of_srq = dev_lim.log2_rsvd_srqs; - offset += delta; - - init_hca.cqc_base_addr_h = base_h; - init_hca.cqc_base_addr_l = base_l + offset; - init_hca.log_num_of_cq = dev_lim.log2_rsvd_cqs + 1; - offset += delta; - - init_hca.eqpc_base_addr_h = base_h; - init_hca.eqpc_base_addr_l = base_l + offset; - offset += delta; - - init_hca.eeec_base_addr_h = base_h; - init_hca.eeec_base_addr_l = base_l + offset; - offset += delta; - - init_hca.eqc_base_addr_h = base_h; - init_hca.eqc_base_addr_l = base_l + offset; - init_hca.log_num_of_eq = LOG2_EQS; - offset += delta; - - init_hca.rdb_base_addr_h = base_h; - init_hca.rdb_base_addr_l = base_l + offset; - offset += delta; - - init_hca.mc_base_addr_h = base_h; - init_hca.mc_base_addr_l = base_l + offset; - init_hca.log_mc_table_entry_sz = LOG2_MC_ENTRY; - init_hca.mc_table_hash_sz = 0; - init_hca.log_mc_table_sz = LOG2_MC_GROUPS; - offset += delta; - - init_hca.mpt_base_addr_h = base_h; - init_hca.mpt_base_addr_l = base_l + offset; - init_hca.log_mpt_sz = dev_lim.log2_rsvd_mrws + 1; - offset += delta; - - init_hca.mtt_base_addr_h = base_h; - init_hca.mtt_base_addr_l = base_l + offset; - - /* this buffer is used for all the commands */ - inprm = (void *)dev_buffers_p->inprm_buf; - /* excute init_hca command */ - prep_init_hca_buf(&init_hca, inprm); - - rc = cmd_init_hca(inprm, MT_STRUCT_SIZE(tavorprm_init_hca_st)); - if (rc) { - ret = -1; - eprintf(""); - goto undo_sys_en; - } else - tprintf("cmd_init_hca() success"); - - /* register a single memory region which covers - 4 GB of the address space which will be used - throughout the driver */ - memset(inprm, 0, SW2HW_MPT_IBUF_SZ); - in_key = MKEY_PREFIX + (1 << dev_lim.log2_rsvd_mrws); - prep_sw2hw_mpt_buf(inprm, in_key); - rc = cmd_sw2hw_mpt(&key, in_key, inprm, SW2HW_MPT_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_init_hca; - } else { - tprintf("cmd_sw2hw_mpt() success, key=0x%lx", key); - } - dev_ib_data.mkey = key; - - eqn = EQN; - /* allocate a single EQ which will receive - all the events */ - eq_buf = dev_buffers_p->eq_buf; - init_eq_buf(eq_buf); /* put in HW ownership */ - prep_sw2hw_eq_buf(inprm, eq_buf); - rc = cmd_sw2hw_eq(SW2HW_EQ_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_sw2hw_mpt; - } else - tprintf("cmd_sw2hw_eq() success"); - - event_mask = (1 << XDEV_EV_TYPE_CQ_COMP) | - (1 << XDEV_EV_TYPE_CQ_ERR) | - (1 << XDEV_EV_TYPE_LOCAL_WQ_CATAS_ERR) | - (1 << XDEV_EV_TYPE_PORT_ERR) | - (1 << XDEV_EV_TYPE_LOCAL_WQ_INVALID_REQ_ERR) | - (1 << XDEV_EV_TYPE_LOCAL_WQ_ACCESS_VIOL_ERR) | - (1 << TAVOR_IF_EV_TYPE_OVERRUN); - rc = cmd_map_eq(eqn, event_mask, 1); - if (rc) { - ret = -1; - eprintf(""); - goto undo_sw2hw_eq; - } else - tprintf("cmd_map_eq() success"); - - dev_ib_data.eq.eqn = eqn; - dev_ib_data.eq.eq_buf = eq_buf; - dev_ib_data.eq.cons_idx = 0; - dev_ib_data.eq.eq_size = 1 << LOG2_EQ_SZ; - *eq_p = &dev_ib_data.eq; - - memset(inprm, 0, INIT_IB_IBUF_SZ); - prep_init_ib_buf(inprm); - rc = cmd_init_ib(port, inprm, INIT_IB_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_sw2hw_eq; - } else - tprintf("cmd_init_ib() success"); - - init_av_array(); - tprintf("init_av_array() done"); - - goto exit; - - undo_sw2hw_eq: - rc = cmd_hw2sw_eq(EQN); - if (rc) { - eprintf(""); - } else - tprintf("cmd_hw2sw_eq() success"); - - undo_sw2hw_mpt: - rc = cmd_hw2sw_mpt(key); - if (rc) - eprintf(""); - else - tprintf("cmd_hw2sw_mpt() success key=0x%lx", key); - - undo_init_hca: - rc = cmd_close_hca(0); - if (rc) { - eprintf(""); - goto undo_sys_en; - } else - tprintf("cmd_close_hca() success"); - - undo_sys_en: - rc = cmd_sys_dis(); - if (rc) { - eprintf(""); - goto undo_sys_en; - } else - tprintf("cmd_sys_dis() success"); - goto exit; - - exit: - return ret; -} - -static void *get_inprm_buf(void) -{ - return dev_buffers_p->inprm_buf; -} - -static void *get_outprm_buf(void) -{ - return dev_buffers_p->outprm_buf; -} - -static void *get_send_wqe_buf(void *wqe, __u8 index) -{ - struct ud_send_wqe_st *snd_wqe = wqe; - - return bus_to_virt(snd_wqe->mpointer[index].local_addr_l); -} - -static void *get_rcv_wqe_buf(void *wqe, __u8 index) -{ - struct recv_wqe_st *rcv_wqe = wqe; - - return bus_to_virt(be32_to_cpu(rcv_wqe->mpointer[index].local_addr_l)); -} - -static void modify_av_params(struct ud_av_st *av, - __u16 dlid, - __u8 g, - __u8 sl, __u8 rate, union ib_gid_u *gid, __u32 qpn) -{ - memset(&av->av, 0, sizeof av->av); - INS_FLD(dev_ib_data.port, &av->av, tavorprm_ud_address_vector_st, - port_number); - INS_FLD(dev_ib_data.pd, &av->av, tavorprm_ud_address_vector_st, pd); - INS_FLD(dlid, &av->av, tavorprm_ud_address_vector_st, rlid); - INS_FLD(g, &av->av, tavorprm_ud_address_vector_st, g); - INS_FLD(sl, &av->av, tavorprm_ud_address_vector_st, sl); - INS_FLD(3, &av->av, tavorprm_ud_address_vector_st, msg); - - if (rate >= 3) - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, max_stat_rate); /* 4x */ - else - INS_FLD(1, &av->av, tavorprm_ud_address_vector_st, max_stat_rate); /* 1x */ - - cpu_to_be_buf(&av->av, sizeof(av->av)); - if (g) { - if (gid) { - INS_FLD(*((__u32 *) (&gid->raw[0])), &av->av, - tavorprm_ud_address_vector_st, rgid_127_96); - INS_FLD(*((__u32 *) (&gid->raw[4])), &av->av, - tavorprm_ud_address_vector_st, rgid_95_64); - INS_FLD(*((__u32 *) (&gid->raw[8])), &av->av, - tavorprm_ud_address_vector_st, rgid_63_32); - INS_FLD(*((__u32 *) (&gid->raw[12])), &av->av, - tavorprm_ud_address_vector_st, rgid_31_0); - } else { - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, - rgid_127_96); - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, - rgid_95_64); - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, - rgid_63_32); - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, - rgid_31_0); - } - } else { - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, rgid_127_96); - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, rgid_95_64); - INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, rgid_63_32); - INS_FLD(2, &av->av, tavorprm_ud_address_vector_st, rgid_31_0); - } - av->dest_qp = qpn; -} - -static void init_cq_buf(union cqe_st *cq_buf, __u8 num_cqes) -{ - memset(cq_buf, 0xff, num_cqes * sizeof cq_buf[0]); -} - -static int post_rcv_buf(struct udqp_st *qp, struct recv_wqe_st *rcv_wqe) -{ - struct recv_doorbell_st dbell; - int rc; - __u32 tmp[2]; - struct recv_wqe_st *tmp_wqe = (struct recv_wqe_st *)tmp; - __u32 *ptr_dst; - - memset(&dbell, 0, sizeof dbell); - INS_FLD(sizeof(*rcv_wqe) >> 4, &dbell, tavorprm_receive_doorbell_st, - nds); - INS_FLD(virt_to_bus(rcv_wqe) >> 6, &dbell, tavorprm_receive_doorbell_st, - nda); - INS_FLD(qp->qpn, &dbell, tavorprm_receive_doorbell_st, qpn); - INS_FLD(1, &dbell, tavorprm_receive_doorbell_st, credits); - - if (qp->last_posted_rcv_wqe) { - memcpy(tmp, qp->last_posted_rcv_wqe, sizeof(tmp)); - be_to_cpu_buf(tmp, sizeof(tmp)); - INS_FLD(1, tmp_wqe->next, wqe_segment_next_st, dbd); - INS_FLD(sizeof(*rcv_wqe) >> 4, tmp_wqe->next, - wqe_segment_next_st, nds); - INS_FLD(virt_to_bus(rcv_wqe) >> 6, tmp_wqe->next, - wqe_segment_next_st, nda_31_6); - /* this is not really opcode but since the struct - is used for both send and receive, in receive this bit must be 1 - which coinsides with nopcode */ - INS_FLD(1, tmp_wqe->next, wqe_segment_next_st, nopcode); - - cpu_to_be_buf(tmp, sizeof(tmp)); - - ptr_dst = (__u32 *) (qp->last_posted_rcv_wqe); - ptr_dst[0] = tmp[0]; - ptr_dst[1] = tmp[1]; - } - rc = cmd_post_doorbell(&dbell, POST_RCV_OFFSET); - if (!rc) { - qp->last_posted_rcv_wqe = rcv_wqe; - } - - return rc; -} - -static int post_send_req(void *qph, void *wqeh, __u8 num_gather) -{ - struct send_doorbell_st dbell; - int rc; - struct udqp_st *qp = qph; - struct ud_send_wqe_st *snd_wqe = wqeh; - struct next_control_seg_st tmp; - __u32 *psrc, *pdst; - __u32 nds; - - tprintf("snd_wqe=0x%lx, virt_to_bus(snd_wqe)=0x%lx", snd_wqe, - virt_to_bus(snd_wqe)); - - memset(&dbell, 0, sizeof dbell); - INS_FLD(XDEV_NOPCODE_SEND, &dbell, tavorprm_send_doorbell_st, nopcode); - INS_FLD(1, &dbell, tavorprm_send_doorbell_st, f); - INS_FLD(virt_to_bus(snd_wqe) >> 6, &dbell, tavorprm_send_doorbell_st, - nda); - nds = - (sizeof(snd_wqe->next) + sizeof(snd_wqe->udseg) + - sizeof(snd_wqe->mpointer[0]) * num_gather) >> 4; - INS_FLD(nds, &dbell, tavorprm_send_doorbell_st, nds); - INS_FLD(qp->qpn, &dbell, tavorprm_send_doorbell_st, qpn); - - tprintf("0= %lx", ((__u32 *) ((void *)(&dbell)))[0]); - tprintf("1= %lx", ((__u32 *) ((void *)(&dbell)))[1]); - - if (qp->last_posted_snd_wqe) { - memcpy(&tmp, &qp->last_posted_snd_wqe->next, sizeof tmp); - be_to_cpu_buf(&tmp, sizeof tmp); - INS_FLD(1, &tmp, wqe_segment_next_st, dbd); - INS_FLD(virt_to_bus(snd_wqe) >> 6, &tmp, wqe_segment_next_st, - nda_31_6); - INS_FLD(nds, &tmp, wqe_segment_next_st, nds); - - psrc = (__u32 *) (&tmp); - pdst = (__u32 *) (&qp->last_posted_snd_wqe->next); - pdst[0] = htonl(psrc[0]); - pdst[1] = htonl(psrc[1]); - } - - rc = cmd_post_doorbell(&dbell, POST_SND_OFFSET); - if (!rc) { - qp->last_posted_snd_wqe = snd_wqe; - } - - return rc; -} - -static int create_mads_qp(void **qp_pp, void **snd_cq_pp, void **rcv_cq_pp) -{ - __u8 i; - int rc; - struct udqp_st *qp; - - qp = &dev_ib_data.mads_qp; - - /* set the pointer to the receive WQEs buffer */ - qp->rcv_wq = dev_buffers_p->mads_qp_rcv_queue; - - qp->send_buf_sz = MAD_BUF_SZ; - qp->rcv_buf_sz = MAD_BUF_SZ; - - qp->recv_wqe_alloc_idx = 0; - qp->max_recv_wqes = NUM_MADS_RCV_WQES; - qp->recv_wqe_cur_free = NUM_MADS_RCV_WQES; - - /* iterrate through the list */ - for (i = 0; i < NUM_MADS_RCV_WQES; ++i) { - /* clear the WQE */ - memset(&qp->rcv_wq[i], 0, sizeof(qp->rcv_wq[i])); - - qp->rcv_wq[i].wqe_cont.qp = qp; - qp->rcv_bufs[i] = ib_buffers.rcv_mad_buf[i]; - } - - /* set the pointer to the send WQEs buffer */ - qp->snd_wq = dev_buffers_p->mads_qp_snd_queue; - - qp->snd_wqe_alloc_idx = 0; - qp->max_snd_wqes = NUM_MADS_SND_WQES; - qp->snd_wqe_cur_free = NUM_MADS_SND_WQES; - - /* iterrate through the list */ - for (i = 0; i < NUM_MADS_SND_WQES; ++i) { - /* clear the WQE */ - memset(&qp->snd_wq[i], 0, sizeof(qp->snd_wq[i])); - - /* link the WQE to the free list */ - qp->snd_wq[i].wqe_cont.qp = qp; - qp->snd_bufs[i] = ib_buffers.send_mad_buf[i]; - } - - /* qp number and cq numbers are already set up */ - qp->snd_cq.cq_buf = dev_buffers_p->mads_snd_cq_buf; - qp->rcv_cq.cq_buf = dev_buffers_p->mads_rcv_cq_buf; - qp->snd_cq.num_cqes = NUM_MADS_SND_CQES; - qp->rcv_cq.num_cqes = NUM_MADS_RCV_CQES; - qp->qkey = GLOBAL_QKEY; - rc = create_udqp(qp); - if (!rc) { - *qp_pp = qp; - *snd_cq_pp = &qp->snd_cq; - *rcv_cq_pp = &qp->rcv_cq; - } - - return rc; -} - -static int create_ipoib_qp(void **qp_pp, - void **snd_cq_pp, void **rcv_cq_pp, __u32 qkey) -{ - __u8 i; - int rc; - struct udqp_st *qp; - qp = &dev_ib_data.ipoib_qp; - - /* set the pointer to the receive WQEs buffer */ - qp->rcv_wq = dev_buffers_p->ipoib_qp_rcv_queue; - - qp->rcv_buf_sz = IPOIB_RCV_BUF_SZ; - - qp->recv_wqe_alloc_idx = 0; - qp->max_recv_wqes = NUM_IPOIB_RCV_WQES; - qp->recv_wqe_cur_free = NUM_IPOIB_RCV_WQES; - - /* iterrate through the list */ - for (i = 0; i < NUM_IPOIB_RCV_WQES; ++i) { - /* clear the WQE */ - memset(&qp->rcv_wq[i], 0, sizeof(qp->rcv_wq[i])); - - /* update data */ - qp->rcv_wq[i].wqe_cont.qp = qp; - qp->rcv_bufs[i] = ib_buffers.ipoib_rcv_buf[i]; - tprintf("rcv_buf=%lx", qp->rcv_bufs[i]); - } - - /* init send queue WQEs list */ - /* set the list empty */ - qp->snd_wqe_alloc_idx = 0; - qp->max_snd_wqes = NUM_IPOIB_SND_WQES; - qp->snd_wqe_cur_free = NUM_IPOIB_SND_WQES; - - /* set the pointer to the send WQEs buffer */ - qp->snd_wq = dev_buffers_p->ipoib_qp_snd_queue; - - /* iterrate through the list */ - for (i = 0; i < NUM_IPOIB_SND_WQES; ++i) { - /* clear the WQE */ - memset(&qp->snd_wq[i], 0, sizeof(qp->snd_wq[i])); - - /* update data */ - qp->snd_wq[i].wqe_cont.qp = qp; - qp->snd_bufs[i] = ib_buffers.send_ipoib_buf[i]; - qp->send_buf_sz = 4; - } - - /* qp number and cq numbers are already set up */ - - qp->snd_cq.cq_buf = dev_buffers_p->ipoib_snd_cq_buf; - qp->rcv_cq.cq_buf = dev_buffers_p->ipoib_rcv_cq_buf; - qp->snd_cq.num_cqes = NUM_IPOIB_SND_CQES; - qp->rcv_cq.num_cqes = NUM_IPOIB_RCV_CQES; - qp->qkey = qkey; - rc = create_udqp(qp); - if (!rc) { - *qp_pp = qp; - *snd_cq_pp = &qp->snd_cq; - *rcv_cq_pp = &qp->rcv_cq; - } - - return rc; -} - -static int create_udqp(struct udqp_st *qp) -{ - int rc, ret = 0; - void *inprm; - struct recv_wqe_st *rcv_wqe; - - inprm = dev_buffers_p->inprm_buf; - - /* create send CQ */ - init_cq_buf(qp->snd_cq.cq_buf, qp->snd_cq.num_cqes); - qp->snd_cq.cons_idx = 0; - memset(inprm, 0, SW2HW_CQ_IBUF_SZ); - prep_sw2hw_cq_buf(inprm, dev_ib_data.eq.eqn, qp->snd_cq.cqn, - qp->snd_cq.cq_buf); - rc = cmd_sw2hw_cq(qp->snd_cq.cqn, inprm, SW2HW_CQ_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto exit; - } - - /* create receive CQ */ - init_cq_buf(qp->rcv_cq.cq_buf, qp->rcv_cq.num_cqes); - qp->rcv_cq.cons_idx = 0; - memset(inprm, 0, SW2HW_CQ_IBUF_SZ); - prep_sw2hw_cq_buf(inprm, dev_ib_data.eq.eqn, qp->rcv_cq.cqn, - qp->rcv_cq.cq_buf); - rc = cmd_sw2hw_cq(qp->rcv_cq.cqn, inprm, SW2HW_CQ_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_snd_cq; - } - - memset(inprm, 0, QPCTX_IBUF_SZ); - prep_rst2init_qpee_buf(inprm, qp->snd_cq.cqn, qp->rcv_cq.cqn, qp->qkey); - rc = cmd_rst2init_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_rcv_cq; - } - - qp->last_posted_rcv_wqe = NULL; - qp->last_posted_snd_wqe = NULL; - - /* post all the buffers to the receive queue */ - while (1) { - /* allocate wqe */ - rcv_wqe = alloc_rcv_wqe(qp); - if (!rcv_wqe) - break; - - /* post the buffer */ - rc = post_rcv_buf(qp, rcv_wqe); - if (rc) { - ret = -1; - eprintf(""); - goto undo_rcv_cq; - } - } - - memset(inprm, 0, QPCTX_IBUF_SZ); - prep_init2rtr_qpee_buf(inprm); - rc = cmd_init2rtr_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_rcv_cq; - } - - memset(inprm, 0, QPCTX_IBUF_SZ); - rc = cmd_rtr2rts_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ); - if (rc) { - ret = -1; - eprintf(""); - goto undo_rcv_cq; - } - - goto exit; - - undo_rcv_cq: - rc = cmd_hw2sw_cq(qp->rcv_cq.cqn); - if (rc) - eprintf(""); - - undo_snd_cq: - rc = cmd_hw2sw_cq(qp->snd_cq.cqn); - if (rc) - eprintf(""); - - exit: - return ret; -} - -static int destroy_udqp(struct udqp_st *qp) -{ - int rc; - - rc = cmd_2err_qpee(qp->qpn); - if (rc) { - eprintf(""); - return rc; - } - tprintf("cmd_2err_qpee(0x%lx) success", qp->qpn); - - rc = cmd_2rst_qpee(qp->qpn); - if (rc) { - eprintf(""); - return rc; - } - tprintf("cmd_2rst_qpee(0x%lx) success", qp->qpn); - - rc = cmd_hw2sw_cq(qp->rcv_cq.cqn); - if (rc) { - eprintf(""); - return rc; - } - tprintf("cmd_hw2sw_cq(0x%lx) success", qp->snd_cq.cqn); - - rc = cmd_hw2sw_cq(qp->snd_cq.cqn); - if (rc) { - eprintf(""); - return rc; - } - tprintf("cmd_hw2sw_cq(0x%lx) success", qp->rcv_cq.cqn); - - return rc; -} - -static void prep_send_wqe_buf(void *qph, - void *avh, - void *wqeh, - const void *buf, - unsigned int offset, __u16 len, __u8 e) -{ - struct udqp_st *qp = qph; - struct ud_av_st *av = avh; - struct ud_send_wqe_st *wqe = wqeh; - - INS_FLD(e, wqe->next.control, wqe_segment_ctrl_send_st, e); - INS_FLD(1, wqe->next.control, wqe_segment_ctrl_send_st, always1); - - wqe->udseg.av_add_h = 0; - wqe->udseg.av_add_l = virt_to_bus(&av->av); - wqe->udseg.dest_qp = av->dest_qp; - wqe->udseg.lkey = dev_ib_data.mkey; - wqe->udseg.qkey = qp->qkey; - - if (buf) { - memcpy(bus_to_virt(wqe->mpointer[0].local_addr_l) + offset, buf, - len); - len += offset; - } - wqe->mpointer[0].byte_count = len; - wqe->mpointer[0].lkey = dev_ib_data.mkey; - - cpu_to_be_buf(wqe, sizeof *wqe); -} - -static void *alloc_ud_av(void) -{ - u8 next_free; - - if (dev_ib_data.udav.udav_next_free == FL_EOL) { - return NULL; - } - - next_free = dev_ib_data.udav.udav_next_free; - dev_ib_data.udav.udav_next_free = - dev_buffers_p->av_array[next_free].ud_av.next_free; - tprintf("allocated udav %d", next_free); - return &dev_buffers_p->av_array[next_free].ud_av; -} - -static void free_ud_av(void *avh) -{ - union ud_av_u *avu; - __u8 idx, old_idx; - struct ud_av_st *av = avh; - - avu = (union ud_av_u *)av; - - idx = avu - dev_buffers_p->av_array; - tprintf("freeing udav idx=%d", idx); - old_idx = dev_ib_data.udav.udav_next_free; - dev_ib_data.udav.udav_next_free = idx; - avu->ud_av.next_free = old_idx; -} - -static int update_cq_cons_idx(struct cq_st *cq) -{ - struct cq_dbell_st dbell; - int rc; - - memset(&dbell, 0, sizeof dbell); - INS_FLD(cq->cqn, &dbell, tavorprm_cq_cmd_doorbell_st, cqn); - INS_FLD(CQ_DBELL_CMD_INC_CONS_IDX, &dbell, tavorprm_cq_cmd_doorbell_st, - cq_cmd); - rc = cmd_post_doorbell(&dbell, CQ_DBELL_OFFSET); - return rc; -} - -static int poll_cq(void *cqh, union cqe_st *cqe_p, u8 * num_cqes) -{ - union cqe_st cqe; - int rc; - u32 *ptr; - struct cq_st *cq = cqh; - - if (cq->cqn < 0x80 || cq->cqn > 0x83) { - eprintf(""); - return -1; - } - ptr = (u32 *) (&(cq->cq_buf[cq->cons_idx])); - barrier(); - if ((ptr[7] & 0x80000000) == 0) { - cqe = cq->cq_buf[cq->cons_idx]; - be_to_cpu_buf(&cqe, sizeof(cqe)); - *cqe_p = cqe; - ptr[7] = 0x80000000; - barrier(); - cq->cons_idx = (cq->cons_idx + 1) % cq->num_cqes; - rc = update_cq_cons_idx(cq); - if (rc) { - return rc; - } - *num_cqes = 1; - } else - *num_cqes = 0; - - return 0; -} - -static void dev2ib_cqe(struct ib_cqe_st *ib_cqe_p, union cqe_st *cqe_p) -{ - __u8 opcode; - __u32 wqe_addr_ba; - - opcode = - EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, opcode); - if (opcode >= CQE_ERROR_OPCODE) - ib_cqe_p->is_error = 1; - else - ib_cqe_p->is_error = 0; - - ib_cqe_p->is_send = - EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, s); - wqe_addr_ba = - EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, - wqe_adr) << 6; - ib_cqe_p->wqe = bus_to_virt(wqe_addr_ba); - -// if (ib_cqe_p->is_send) { -// be_to_cpu_buf(ib_cqe_p->wqe, sizeof(struct ud_send_wqe_st)); -// } -// else { -// be_to_cpu_buf(ib_cqe_p->wqe, sizeof(struct recv_wqe_st)); -// } - ib_cqe_p->count = - EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, - byte_cnt); -} - -static int ib_poll_cq(void *cqh, struct ib_cqe_st *ib_cqe_p, u8 * num_cqes) -{ - int rc; - union cqe_st cqe; - struct cq_st *cq = cqh; - __u8 opcode; - - rc = poll_cq(cq, &cqe, num_cqes); - if (rc || ((*num_cqes) == 0)) { - return rc; - } - - dev2ib_cqe(ib_cqe_p, &cqe); - - opcode = - EX_FLD(cqe.good_cqe, tavorprm_completion_queue_entry_st, opcode); - if (opcode >= CQE_ERROR_OPCODE) { - struct ud_send_wqe_st *wqe_p, wqe; - __u32 *ptr; - unsigned int i; - - wqe_p = - bus_to_virt(EX_FLD - (cqe.error_cqe, - tavorprm_completion_with_error_st, - wqe_addr) << 6); - eprintf("syndrome=0x%lx", - EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, - syndrome)); - eprintf("wqe_addr=0x%lx", wqe_p); - eprintf("wqe_size=0x%lx", - EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, - wqe_size)); - eprintf("myqpn=0x%lx", - EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, - myqpn)); - eprintf("db_cnt=0x%lx", - EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, - db_cnt)); - memcpy(&wqe, wqe_p, sizeof wqe); - be_to_cpu_buf(&wqe, sizeof wqe); - - eprintf("dumping wqe..."); - ptr = (__u32 *) (&wqe); - for (i = 0; i < sizeof wqe; i += 4) { - printf("%lx : ", ptr[i >> 2]); - } - - } - - return rc; -} - -/* always work on ipoib qp */ -static int add_qp_to_mcast_group(union ib_gid_u mcast_gid, __u8 add) -{ - void *mg; - __u8 *tmp; - int rc; - __u16 mgid_hash; - void *mgmqp_p; - - tmp = dev_buffers_p->inprm_buf; - memcpy(tmp, mcast_gid.raw, 16); - be_to_cpu_buf(tmp, 16); - rc = cmd_mgid_hash(tmp, &mgid_hash); - if (!rc) { - mg = (void *)dev_buffers_p->inprm_buf; - memset(mg, 0, MT_STRUCT_SIZE(tavorprm_mgm_entry_st)); - INS_FLD(mcast_gid.as_u32.dw[0], mg, tavorprm_mgm_entry_st, mgid_128_96); // memcpy(&mg->mgid_128_96, &mcast_gid.raw[0], 4); - INS_FLD(mcast_gid.as_u32.dw[1], mg, tavorprm_mgm_entry_st, mgid_95_64); // memcpy(&mg->mgid_95_64, &mcast_gid.raw[4], 4); - INS_FLD(mcast_gid.as_u32.dw[2], mg, tavorprm_mgm_entry_st, mgid_63_32); //memcpy(&mg->mgid_63_32, &mcast_gid.raw[8], 4); - INS_FLD(mcast_gid.as_u32.dw[3], mg, tavorprm_mgm_entry_st, mgid_31_0); //memcpy(&mg->mgid_31_0, &mcast_gid.raw[12], 4); - be_to_cpu_buf(mg + MT_BYTE_OFFSET(tavorprm_mgm_entry_st, mgid_128_96), 16); //be_to_cpu_buf(&mg->mgid_128_96, 16); - mgmqp_p = mg + MT_BYTE_OFFSET(tavorprm_mgm_entry_st, mgmqp_0); - INS_FLD(dev_ib_data.ipoib_qp.qpn, mgmqp_p, tavorprm_mgmqp_st, qpn_i); //mg->mgmqp[0].qpn = dev_ib_data.ipoib_qp.qpn; - INS_FLD(add, mgmqp_p, tavorprm_mgmqp_st, qi); //mg->mgmqp[0].valid = add ? 1 : 0; - rc = cmd_write_mgm(mg, mgid_hash); - } - return rc; -} - -static int clear_interrupt(void) -{ - __u32 ecr; - int ret = 0; - - if (gw_read_cr(0x80704, &ecr)) { - eprintf(""); - } else { - if (ecr) { - ret = 1; - } - } - gw_write_cr(0xf00d8, 0x80000000); /* clear int */ - gw_write_cr(0x8070c, 0xffffffff); - - return ret; -} - -static struct ud_send_wqe_st *alloc_send_wqe(udqp_t qph) -{ - struct udqp_st *qp = qph; - __u8 new_entry; - struct ud_send_wqe_st *wqe; - - if (qp->snd_wqe_cur_free == 0) { - return NULL; - } - new_entry = qp->snd_wqe_alloc_idx; - - wqe = &qp->snd_wq[new_entry].wqe; - qp->snd_wqe_cur_free--; - qp->snd_wqe_alloc_idx = (qp->snd_wqe_alloc_idx + 1) % qp->max_snd_wqes; - - memset(wqe, 0, sizeof *wqe); - - wqe->mpointer[0].local_addr_l = virt_to_bus(qp->snd_bufs[new_entry]); - - return wqe; -} - -/* - * alloc_rcv_wqe - * - * Note: since we work directly on the work queue, wqes - * are left in big endian - */ -static struct recv_wqe_st *alloc_rcv_wqe(struct udqp_st *qp) -{ - __u8 new_entry; - struct recv_wqe_st *wqe; - - if (qp->recv_wqe_cur_free == 0) { - return NULL; - } - - new_entry = qp->recv_wqe_alloc_idx; - wqe = &qp->rcv_wq[new_entry].wqe; - - qp->recv_wqe_cur_free--; - qp->recv_wqe_alloc_idx = - (qp->recv_wqe_alloc_idx + 1) % qp->max_recv_wqes; - - memset(wqe, 0, sizeof *wqe); - - /* GRH is always required */ - wqe->mpointer[0].local_addr_h = 0; - wqe->mpointer[0].local_addr_l = virt_to_bus(qp->rcv_bufs[new_entry]); - wqe->mpointer[0].lkey = dev_ib_data.mkey; - wqe->mpointer[0].byte_count = GRH_SIZE; - - wqe->mpointer[1].local_addr_h = 0; - wqe->mpointer[1].local_addr_l = - virt_to_bus(qp->rcv_bufs[new_entry] + GRH_SIZE); - wqe->mpointer[1].lkey = dev_ib_data.mkey; - wqe->mpointer[1].byte_count = qp->rcv_buf_sz; - - tprintf("rcv_buf=%lx\n", qp->rcv_bufs[new_entry]); - - /* we do it only on the data segment since the control - segment is always owned by HW */ - cpu_to_be_buf(wqe, sizeof *wqe); - -// tprintf("alloc wqe= 0x%x", wqe); - return wqe; -} - -static int free_send_wqe(struct ud_send_wqe_st *wqe) -{ - union ud_send_wqe_u *wqe_u; - struct udqp_st *qp; - - wqe_u = (union ud_send_wqe_u *)wqe; - qp = wqe_u->wqe_cont.qp; - - if (qp->snd_wqe_cur_free >= qp->max_snd_wqes) { - return -1; - } - - qp->snd_wqe_cur_free++; - - return 0; -} - -static int free_rcv_wqe(struct recv_wqe_st *wqe) -{ - union recv_wqe_u *wqe_u; - struct udqp_st *qp; - - wqe_u = (union recv_wqe_u *)wqe; - qp = wqe_u->wqe_cont.qp; - - if (qp->recv_wqe_cur_free >= qp->max_recv_wqes) { - return -1; - } - - qp->recv_wqe_cur_free++; - - return 0; -} - -static int free_wqe(void *wqe) -{ - int rc = 0; - struct recv_wqe_st *rcv_wqe; - -// tprintf("free wqe= 0x%x", wqe); - if ((wqe >= (void *)(dev_ib_data.ipoib_qp.rcv_wq)) && - (wqe < - (void *)(&dev_ib_data.ipoib_qp.rcv_wq[NUM_IPOIB_RCV_WQES]))) { - /* ipoib receive wqe */ - free_rcv_wqe(wqe); - rcv_wqe = alloc_rcv_wqe(&dev_ib_data.ipoib_qp); - if (rcv_wqe) { - rc = post_rcv_buf(&dev_ib_data.ipoib_qp, rcv_wqe); - if (rc) { - eprintf(""); - } - } - } else if (wqe >= (void *)(dev_ib_data.ipoib_qp.snd_wq) && - wqe < - (void *)(&dev_ib_data.ipoib_qp.snd_wq[NUM_IPOIB_SND_WQES])) { - /* ipoib send wqe */ - free_send_wqe(wqe); - } else if (wqe >= (void *)(dev_ib_data.mads_qp.rcv_wq) && - wqe < - (void *)(&dev_ib_data.mads_qp.rcv_wq[NUM_MADS_RCV_WQES])) { - /* mads receive wqe */ - free_rcv_wqe(wqe); - rcv_wqe = alloc_rcv_wqe(&dev_ib_data.mads_qp); - if (rcv_wqe) { - rc = post_rcv_buf(&dev_ib_data.mads_qp, rcv_wqe); - if (rc) { - eprintf(""); - } - } - } else if (wqe >= (void *)(dev_ib_data.mads_qp.snd_wq) && - wqe < - (void *)(&dev_ib_data.mads_qp.snd_wq[NUM_MADS_SND_WQES])) { - /* mads send wqe */ - free_send_wqe(wqe); - } else { - rc = -1; - eprintf(""); - } - - return rc; -} - -static int update_eq_cons_idx(struct eq_st *eq) -{ - struct eq_dbell_st dbell; - int rc; - - memset(&dbell, 0, sizeof dbell); - INS_FLD(dev_ib_data.eq.eqn, &dbell, tavorprm_eq_cmd_doorbell_st, eqn); - INS_FLD(EQ_DBELL_CMD_SET_CONS_IDX, &dbell, tavorprm_eq_cmd_doorbell_st, - eq_cmd); - INS_FLD(eq->cons_idx, &dbell, tavorprm_eq_cmd_doorbell_st, eq_param); - rc = cmd_post_doorbell(&dbell, EQ_DBELL_OFFSET); - - return rc; -} - -static void dev2ib_eqe(struct ib_eqe_st *ib_eqe_p, void *eqe_p) -{ - void *tmp; - - ib_eqe_p->event_type = - EX_FLD(eqe_p, tavorprm_event_queue_entry_st, event_type); - - tmp = eqe_p + MT_BYTE_OFFSET(tavorprm_event_queue_entry_st, event_data); - ib_eqe_p->cqn = EX_FLD(tmp, tavorprm_completion_event_st, cqn); -} - -static int poll_eq(struct ib_eqe_st *ib_eqe_p, __u8 * num_eqes) -{ - struct eqe_t eqe; - __u8 owner; - int rc; - __u32 *ptr; - struct eq_st *eq = &dev_ib_data.eq; - - ptr = (__u32 *) (&(eq->eq_buf[eq->cons_idx])); - tprintf("cons)idx=%d, addr(eqe)=%x, val=0x%x", eq->cons_idx, virt_to_bus(ptr), ptr[7]); - owner = (ptr[7] & 0x80000000) ? OWNER_HW : OWNER_SW; - if (owner == OWNER_SW) { - tprintf("got eqe"); - eqe = eq->eq_buf[eq->cons_idx]; - be_to_cpu_buf(&eqe, sizeof(eqe)); - dev2ib_eqe(ib_eqe_p, &eqe); - ptr[7] |= 0x80000000; - eq->eq_buf[eq->cons_idx] = eqe; - eq->cons_idx = (eq->cons_idx + 1) % eq->eq_size; - rc = update_eq_cons_idx(eq); - if (rc) { - return -1; - } - *num_eqes = 1; - } else { - *num_eqes = 0; - } - return 0; -} - -static int ib_device_close(void) -{ - iounmap(tavor_pci_dev.uar); - iounmap(tavor_pci_dev.cr_space); - iounmap(dev_ib_data.error_buf_addr); - return 0; -} - -static __u32 dev_get_qpn(void *qph) -{ - struct udqp_st *qp = qph; - - return qp->qpn; -} - -static void dev_post_dbell(void *dbell, __u32 offset) -{ - __u32 *ptr; - unsigned long address; - - ptr = dbell; - tprintf("ptr[0]= 0x%lx", ptr[0]); - tprintf("ptr[1]= 0x%lx", ptr[1]); - address = (unsigned long)(tavor_pci_dev.uar) + offset; - tprintf("va=0x%lx pa=0x%lx", address, - virt_to_bus((const void *)address)); - writel(htonl(ptr[0]), tavor_pci_dev.uar + offset); - barrier(); - address += 4; - tprintf("va=0x%lx pa=0x%lx", address, - virt_to_bus((const void *)address)); - writel(htonl(ptr[1]), tavor_pci_dev.uar + offset + 4); -} - - |