diff options
Diffstat (limited to 'drivers/scsi/mpt3sas/mpt3sas_base.c')
| -rw-r--r-- | drivers/scsi/mpt3sas/mpt3sas_base.c | 185 |
1 files changed, 156 insertions, 29 deletions
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index e57774472e75..8aacbd1e7db2 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -94,6 +94,11 @@ module_param(max_msix_vectors, int, 0); MODULE_PARM_DESC(max_msix_vectors, " max msix vectors"); +static int irqpoll_weight = -1; +module_param(irqpoll_weight, int, 0); +MODULE_PARM_DESC(irqpoll_weight, + "irq poll weight (default= one fourth of HBA queue depth)"); + static int mpt3sas_fwfault_debug; MODULE_PARM_DESC(mpt3sas_fwfault_debug, " enable detection of firmware fault and halt firmware - (default=0)"); @@ -1382,20 +1387,30 @@ union reply_descriptor { } u; }; +static u32 base_mod64(u64 dividend, u32 divisor) +{ + u32 remainder; + + if (!divisor) + pr_err("mpt3sas: DIVISOR is zero, in div fn\n"); + remainder = do_div(dividend, divisor); + return remainder; +} + /** - * _base_interrupt - MPT adapter (IOC) specific interrupt handler. - * @irq: irq number (not used) - * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure + * _base_process_reply_queue - Process reply descriptors from reply + * descriptor post queue. + * @reply_q: per IRQ's reply queue object. * - * Return: IRQ_HANDLED if processed, else IRQ_NONE. + * Return: number of reply descriptors processed from reply + * descriptor queue. */ -static irqreturn_t -_base_interrupt(int irq, void *bus_id) +static int +_base_process_reply_queue(struct adapter_reply_queue *reply_q) { - struct adapter_reply_queue *reply_q = bus_id; union reply_descriptor rd; - u32 completed_cmds; - u8 request_desript_type; + u64 completed_cmds; + u8 request_descript_type; u16 smid; u8 cb_idx; u32 reply; @@ -1404,21 +1419,18 @@ _base_interrupt(int irq, void *bus_id) Mpi2ReplyDescriptorsUnion_t *rpf; u8 rc; - if (ioc->mask_interrupts) - return IRQ_NONE; - + completed_cmds = 0; if (!atomic_add_unless(&reply_q->busy, 1, 1)) - return IRQ_NONE; + return completed_cmds; rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index]; - request_desript_type = rpf->Default.ReplyFlags + request_descript_type = rpf->Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; - if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { + if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { atomic_dec(&reply_q->busy); - return IRQ_NONE; + return completed_cmds; } - completed_cmds = 0; cb_idx = 0xFF; do { rd.word = le64_to_cpu(rpf->Words); @@ -1426,11 +1438,11 @@ _base_interrupt(int irq, void *bus_id) goto out; reply = 0; smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1); - if (request_desript_type == + if (request_descript_type == MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS || - request_desript_type == + request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS || - request_desript_type == + request_descript_type == MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) { cb_idx = _base_get_cb_idx(ioc, smid); if ((likely(cb_idx < MPT_MAX_CALLBACKS)) && @@ -1440,7 +1452,7 @@ _base_interrupt(int irq, void *bus_id) if (rc) mpt3sas_base_free_smid(ioc, smid); } - } else if (request_desript_type == + } else if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) { reply = le32_to_cpu( rpf->AddressReply.ReplyFrameAddress); @@ -1486,7 +1498,7 @@ _base_interrupt(int irq, void *bus_id) (reply_q->reply_post_host_index == (ioc->reply_post_queue_depth - 1)) ? 0 : reply_q->reply_post_host_index + 1; - request_desript_type = + request_descript_type = reply_q->reply_post_free[reply_q->reply_post_host_index]. Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; completed_cmds++; @@ -1495,7 +1507,7 @@ _base_interrupt(int irq, void *bus_id) * So that FW can find enough entries to post the Reply * Descriptors in the reply descriptor post queue. */ - if (completed_cmds > ioc->hba_queue_depth/3) { + if (!base_mod64(completed_cmds, ioc->thresh_hold)) { if (ioc->combined_reply_queue) { writel(reply_q->reply_post_host_index | ((msix_index & 7) << @@ -1507,9 +1519,14 @@ _base_interrupt(int irq, void *bus_id) MPI2_RPHI_MSIX_INDEX_SHIFT), &ioc->chip->ReplyPostHostIndex); } - completed_cmds = 1; + if (!reply_q->irq_poll_scheduled) { + reply_q->irq_poll_scheduled = true; + irq_poll_sched(&reply_q->irqpoll); + } + atomic_dec(&reply_q->busy); + return completed_cmds; } - if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) + if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) goto out; if (!reply_q->reply_post_host_index) rpf = reply_q->reply_post_free; @@ -1521,14 +1538,14 @@ _base_interrupt(int irq, void *bus_id) if (!completed_cmds) { atomic_dec(&reply_q->busy); - return IRQ_NONE; + return completed_cmds; } if (ioc->is_warpdrive) { writel(reply_q->reply_post_host_index, ioc->reply_post_host_index[msix_index]); atomic_dec(&reply_q->busy); - return IRQ_HANDLED; + return completed_cmds; } /* Update Reply Post Host Index. @@ -1555,7 +1572,82 @@ _base_interrupt(int irq, void *bus_id) MPI2_RPHI_MSIX_INDEX_SHIFT), &ioc->chip->ReplyPostHostIndex); atomic_dec(&reply_q->busy); - return IRQ_HANDLED; + return completed_cmds; +} + +/** + * _base_interrupt - MPT adapter (IOC) specific interrupt handler. + * @irq: irq number (not used) + * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure + * + * Return: IRQ_HANDLED if processed, else IRQ_NONE. + */ +static irqreturn_t +_base_interrupt(int irq, void *bus_id) +{ + struct adapter_reply_queue *reply_q = bus_id; + struct MPT3SAS_ADAPTER *ioc = reply_q->ioc; + + if (ioc->mask_interrupts) + return IRQ_NONE; + if (reply_q->irq_poll_scheduled) + return IRQ_HANDLED; + return ((_base_process_reply_queue(reply_q) > 0) ? + IRQ_HANDLED : IRQ_NONE); +} + +/** + * _base_irqpoll - IRQ poll callback handler + * @irqpoll - irq_poll object + * @budget - irq poll weight + * + * returns number of reply descriptors processed + */ +static int +_base_irqpoll(struct irq_poll *irqpoll, int budget) +{ + struct adapter_reply_queue *reply_q; + int num_entries = 0; + + reply_q = container_of(irqpoll, struct adapter_reply_queue, + irqpoll); + if (reply_q->irq_line_enable) { + disable_irq(reply_q->os_irq); + reply_q->irq_line_enable = false; + } + num_entries = _base_process_reply_queue(reply_q); + if (num_entries < budget) { + irq_poll_complete(irqpoll); + reply_q->irq_poll_scheduled = false; + reply_q->irq_line_enable = true; + enable_irq(reply_q->os_irq); + } + + return num_entries; +} + +/** + * _base_init_irqpolls - initliaze IRQ polls + * @ioc: per adapter object + * + * returns nothing + */ +static void +_base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc) +{ + struct adapter_reply_queue *reply_q, *next; + + if (list_empty(&ioc->reply_queue_list)) + return; + + list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { + irq_poll_init(&reply_q->irqpoll, + ioc->hba_queue_depth/4, _base_irqpoll); + reply_q->irq_poll_scheduled = false; + reply_q->irq_line_enable = true; + reply_q->os_irq = pci_irq_vector(ioc->pdev, + reply_q->msix_index); + } } /** @@ -1596,6 +1688,17 @@ mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc) /* TMs are on msix_index == 0 */ if (reply_q->msix_index == 0) continue; + if (reply_q->irq_poll_scheduled) { + /* Calling irq_poll_disable will wait for any pending + * callbacks to have completed. + */ + irq_poll_disable(&reply_q->irqpoll); + irq_poll_enable(&reply_q->irqpoll); + reply_q->irq_poll_scheduled = false; + reply_q->irq_line_enable = true; + enable_irq(reply_q->os_irq); + continue; + } synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index)); } } @@ -2757,6 +2860,11 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) if (!_base_is_controller_msix_enabled(ioc)) return; + ioc->msix_load_balance = false; + if (ioc->reply_queue_count < num_online_cpus()) { + ioc->msix_load_balance = true; + return; + } memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz); @@ -3015,6 +3123,8 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) if (r) goto out_fail; + if (!ioc->is_driver_loading) + _base_init_irqpolls(ioc); /* Use the Combined reply queue feature only for SAS3 C0 & higher * revision HBAs and also only when reply queue count is greater than 8 */ @@ -3158,6 +3268,12 @@ mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr) static inline u8 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc) { + /* Enables reply_queue load balancing */ + if (ioc->msix_load_balance) + return ioc->reply_queue_count ? + base_mod64(atomic64_add_return(1, + &ioc->total_io_cnt), ioc->reply_queue_count) : 0; + return ioc->cpu_msix_table[raw_smp_processor_id()]; } @@ -3281,12 +3397,18 @@ mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) if (smid < ioc->hi_priority_smid) { struct scsiio_tracker *st; + void *request; st = _get_st_from_smid(ioc, smid); if (!st) { _base_recovery_check(ioc); return; } + + /* Clear MPI request frame */ + request = mpt3sas_base_get_msg_frame(ioc, smid); + memset(request, 0, ioc->request_sz); + mpt3sas_base_clear_st(ioc, st); _base_recovery_check(ioc); return; @@ -3327,7 +3449,6 @@ _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr, spin_lock_irqsave(writeq_lock, flags); __raw_writel((u32)(b), addr); __raw_writel((u32)(b >> 32), (addr + 4)); - mmiowb(); spin_unlock_irqrestore(writeq_lock, flags); } @@ -6501,6 +6622,12 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) if (r) goto out_free_resources; + if (irqpoll_weight > 0) + ioc->thresh_hold = irqpoll_weight; + else + ioc->thresh_hold = ioc->hba_queue_depth/4; + + _base_init_irqpolls(ioc); init_waitqueue_head(&ioc->reset_wq); /* allocate memory pd handle bitmask list */ |