summaryrefslogtreecommitdiffstats
path: root/drivers/scsi/lpfc/lpfc_sli.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/lpfc/lpfc_sli.c')
-rw-r--r--drivers/scsi/lpfc/lpfc_sli.c692
1 files changed, 484 insertions, 208 deletions
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 5f5528a12308..cb17e2b2be81 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -1,8 +1,7 @@
-
/*******************************************************************
* This file is part of the Emulex Linux Device Driver for *
* Fibre Channel Host Bus Adapters. *
- * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
* “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
* Copyright (C) 2004-2016 Emulex. All rights reserved. *
* EMULEX and SLI are trademarks of Emulex. *
@@ -36,6 +35,9 @@
#include <scsi/scsi_transport_fc.h>
#include <scsi/fc/fc_fs.h>
#include <linux/aer.h>
+#ifdef CONFIG_X86
+#include <asm/set_memory.h>
+#endif
#include <linux/nvme-fc-driver.h>
@@ -107,12 +109,14 @@ lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
* The caller is expected to hold the hbalock when calling this routine.
**/
static int
-lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
+lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
{
union lpfc_wqe *temp_wqe;
struct lpfc_register doorbell;
uint32_t host_index;
uint32_t idx;
+ uint32_t i = 0;
+ uint8_t *tmp;
/* sanity check on queue memory */
if (unlikely(!q))
@@ -129,10 +133,25 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
/* set consumption flag every once in a while */
if (!((q->host_index + 1) % q->entry_repost))
bf_set(wqe_wqec, &wqe->generic.wqe_com, 1);
+ else
+ bf_set(wqe_wqec, &wqe->generic.wqe_com, 0);
if (q->phba->sli3_options & LPFC_SLI4_PHWQ_ENABLED)
bf_set(wqe_wqid, &wqe->generic.wqe_com, q->queue_id);
lpfc_sli_pcimem_bcopy(wqe, temp_wqe, q->entry_size);
- /* ensure WQE bcopy flushed before doorbell write */
+ if (q->dpp_enable && q->phba->cfg_enable_dpp) {
+ /* write to DPP aperture taking advatage of Combined Writes */
+ tmp = (uint8_t *)temp_wqe;
+#ifdef __raw_writeq
+ for (i = 0; i < q->entry_size; i += sizeof(uint64_t))
+ __raw_writeq(*((uint64_t *)(tmp + i)),
+ q->dpp_regaddr + i);
+#else
+ for (i = 0; i < q->entry_size; i += sizeof(uint32_t))
+ __raw_writel(*((uint32_t *)(tmp + i)),
+ q->dpp_regaddr + i);
+#endif
+ }
+ /* ensure WQE bcopy and DPP flushed before doorbell write */
wmb();
/* Update the host index before invoking device */
@@ -143,9 +162,18 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
/* Ring Doorbell */
doorbell.word0 = 0;
if (q->db_format == LPFC_DB_LIST_FORMAT) {
- bf_set(lpfc_wq_db_list_fm_num_posted, &doorbell, 1);
- bf_set(lpfc_wq_db_list_fm_index, &doorbell, host_index);
- bf_set(lpfc_wq_db_list_fm_id, &doorbell, q->queue_id);
+ if (q->dpp_enable && q->phba->cfg_enable_dpp) {
+ bf_set(lpfc_if6_wq_db_list_fm_num_posted, &doorbell, 1);
+ bf_set(lpfc_if6_wq_db_list_fm_dpp, &doorbell, 1);
+ bf_set(lpfc_if6_wq_db_list_fm_dpp_id, &doorbell,
+ q->dpp_id);
+ bf_set(lpfc_if6_wq_db_list_fm_id, &doorbell,
+ q->queue_id);
+ } else {
+ bf_set(lpfc_wq_db_list_fm_num_posted, &doorbell, 1);
+ bf_set(lpfc_wq_db_list_fm_index, &doorbell, host_index);
+ bf_set(lpfc_wq_db_list_fm_id, &doorbell, q->queue_id);
+ }
} else if (q->db_format == LPFC_DB_RING_FORMAT) {
bf_set(lpfc_wq_db_ring_fm_num_posted, &doorbell, 1);
bf_set(lpfc_wq_db_ring_fm_id, &doorbell, q->queue_id);
@@ -262,16 +290,18 @@ lpfc_sli4_mq_release(struct lpfc_queue *q)
static struct lpfc_eqe *
lpfc_sli4_eq_get(struct lpfc_queue *q)
{
+ struct lpfc_hba *phba;
struct lpfc_eqe *eqe;
uint32_t idx;
/* sanity check on queue memory */
if (unlikely(!q))
return NULL;
+ phba = q->phba;
eqe = q->qe[q->hba_index].eqe;
/* If the next EQE is not valid then we are done */
- if (!bf_get_le32(lpfc_eqe_valid, eqe))
+ if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid)
return NULL;
/* If the host has not yet processed the next entry then we are done */
idx = ((q->hba_index + 1) % q->entry_count);
@@ -279,6 +309,10 @@ lpfc_sli4_eq_get(struct lpfc_queue *q)
return NULL;
q->hba_index = idx;
+ /* if the index wrapped around, toggle the valid bit */
+ if (phba->sli4_hba.pc_sli4_params.eqav && !q->hba_index)
+ q->qe_valid = (q->qe_valid) ? 0 : 1;
+
/*
* insert barrier for instruction interlock : data from the hardware
@@ -298,7 +332,7 @@ lpfc_sli4_eq_get(struct lpfc_queue *q)
* @q: The Event Queue to disable interrupts
*
**/
-static inline void
+inline void
lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
{
struct lpfc_register doorbell;
@@ -309,7 +343,26 @@ lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
- writel(doorbell.word0, q->phba->sli4_hba.EQCQDBregaddr);
+ writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
+}
+
+/**
+ * lpfc_sli4_if6_eq_clr_intr - Turn off interrupts from this EQ
+ * @q: The Event Queue to disable interrupts
+ *
+ **/
+inline void
+lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q)
+{
+ struct lpfc_register doorbell;
+
+ doorbell.word0 = 0;
+ bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
+ bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
+ bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
+ (q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
+ bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
+ writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
}
/**
@@ -331,17 +384,21 @@ uint32_t
lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm)
{
uint32_t released = 0;
+ struct lpfc_hba *phba;
struct lpfc_eqe *temp_eqe;
struct lpfc_register doorbell;
/* sanity check on queue memory */
if (unlikely(!q))
return 0;
+ phba = q->phba;
/* while there are valid entries */
while (q->hba_index != q->host_index) {
- temp_eqe = q->qe[q->host_index].eqe;
- bf_set_le32(lpfc_eqe_valid, temp_eqe, 0);
+ if (!phba->sli4_hba.pc_sli4_params.eqav) {
+ temp_eqe = q->qe[q->host_index].eqe;
+ bf_set_le32(lpfc_eqe_valid, temp_eqe, 0);
+ }
released++;
q->host_index = ((q->host_index + 1) % q->entry_count);
}
@@ -359,10 +416,63 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm)
bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
- writel(doorbell.word0, q->phba->sli4_hba.EQCQDBregaddr);
+ writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
/* PCI read to flush PCI pipeline on re-arming for INTx mode */
if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
- readl(q->phba->sli4_hba.EQCQDBregaddr);
+ readl(q->phba->sli4_hba.EQDBregaddr);
+ return released;
+}
+
+/**
+ * lpfc_sli4_if6_eq_release - Indicates the host has finished processing an EQ
+ * @q: The Event Queue that the host has completed processing for.
+ * @arm: Indicates whether the host wants to arms this CQ.
+ *
+ * This routine will mark all Event Queue Entries on @q, from the last
+ * known completed entry to the last entry that was processed, as completed
+ * by clearing the valid bit for each completion queue entry. Then it will
+ * notify the HBA, by ringing the doorbell, that the EQEs have been processed.
+ * The internal host index in the @q will be updated by this routine to indicate
+ * that the host has finished processing the entries. The @arm parameter
+ * indicates that the queue should be rearmed when ringing the doorbell.
+ *
+ * This function will return the number of EQEs that were popped.
+ **/
+uint32_t
+lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm)
+{
+ uint32_t released = 0;
+ struct lpfc_hba *phba;
+ struct lpfc_eqe *temp_eqe;
+ struct lpfc_register doorbell;
+
+ /* sanity check on queue memory */
+ if (unlikely(!q))
+ return 0;
+ phba = q->phba;
+
+ /* while there are valid entries */
+ while (q->hba_index != q->host_index) {
+ if (!phba->sli4_hba.pc_sli4_params.eqav) {
+ temp_eqe = q->qe[q->host_index].eqe;
+ bf_set_le32(lpfc_eqe_valid, temp_eqe, 0);
+ }
+ released++;
+ q->host_index = ((q->host_index + 1) % q->entry_count);
+ }
+ if (unlikely(released == 0 && !arm))
+ return 0;
+
+ /* ring doorbell for number popped */
+ doorbell.word0 = 0;
+ if (arm)
+ bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1);
+ bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, released);
+ bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id);
+ writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr);
+ /* PCI read to flush PCI pipeline on re-arming for INTx mode */
+ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM))
+ readl(q->phba->sli4_hba.EQDBregaddr);
return released;
}
@@ -378,23 +488,28 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm)
static struct lpfc_cqe *
lpfc_sli4_cq_get(struct lpfc_queue *q)
{
+ struct lpfc_hba *phba;
struct lpfc_cqe *cqe;
uint32_t idx;
/* sanity check on queue memory */
if (unlikely(!q))
return NULL;
+ phba = q->phba;
+ cqe = q->qe[q->hba_index].cqe;
/* If the next CQE is not valid then we are done */
- if (!bf_get_le32(lpfc_cqe_valid, q->qe[q->hba_index].cqe))
+ if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid)
return NULL;
/* If the host has not yet processed the next entry then we are done */
idx = ((q->hba_index + 1) % q->entry_count);
if (idx == q->host_index)
return NULL;
- cqe = q->qe[q->hba_index].cqe;
q->hba_index = idx;
+ /* if the index wrapped around, toggle the valid bit */
+ if (phba->sli4_hba.pc_sli4_params.cqav && !q->hba_index)
+ q->qe_valid = (q->qe_valid) ? 0 : 1;
/*
* insert barrier for instruction interlock : data from the hardware
@@ -427,16 +542,21 @@ uint32_t
lpfc_sli4_cq_release(struct lpfc_queue *q, bool arm)
{
uint32_t released = 0;
+ struct lpfc_hba *phba;
struct lpfc_cqe *temp_qe;
struct lpfc_register doorbell;
/* sanity check on queue memory */
if (unlikely(!q))
return 0;
+ phba = q->phba;
+
/* while there are valid entries */
while (q->hba_index != q->host_index) {
- temp_qe = q->qe[q->host_index].cqe;
- bf_set_le32(lpfc_cqe_valid, temp_qe, 0);
+ if (!phba->sli4_hba.pc_sli4_params.cqav) {
+ temp_qe = q->qe[q->host_index].cqe;
+ bf_set_le32(lpfc_cqe_valid, temp_qe, 0);
+ }
released++;
q->host_index = ((q->host_index + 1) % q->entry_count);
}
@@ -452,7 +572,57 @@ lpfc_sli4_cq_release(struct lpfc_queue *q, bool arm)
bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell,
(q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT));
bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id);
- writel(doorbell.word0, q->phba->sli4_hba.EQCQDBregaddr);
+ writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
+ return released;
+}
+
+/**
+ * lpfc_sli4_if6_cq_release - Indicates the host has finished processing a CQ
+ * @q: The Completion Queue that the host has completed processing for.
+ * @arm: Indicates whether the host wants to arms this CQ.
+ *
+ * This routine will mark all Completion queue entries on @q, from the last
+ * known completed entry to the last entry that was processed, as completed
+ * by clearing the valid bit for each completion queue entry. Then it will
+ * notify the HBA, by ringing the doorbell, that the CQEs have been processed.
+ * The internal host index in the @q will be updated by this routine to indicate
+ * that the host has finished processing the entries. The @arm parameter
+ * indicates that the queue should be rearmed when ringing the doorbell.
+ *
+ * This function will return the number of CQEs that were released.
+ **/
+uint32_t
+lpfc_sli4_if6_cq_release(struct lpfc_queue *q, bool arm)
+{
+ uint32_t released = 0;
+ struct lpfc_hba *phba;
+ struct lpfc_cqe *temp_qe;
+ struct lpfc_register doorbell;
+
+ /* sanity check on queue memory */
+ if (unlikely(!q))
+ return 0;
+ phba = q->phba;
+
+ /* while there are valid entries */
+ while (q->hba_index != q->host_index) {
+ if (!phba->sli4_hba.pc_sli4_params.cqav) {
+ temp_qe = q->qe[q->host_index].cqe;
+ bf_set_le32(lpfc_cqe_valid, temp_qe, 0);
+ }
+ released++;
+ q->host_index = ((q->host_index + 1) % q->entry_count);
+ }
+ if (unlikely(released == 0 && !arm))
+ return 0;
+
+ /* ring doorbell for number popped */
+ doorbell.word0 = 0;
+ if (arm)
+ bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1);
+ bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, released);
+ bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id);
+ writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr);
return released;
}
@@ -2218,18 +2388,18 @@ lpfc_sli_chk_mbx_command(uint8_t mbxCommand)
void
lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
{
- wait_queue_head_t *pdone_q;
unsigned long drvr_flag;
+ struct completion *pmbox_done;
/*
- * If pdone_q is empty, the driver thread gave up waiting and
+ * If pmbox_done is empty, the driver thread gave up waiting and
* continued running.
*/
pmboxq->mbox_flag |= LPFC_MBX_WAKE;
spin_lock_irqsave(&phba->hbalock, drvr_flag);
- pdone_q = (wait_queue_head_t *) pmboxq->context1;
- if (pdone_q)
- wake_up_interruptible(pdone_q);
+ pmbox_done = (struct completion *)pmboxq->context3;
+ if (pmbox_done)
+ complete(pmbox_done);
spin_unlock_irqrestore(&phba->hbalock, drvr_flag);
return;
}
@@ -2330,7 +2500,7 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) {
if (phba->sli_rev == LPFC_SLI_REV4 &&
(bf_get(lpfc_sli_intf_if_type,
- &phba->sli4_hba.sli_intf) ==
+ &phba->sli4_hba.sli_intf) >=
LPFC_SLI_INTF_IF_TYPE_2)) {
if (ndlp) {
lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI,
@@ -3776,6 +3946,7 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba)
struct lpfc_sli *psli = &phba->sli;
struct lpfc_sli_ring *pring;
uint32_t i;
+ struct lpfc_iocbq *piocb, *next_iocb;
spin_lock_irq(&phba->hbalock);
/* Indicate the I/O queues are flushed */
@@ -3790,6 +3961,9 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba)
spin_lock_irq(&pring->ring_lock);
/* Retrieve everything on txq */
list_splice_init(&pring->txq, &txq);
+ list_for_each_entry_safe(piocb, next_iocb,
+ &pring->txcmplq, list)
+ piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
/* Retrieve everything on the txcmplq */
list_splice_init(&pring->txcmplq, &txcmplq);
pring->txq_cnt = 0;
@@ -3811,6 +3985,9 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba)
spin_lock_irq(&phba->hbalock);
/* Retrieve everything on txq */
list_splice_init(&pring->txq, &txq);
+ list_for_each_entry_safe(piocb, next_iocb,
+ &pring->txcmplq, list)
+ piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
/* Retrieve everything on the txcmplq */
list_splice_init(&pring->txcmplq, &txcmplq);
pring->txq_cnt = 0;
@@ -3842,6 +4019,7 @@ lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba)
LIST_HEAD(txcmplq);
struct lpfc_sli_ring *pring;
uint32_t i;
+ struct lpfc_iocbq *piocb, *next_iocb;
if (phba->sli_rev < LPFC_SLI_REV4)
return;
@@ -3858,8 +4036,11 @@ lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba)
for (i = 0; i < phba->cfg_nvme_io_channel; i++) {
pring = phba->sli4_hba.nvme_wq[i]->pring;
- /* Retrieve everything on the txcmplq */
spin_lock_irq(&pring->ring_lock);
+ list_for_each_entry_safe(piocb, next_iocb,
+ &pring->txcmplq, list)
+ piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
+ /* Retrieve everything on the txcmplq */
list_splice_init(&pring->txcmplq, &txcmplq);
pring->txcmplq_cnt = 0;
spin_unlock_irq(&pring->ring_lock);
@@ -4812,13 +4993,14 @@ lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode)
phba->hbq_get = phba->mbox->us.s3_pgp.hbq_get;
phba->port_gp = phba->mbox->us.s3_pgp.port;
- if (phba->cfg_enable_bg) {
- if (pmb->u.mb.un.varCfgPort.gbg)
- phba->sli3_options |= LPFC_SLI3_BG_ENABLED;
- else
+ if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
+ if (pmb->u.mb.un.varCfgPort.gbg == 0) {
+ phba->cfg_enable_bg = 0;
+ phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
"0443 Adapter did not grant "
"BlockGuard\n");
+ }
}
} else {
phba->hbq_get = NULL;
@@ -5290,41 +5472,42 @@ static void
lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba)
{
int qidx;
+ struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
- lpfc_sli4_cq_release(phba->sli4_hba.mbx_cq, LPFC_QUEUE_REARM);
- lpfc_sli4_cq_release(phba->sli4_hba.els_cq, LPFC_QUEUE_REARM);
- if (phba->sli4_hba.nvmels_cq)
- lpfc_sli4_cq_release(phba->sli4_hba.nvmels_cq,
+ sli4_hba->sli4_cq_release(sli4_hba->mbx_cq, LPFC_QUEUE_REARM);
+ sli4_hba->sli4_cq_release(sli4_hba->els_cq, LPFC_QUEUE_REARM);
+ if (sli4_hba->nvmels_cq)
+ sli4_hba->sli4_cq_release(sli4_hba->nvmels_cq,
LPFC_QUEUE_REARM);
- if (phba->sli4_hba.fcp_cq)
+ if (sli4_hba->fcp_cq)
for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++)
- lpfc_sli4_cq_release(phba->sli4_hba.fcp_cq[qidx],
+ sli4_hba->sli4_cq_release(sli4_hba->fcp_cq[qidx],
LPFC_QUEUE_REARM);
- if (phba->sli4_hba.nvme_cq)
+ if (sli4_hba->nvme_cq)
for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++)
- lpfc_sli4_cq_release(phba->sli4_hba.nvme_cq[qidx],
+ sli4_hba->sli4_cq_release(sli4_hba->nvme_cq[qidx],
LPFC_QUEUE_REARM);
if (phba->cfg_fof)
- lpfc_sli4_cq_release(phba->sli4_hba.oas_cq, LPFC_QUEUE_REARM);
+ sli4_hba->sli4_cq_release(sli4_hba->oas_cq, LPFC_QUEUE_REARM);
- if (phba->sli4_hba.hba_eq)
+ if (sli4_hba->hba_eq)
for (qidx = 0; qidx < phba->io_channel_irqs; qidx++)
- lpfc_sli4_eq_release(phba->sli4_hba.hba_eq[qidx],
- LPFC_QUEUE_REARM);
+ sli4_hba->sli4_eq_release(sli4_hba->hba_eq[qidx],
+ LPFC_QUEUE_REARM);
if (phba->nvmet_support) {
for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) {
- lpfc_sli4_cq_release(
- phba->sli4_hba.nvmet_cqset[qidx],
+ sli4_hba->sli4_cq_release(
+ sli4_hba->nvmet_cqset[qidx],
LPFC_QUEUE_REARM);
}
}
if (phba->cfg_fof)
- lpfc_sli4_eq_release(phba->sli4_hba.fof_eq, LPFC_QUEUE_REARM);
+ sli4_hba->sli4_eq_release(sli4_hba->fof_eq, LPFC_QUEUE_REARM);
}
/**
@@ -6533,9 +6716,11 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
struct lpfc_rqe hrqe;
struct lpfc_rqe drqe;
struct lpfc_rqb *rqbp;
+ unsigned long flags;
struct rqb_dmabuf *rqb_buffer;
LIST_HEAD(rqb_buf_list);
+ spin_lock_irqsave(&phba->hbalock, flags);
rqbp = hrq->rqbp;
for (i = 0; i < count; i++) {
/* IF RQ is already full, don't bother */
@@ -6559,6 +6744,15 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys);
rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe);
if (rc < 0) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "6421 Cannot post to HRQ %d: %x %x %x "
+ "DRQ %x %x\n",
+ hrq->queue_id,
+ hrq->host_index,
+ hrq->hba_index,
+ hrq->entry_count,
+ drq->host_index,
+ drq->hba_index);
rqbp->rqb_free_buffer(phba, rqb_buffer);
} else {
list_add_tail(&rqb_buffer->hbuf.list,
@@ -6566,6 +6760,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq,
rqbp->buffer_count++;
}
}
+ spin_unlock_irqrestore(&phba->hbalock, flags);
return 1;
}
@@ -6693,6 +6888,18 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
/* Save information as VPD data */
phba->vpd.rev.biuRev = mqe->un.read_rev.first_hw_rev;
phba->vpd.rev.smRev = mqe->un.read_rev.second_hw_rev;
+
+ /*
+ * This is because first G7 ASIC doesn't support the standard
+ * 0x5a NVME cmd descriptor type/subtype
+ */
+ if ((bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) ==
+ LPFC_SLI_INTF_IF_TYPE_6) &&
+ (phba->vpd.rev.biuRev == LPFC_G7_ASIC_1) &&
+ (phba->vpd.rev.smRev == 0) &&
+ (phba->cfg_nvme_embed_cmd == 1))
+ phba->cfg_nvme_embed_cmd = 0;
+
phba->vpd.rev.endecRev = mqe->un.read_rev.third_hw_rev;
phba->vpd.rev.fcphHigh = bf_get(lpfc_mbx_rd_rev_fcph_high,
&mqe->un.read_rev);
@@ -6771,21 +6978,26 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
"0378 No support for fcpi mode.\n");
ftr_rsp++;
}
- if (bf_get(lpfc_mbx_rq_ftr_rsp_perfh, &mqe->un.req_ftrs))
- phba->sli3_options |= LPFC_SLI4_PERFH_ENABLED;
- else
- phba->sli3_options &= ~LPFC_SLI4_PERFH_ENABLED;
+
+ /* Performance Hints are ONLY for FCoE */
+ if (phba->hba_flag & HBA_FCOE_MODE) {
+ if (bf_get(lpfc_mbx_rq_ftr_rsp_perfh, &mqe->un.req_ftrs))
+ phba->sli3_options |= LPFC_SLI4_PERFH_ENABLED;
+ else
+ phba->sli3_options &= ~LPFC_SLI4_PERFH_ENABLED;
+ }
+
/*
* If the port cannot support the host's requested features
* then turn off the global config parameters to disable the
* feature in the driver. This is not a fatal error.
*/
- phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
- if (phba->cfg_enable_bg) {
- if (bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs))
- phba->sli3_options |= LPFC_SLI3_BG_ENABLED;
- else
+ if (phba->sli3_options & LPFC_SLI3_BG_ENABLED) {
+ if (!(bf_get(lpfc_mbx_rq_ftr_rsp_dif, &mqe->un.req_ftrs))) {
+ phba->cfg_enable_bg = 0;
+ phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED;
ftr_rsp++;
+ }
}
if (phba->max_vpi && phba->cfg_enable_npiv &&
@@ -7209,6 +7421,7 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
struct lpfc_queue *mcq;
struct lpfc_mcqe *mcqe;
bool pending_completions = false;
+ uint8_t qe_valid;
if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4))
return false;
@@ -7217,7 +7430,8 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
mcq = phba->sli4_hba.mbx_cq;
idx = mcq->hba_index;
- while (bf_get_le32(lpfc_cqe_valid, mcq->qe[idx].cqe)) {
+ qe_valid = mcq->qe_valid;
+ while (bf_get_le32(lpfc_cqe_valid, mcq->qe[idx].cqe) == qe_valid) {
mcqe = (struct lpfc_mcqe *)mcq->qe[idx].cqe;
if (bf_get_le32(lpfc_trailer_completed, mcqe) &&
(!bf_get_le32(lpfc_trailer_async, mcqe))) {
@@ -7227,6 +7441,10 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
idx = (idx + 1) % mcq->entry_count;
if (mcq->hba_index == idx)
break;
+
+ /* if the index wrapped around, toggle the valid bit */
+ if (phba->sli4_hba.pc_sli4_params.cqav && !idx)
+ qe_valid = (qe_valid) ? 0 : 1;
}
return pending_completions;
@@ -7246,7 +7464,7 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
bool
lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
{
-
+ struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba;
uint32_t eqidx;
struct lpfc_queue *fpeq = NULL;
struct lpfc_eqe *eqe;
@@ -7257,11 +7475,11 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
/* Find the eq associated with the mcq */
- if (phba->sli4_hba.hba_eq)
+ if (sli4_hba->hba_eq)
for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++)
- if (phba->sli4_hba.hba_eq[eqidx]->queue_id ==
- phba->sli4_hba.mbx_cq->assoc_qid) {
- fpeq = phba->sli4_hba.hba_eq[eqidx];
+ if (sli4_hba->hba_eq[eqidx]->queue_id ==
+ sli4_hba->mbx_cq->assoc_qid) {
+ fpeq = sli4_hba->hba_eq[eqidx];
break;
}
if (!fpeq)
@@ -7269,7 +7487,7 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
/* Turn off interrupts from this EQ */
- lpfc_sli4_eq_clr_intr(fpeq);
+ sli4_hba->sli4_eq_clr_intr(fpeq);
/* Check to see if a mbox completion is pending */
@@ -7290,7 +7508,7 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba)
/* Always clear and re-arm the EQ */
- lpfc_sli4_eq_release(fpeq, LPFC_QUEUE_REARM);
+ sli4_hba->sli4_eq_release(fpeq, LPFC_QUEUE_REARM);
return mbox_pending;
@@ -8100,7 +8318,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
} else if (flag == MBX_POLL) {
lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
"(%d):2542 Try to issue mailbox command "
- "x%x (x%x/x%x) synchronously ahead of async"
+ "x%x (x%x/x%x) synchronously ahead of async "
"mailbox command queue: x%x x%x\n",
mboxq->vport ? mboxq->vport->vpi : 0,
mboxq->u.mb.mbxCommand,
@@ -8664,7 +8882,7 @@ lpfc_sli4_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq,
**/
static int
lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
- union lpfc_wqe *wqe)
+ union lpfc_wqe128 *wqe)
{
uint32_t xmit_len = 0, total_len = 0;
uint8_t ct = 0;
@@ -8767,7 +8985,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
iocbq->context2)->virt);
if_type = bf_get(lpfc_sli_intf_if_type,
&phba->sli4_hba.sli_intf);
- if (if_type == LPFC_SLI_INTF_IF_TYPE_2) {
+ if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
if (pcmd && (*pcmd == ELS_CMD_FLOGI ||
*pcmd == ELS_CMD_SCR ||
*pcmd == ELS_CMD_FDISC ||
@@ -8870,31 +9088,36 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
}
/* Note, word 10 is already initialized to 0 */
+ /* Don't set PBDE for Perf hints, just fcp_embed_pbde */
+ if (phba->fcp_embed_pbde)
+ bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
+ else
+ bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0);
+
if (phba->fcp_embed_io) {
struct lpfc_scsi_buf *lpfc_cmd;
struct sli4_sge *sgl;
- union lpfc_wqe128 *wqe128;
struct fcp_cmnd *fcp_cmnd;
uint32_t *ptr;
/* 128 byte wqe support here */
- wqe128 = (union lpfc_wqe128 *)wqe;
lpfc_cmd = iocbq->context1;
sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl;
fcp_cmnd = lpfc_cmd->fcp_cmnd;
/* Word 0-2 - FCP_CMND */
- wqe128->generic.bde.tus.f.bdeFlags =
+ wqe->generic.bde.tus.f.bdeFlags =
BUFF_TYPE_BDE_IMMED;
- wqe128->generic.bde.tus.f.bdeSize = sgl->sge_len;
- wqe128->generic.bde.addrHigh = 0;
- wqe128->generic.bde.addrLow = 88; /* Word 22 */
+ wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
+ wqe->generic.bde.addrHigh = 0;
+ wqe->generic.bde.addrLow = 88; /* Word 22 */
- bf_set(wqe_wqes, &wqe128->fcp_iwrite.wqe_com, 1);
+ bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
+ bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
/* Word 22-29 FCP CMND Payload */
- ptr = &wqe128->words[22];
+ ptr = &wqe->words[22];
memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
}
break;
@@ -8929,31 +9152,36 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
}
/* Note, word 10 is already initialized to 0 */
+ /* Don't set PBDE for Perf hints, just fcp_embed_pbde */
+ if (phba->fcp_embed_pbde)
+ bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
+ else
+ bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0);
+
if (phba->fcp_embed_io) {
struct lpfc_scsi_buf *lpfc_cmd;
struct sli4_sge *sgl;
- union lpfc_wqe128 *wqe128;
struct fcp_cmnd *fcp_cmnd;
uint32_t *ptr;
/* 128 byte wqe support here */
- wqe128 = (union lpfc_wqe128 *)wqe;
lpfc_cmd = iocbq->context1;
sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl;
fcp_cmnd = lpfc_cmd->fcp_cmnd;
/* Word 0-2 - FCP_CMND */
- wqe128->generic.bde.tus.f.bdeFlags =
+ wqe->generic.bde.tus.f.bdeFlags =
BUFF_TYPE_BDE_IMMED;
- wqe128->generic.bde.tus.f.bdeSize = sgl->sge_len;
- wqe128->generic.bde.addrHigh = 0;
- wqe128->generic.bde.addrLow = 88; /* Word 22 */
+ wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
+ wqe->generic.bde.addrHigh = 0;
+ wqe->generic.bde.addrLow = 88; /* Word 22 */
- bf_set(wqe_wqes, &wqe128->fcp_iread.wqe_com, 1);
+ bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
+ bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
/* Word 22-29 FCP CMND Payload */
- ptr = &wqe128->words[22];
+ ptr = &wqe->words[22];
memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
}
break;
@@ -8990,28 +9218,27 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
if (phba->fcp_embed_io) {
struct lpfc_scsi_buf *lpfc_cmd;
struct sli4_sge *sgl;
- union lpfc_wqe128 *wqe128;
struct fcp_cmnd *fcp_cmnd;
uint32_t *ptr;
/* 128 byte wqe support here */
- wqe128 = (union lpfc_wqe128 *)wqe;
lpfc_cmd = iocbq->context1;
sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl;
fcp_cmnd = lpfc_cmd->fcp_cmnd;
/* Word 0-2 - FCP_CMND */
- wqe128->generic.bde.tus.f.bdeFlags =
+ wqe->generic.bde.tus.f.bdeFlags =
BUFF_TYPE_BDE_IMMED;
- wqe128->generic.bde.tus.f.bdeSize = sgl->sge_len;
- wqe128->generic.bde.addrHigh = 0;
- wqe128->generic.bde.addrLow = 88; /* Word 22 */
+ wqe->generic.bde.tus.f.bdeSize = sgl->sge_len;
+ wqe->generic.bde.addrHigh = 0;
+ wqe->generic.bde.addrLow = 88; /* Word 22 */
- bf_set(wqe_wqes, &wqe128->fcp_icmd.wqe_com, 1);
+ bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
+ bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
/* Word 22-29 FCP CMND Payload */
- ptr = &wqe128->words[22];
+ ptr = &wqe->words[22];
memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd));
}
break;
@@ -9064,7 +9291,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq,
if_type = bf_get(lpfc_sli_intf_if_type,
&phba->sli4_hba.sli_intf);
- if (if_type == LPFC_SLI_INTF_IF_TYPE_2) {
+ if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) {
if (iocbq->vport->fc_flag & FC_PT2PT) {
bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1);
bf_set(els_rsp64_sid, &wqe->xmit_els_rsp,
@@ -9249,8 +9476,7 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
struct lpfc_iocbq *piocb, uint32_t flag)
{
struct lpfc_sglq *sglq;
- union lpfc_wqe *wqe;
- union lpfc_wqe128 wqe128;
+ union lpfc_wqe128 wqe;
struct lpfc_queue *wq;
struct lpfc_sli_ring *pring;
@@ -9270,9 +9496,7 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
/*
* The WQE can be either 64 or 128 bytes,
- * so allocate space on the stack assuming the largest.
*/
- wqe = (union lpfc_wqe *)&wqe128;
lockdep_assert_held(&phba->hbalock);
@@ -9322,10 +9546,10 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
return IOCB_ERROR;
}
- if (lpfc_sli4_iocb2wqe(phba, piocb, wqe))
+ if (lpfc_sli4_iocb2wqe(phba, piocb, &wqe))
return IOCB_ERROR;
- if (lpfc_sli4_wq_put(wq, wqe))
+ if (lpfc_sli4_wq_put(wq, &wqe))
return IOCB_ERROR;
lpfc_sli_ringtxcmpl_put(phba, pring, piocb);
@@ -9470,7 +9694,7 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
fpeq = phba->sli4_hba.hba_eq[idx];
/* Turn off interrupts from this EQ */
- lpfc_sli4_eq_clr_intr(fpeq);
+ phba->sli4_hba.sli4_eq_clr_intr(fpeq);
/*
* Process all the events on FCP EQ
@@ -9482,7 +9706,7 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
}
/* Always clear and re-arm the EQ */
- lpfc_sli4_eq_release(fpeq,
+ phba->sli4_hba.sli4_eq_release(fpeq,
LPFC_QUEUE_REARM);
}
atomic_inc(&hba_eq_hdl->hba_eq_in_use);
@@ -10695,7 +10919,7 @@ lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
{
struct lpfc_vport *vport = cmdiocb->vport;
struct lpfc_iocbq *abtsiocbp;
- union lpfc_wqe *abts_wqe;
+ union lpfc_wqe128 *abts_wqe;
int retval;
/*
@@ -11442,31 +11666,25 @@ int
lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq,
uint32_t timeout)
{
- DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_q);
- MAILBOX_t *mb = NULL;
+ struct completion mbox_done;
int retval;
unsigned long flag;
- /* The caller might set context1 for extended buffer */
- if (pmboxq->context1)
- mb = (MAILBOX_t *)pmboxq->context1;
-
pmboxq->mbox_flag &= ~LPFC_MBX_WAKE;
/* setup wake call as IOCB callback */
pmboxq->mbox_cmpl = lpfc_sli_wake_mbox_wait;
- /* setup context field to pass wait_queue pointer to wake function */
- pmboxq->context1 = &done_q;
+ /* setup context3 field to pass wait_queue pointer to wake function */
+ init_completion(&mbox_done);
+ pmboxq->context3 = &mbox_done;
/* now issue the command */
retval = lpfc_sli_issue_mbox(phba, pmboxq, MBX_NOWAIT);
if (retval == MBX_BUSY || retval == MBX_SUCCESS) {
- wait_event_interruptible_timeout(done_q,
- pmboxq->mbox_flag & LPFC_MBX_WAKE,
- msecs_to_jiffies(timeout * 1000));
+ wait_for_completion_timeout(&mbox_done,
+ msecs_to_jiffies(timeout * 1000));
spin_lock_irqsave(&phba->hbalock, flag);
- /* restore the possible extended buffer for free resource */
- pmboxq->context1 = (uint8_t *)mb;
+ pmboxq->context3 = NULL;
/*
* if LPFC_MBX_WAKE flag is set the mailbox is completed
* else do not free the resources.
@@ -11478,11 +11696,7 @@ lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq,
pmboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
}
spin_unlock_irqrestore(&phba->hbalock, flag);
- } else {
- /* restore the possible extended buffer for free resource */
- pmboxq->context1 = (uint8_t *)mb;
}
-
return retval;
}
@@ -11648,6 +11862,7 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba)
}
break;
case LPFC_SLI_INTF_IF_TYPE_2:
+ case LPFC_SLI_INTF_IF_TYPE_6:
if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr,
&portstat_reg.word0) ||
lpfc_readl(phba->sli4_hba.PSMPHRregaddr,
@@ -13112,7 +13327,7 @@ lpfc_sli4_sp_process_cq(struct work_struct *work)
"(x%x), type (%d)\n", cq->queue_id, cq->type);
/* In any case, flash and re-arm the RCQ */
- lpfc_sli4_cq_release(cq, LPFC_QUEUE_REARM);
+ phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM);
/* wake up worker thread if there are works to be done */
if (workposted)
@@ -13230,6 +13445,8 @@ lpfc_sli4_fp_handle_rel_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
if (childwq->queue_id == hba_wqid) {
lpfc_sli4_wq_release(childwq,
bf_get(lpfc_wcqe_r_wqe_index, wcqe));
+ if (childwq->q_flag & HBA_NVMET_WQFULL)
+ lpfc_nvmet_wqfull_process(phba, childwq);
wqid_matched = true;
break;
}
@@ -13542,7 +13759,7 @@ lpfc_sli4_hba_process_cq(struct work_struct *work)
"queue fcpcqid=%d\n", cq->queue_id);
/* In any case, flash and re-arm the CQ */
- lpfc_sli4_cq_release(cq, LPFC_QUEUE_REARM);
+ phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM);
/* wake up worker thread if there are works to be done */
if (workposted)
@@ -13559,7 +13776,7 @@ lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq)
;
/* Clear and re-arm the EQ */
- lpfc_sli4_eq_release(eq, LPFC_QUEUE_REARM);
+ phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM);
}
@@ -13707,7 +13924,7 @@ lpfc_sli4_fof_intr_handler(int irq, void *dev_id)
}
}
/* Always clear and re-arm the fast-path EQ */
- lpfc_sli4_eq_release(eq, LPFC_QUEUE_REARM);
+ phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM);
return IRQ_HANDLED;
}
@@ -13765,7 +13982,7 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
if (lpfc_fcp_look_ahead) {
if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use))
- lpfc_sli4_eq_clr_intr(fpeq);
+ phba->sli4_hba.sli4_eq_clr_intr(fpeq);
else {
atomic_inc(&hba_eq_hdl->hba_eq_in_use);
return IRQ_NONE;
@@ -13800,7 +14017,7 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
fpeq->EQ_max_eqe = ecount;
/* Always clear and re-arm the fast-path EQ */
- lpfc_sli4_eq_release(fpeq, LPFC_QUEUE_REARM);
+ phba->sli4_hba.sli4_eq_release(fpeq, LPFC_QUEUE_REARM);
if (unlikely(ecount == 0)) {
fpeq->EQ_no_entry++;
@@ -13948,6 +14165,7 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
INIT_LIST_HEAD(&queue->list);
INIT_LIST_HEAD(&queue->wq_list);
+ INIT_LIST_HEAD(&queue->wqfull_list);
INIT_LIST_HEAD(&queue->page_list);
INIT_LIST_HEAD(&queue->child_list);
@@ -14173,11 +14391,21 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax)
LPFC_MBOX_OPCODE_EQ_CREATE,
length, LPFC_SLI4_MBX_EMBED);
eq_create = &mbox->u.mqe.un.eq_create;
+ shdr = (union lpfc_sli4_cfg_shdr *) &eq_create->header.cfg_shdr;
bf_set(lpfc_mbx_eq_create_num_pages, &eq_create->u.request,
eq->page_count);
bf_set(lpfc_eq_context_size, &eq_create->u.request.context,
LPFC_EQE_SIZE);
bf_set(lpfc_eq_context_valid, &eq_create->u.request.context, 1);
+
+ /* Use version 2 of CREATE_EQ if eqav is set */
+ if (phba->sli4_hba.pc_sli4_params.eqav) {
+ bf_set(lpfc_mbox_hdr_version, &shdr->request,
+ LPFC_Q_CREATE_VERSION_2);
+ bf_set(lpfc_eq_context_autovalid, &eq_create->u.request.context,
+ phba->sli4_hba.pc_sli4_params.eqav);
+ }
+
/* don't setup delay multiplier using EQ_CREATE */
dmult = 0;
bf_set(lpfc_eq_context_delay_multi, &eq_create->u.request.context,
@@ -14222,7 +14450,6 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax)
mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
mbox->context1 = NULL;
rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL);
- shdr = (union lpfc_sli4_cfg_shdr *) &eq_create->header.cfg_shdr;
shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
if (shdr_status || shdr_add_status || rc) {
@@ -14305,6 +14532,8 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq,
(cq->page_size / SLI4_PAGE_SIZE));
bf_set(lpfc_cq_eq_id_2, &cq_create->u.request.context,
eq->queue_id);
+ bf_set(lpfc_cq_context_autovalid, &cq_create->u.request.context,
+ phba->sli4_hba.pc_sli4_params.cqav);
} else {
bf_set(lpfc_cq_eq_id, &cq_create->u.request.context,
eq->queue_id);
@@ -14476,6 +14705,9 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp,
&cq_set->u.request, 0);
bf_set(lpfc_mbx_cq_create_set_num_cq,
&cq_set->u.request, numcq);
+ bf_set(lpfc_mbx_cq_create_set_autovalid,
+ &cq_set->u.request,
+ phba->sli4_hba.pc_sli4_params.cqav);
switch (cq->entry_count) {
case 2048:
case 4096:
@@ -14881,6 +15113,9 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
void __iomem *bar_memmap_p;
uint32_t db_offset;
uint16_t pci_barset;
+ uint8_t dpp_barset;
+ uint32_t dpp_offset;
+ unsigned long pg_addr;
uint8_t wq_create_version;
/* sanity check on queue memory */
@@ -14908,43 +15143,19 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
bf_set(lpfc_mbox_hdr_version, &shdr->request,
phba->sli4_hba.pc_sli4_params.wqv);
+ if ((phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT) ||
+ (wq->page_size > SLI4_PAGE_SIZE))
+ wq_create_version = LPFC_Q_CREATE_VERSION_1;
+ else
+ wq_create_version = LPFC_Q_CREATE_VERSION_0;
+
+
if (phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT)
wq_create_version = LPFC_Q_CREATE_VERSION_1;
else
wq_create_version = LPFC_Q_CREATE_VERSION_0;
switch (wq_create_version) {
- case LPFC_Q_CREATE_VERSION_0:
- switch (wq->entry_size) {
- default:
- case 64:
- /* Nothing to do, version 0 ONLY supports 64 byte */
- page = wq_create->u.request.page;
- break;
- case 128:
- if (!(phba->sli4_hba.pc_sli4_params.wqsize &
- LPFC_WQ_SZ128_SUPPORT)) {
- status = -ERANGE;
- goto out;
- }
- /* If we get here the HBA MUST also support V1 and
- * we MUST use it
- */
- bf_set(lpfc_mbox_hdr_version, &shdr->request,
- LPFC_Q_CREATE_VERSION_1);
-
- bf_set(lpfc_mbx_wq_create_wqe_count,
- &wq_create->u.request_1, wq->entry_count);
- bf_set(lpfc_mbx_wq_create_wqe_size,
- &wq_create->u.request_1,
- LPFC_WQ_WQE_SIZE_128);
- bf_set(lpfc_mbx_wq_create_page_size,
- &wq_create->u.request_1,
- LPFC_WQ_PAGE_SIZE_4096);
- page = wq_create->u.request_1.page;
- break;
- }
- break;
case LPFC_Q_CREATE_VERSION_1:
bf_set(lpfc_mbx_wq_create_wqe_count, &wq_create->u.request_1,
wq->entry_count);
@@ -14959,24 +15170,21 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
LPFC_WQ_WQE_SIZE_64);
break;
case 128:
- if (!(phba->sli4_hba.pc_sli4_params.wqsize &
- LPFC_WQ_SZ128_SUPPORT)) {
- status = -ERANGE;
- goto out;
- }
bf_set(lpfc_mbx_wq_create_wqe_size,
&wq_create->u.request_1,
LPFC_WQ_WQE_SIZE_128);
break;
}
+ /* Request DPP by default */
+ bf_set(lpfc_mbx_wq_create_dpp_req, &wq_create->u.request_1, 1);
bf_set(lpfc_mbx_wq_create_page_size,
&wq_create->u.request_1,
(wq->page_size / SLI4_PAGE_SIZE));
page = wq_create->u.request_1.page;
break;
default:
- status = -ERANGE;
- goto out;
+ page = wq_create->u.request.page;
+ break;
}
list_for_each_entry(dmabuf, &wq->page_list, list) {
@@ -15000,52 +15208,120 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,
status = -ENXIO;
goto out;
}
- wq->queue_id = bf_get(lpfc_mbx_wq_create_q_id, &wq_create->u.response);
+
+ if (wq_create_version == LPFC_Q_CREATE_VERSION_0)
+ wq->queue_id = bf_get(lpfc_mbx_wq_create_q_id,
+ &wq_create->u.response);
+ else
+ wq->queue_id = bf_get(lpfc_mbx_wq_create_v1_q_id,
+ &wq_create->u.response_1);
+
if (wq->queue_id == 0xFFFF) {
status = -ENXIO;
goto out;
}
- if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
- wq->db_format = bf_get(lpfc_mbx_wq_create_db_format,
- &wq_create->u.response);
- if ((wq->db_format != LPFC_DB_LIST_FORMAT) &&
- (wq->db_format != LPFC_DB_RING_FORMAT)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3265 WQ[%d] doorbell format not "
- "supported: x%x\n", wq->queue_id,
- wq->db_format);
- status = -EINVAL;
- goto out;
- }
- pci_barset = bf_get(lpfc_mbx_wq_create_bar_set,
- &wq_create->u.response);
- bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset);
- if (!bar_memmap_p) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3263 WQ[%d] failed to memmap pci "
- "barset:x%x\n", wq->queue_id,
- pci_barset);
- status = -ENOMEM;
- goto out;
- }
- db_offset = wq_create->u.response.doorbell_offset;
- if ((db_offset != LPFC_ULP0_WQ_DOORBELL) &&
- (db_offset != LPFC_ULP1_WQ_DOORBELL)) {
- lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
- "3252 WQ[%d] doorbell offset not "
- "supported: x%x\n", wq->queue_id,
- db_offset);
- status = -EINVAL;
- goto out;
- }
- wq->db_regaddr = bar_memmap_p + db_offset;
- lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
- "3264 WQ[%d]: barset:x%x, offset:x%x, "
- "format:x%x\n", wq->queue_id, pci_barset,
- db_offset, wq->db_format);
+
+ wq->db_format = LPFC_DB_LIST_FORMAT;
+ if (wq_create_version == LPFC_Q_CREATE_VERSION_0) {
+ if (phba->sli4_hba.fw_func_mode & LPFC_DUA_MODE) {
+ wq->db_format = bf_get(lpfc_mbx_wq_create_db_format,
+ &wq_create->u.response);
+ if ((wq->db_format != LPFC_DB_LIST_FORMAT) &&
+ (wq->db_format != LPFC_DB_RING_FORMAT)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3265 WQ[%d] doorbell format "
+ "not supported: x%x\n",
+ wq->queue_id, wq->db_format);
+ status = -EINVAL;
+ goto out;
+ }
+ pci_barset = bf_get(lpfc_mbx_wq_create_bar_set,
+ &wq_create->u.response);
+ bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
+ pci_barset);
+ if (!bar_memmap_p) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3263 WQ[%d] failed to memmap "
+ "pci barset:x%x\n",
+ wq->queue_id, pci_barset);
+ status = -ENOMEM;
+ goto out;
+ }
+ db_offset = wq_create->u.response.doorbell_offset;
+ if ((db_offset != LPFC_ULP0_WQ_DOORBELL) &&
+ (db_offset != LPFC_ULP1_WQ_DOORBELL)) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3252 WQ[%d] doorbell offset "
+ "not supported: x%x\n",
+ wq->queue_id, db_offset);
+ status = -EINVAL;
+ goto out;
+ }
+ wq->db_regaddr = bar_memmap_p + db_offset;
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "3264 WQ[%d]: barset:x%x, offset:x%x, "
+ "format:x%x\n", wq->queue_id,
+ pci_barset, db_offset, wq->db_format);
+ } else
+ wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
} else {
- wq->db_format = LPFC_DB_LIST_FORMAT;
- wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
+ /* Check if DPP was honored by the firmware */
+ wq->dpp_enable = bf_get(lpfc_mbx_wq_create_dpp_rsp,
+ &wq_create->u.response_1);
+ if (wq->dpp_enable) {
+ pci_barset = bf_get(lpfc_mbx_wq_create_v1_bar_set,
+ &wq_create->u.response_1);
+ bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
+ pci_barset);
+ if (!bar_memmap_p) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3267 WQ[%d] failed to memmap "
+ "pci barset:x%x\n",
+ wq->queue_id, pci_barset);
+ status = -ENOMEM;
+ goto out;
+ }
+ db_offset = wq_create->u.response_1.doorbell_offset;
+ wq->db_regaddr = bar_memmap_p + db_offset;
+ wq->dpp_id = bf_get(lpfc_mbx_wq_create_dpp_id,
+ &wq_create->u.response_1);
+ dpp_barset = bf_get(lpfc_mbx_wq_create_dpp_bar,
+ &wq_create->u.response_1);
+ bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba,
+ dpp_barset);
+ if (!bar_memmap_p) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3268 WQ[%d] failed to memmap "
+ "pci barset:x%x\n",
+ wq->queue_id, dpp_barset);
+ status = -ENOMEM;
+ goto out;
+ }
+ dpp_offset = wq_create->u.response_1.dpp_offset;
+ wq->dpp_regaddr = bar_memmap_p + dpp_offset;
+ lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+ "3271 WQ[%d]: barset:x%x, offset:x%x, "
+ "dpp_id:x%x dpp_barset:x%x "
+ "dpp_offset:x%x\n",
+ wq->queue_id, pci_barset, db_offset,
+ wq->dpp_id, dpp_barset, dpp_offset);
+
+ /* Enable combined writes for DPP aperture */
+ pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK;
+#ifdef CONFIG_X86
+ rc = set_memory_wc(pg_addr, 1);
+ if (rc) {
+ lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+ "3272 Cannot setup Combined "
+ "Write on WQ[%d] - disable DPP\n",
+ wq->queue_id);
+ phba->cfg_enable_dpp = 0;
+ }
+#else
+ phba->cfg_enable_dpp = 0;
+#endif
+ } else
+ wq->db_regaddr = phba->sli4_hba.WQDBregaddr;
}
wq->pring = kzalloc(sizeof(struct lpfc_sli_ring), GFP_KERNEL);
if (wq->pring == NULL) {
@@ -18616,6 +18892,7 @@ lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list,
"status x%x add_status x%x, mbx status x%x\n",
shdr_status, shdr_add_status, rc);
rc = -ENXIO;
+ *offset = shdr_add_status;
} else
*offset += wr_object->u.response.actual_write_length;
return rc;
@@ -18753,8 +19030,7 @@ lpfc_drain_txq(struct lpfc_hba *phba)
unsigned long iflags = 0;
char *fail_msg = NULL;
struct lpfc_sglq *sglq;
- union lpfc_wqe128 wqe128;
- union lpfc_wqe *wqe = (union lpfc_wqe *) &wqe128;
+ union lpfc_wqe128 wqe;
uint32_t txq_cnt = 0;
pring = lpfc_phba_elsring(phba);
@@ -18797,9 +19073,9 @@ lpfc_drain_txq(struct lpfc_hba *phba)
piocbq->sli4_xritag = sglq->sli4_xritag;
if (NO_XRI == lpfc_sli4_bpl2sgl(phba, piocbq, sglq))
fail_msg = "to convert bpl to sgl";
- else if (lpfc_sli4_iocb2wqe(phba, piocbq, wqe))
+ else if (lpfc_sli4_iocb2wqe(phba, piocbq, &wqe))
fail_msg = "to convert iocb to wqe";
- else if (lpfc_sli4_wq_put(phba->sli4_hba.els_wq, wqe))
+ else if (lpfc_sli4_wq_put(phba->sli4_hba.els_wq, &wqe))
fail_msg = " - Wq is full";
else
lpfc_sli_ringtxcmpl_put(phba, pring, piocbq);
@@ -18849,7 +19125,7 @@ lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq,
struct ulp_bde64 bde;
struct sli4_sge *sgl = NULL;
struct lpfc_dmabuf *dmabuf;
- union lpfc_wqe *wqe;
+ union lpfc_wqe128 *wqe;
int numBdes = 0;
int i = 0;
uint32_t offset = 0; /* accumulated offset in the sg request list */
@@ -18958,7 +19234,7 @@ int
lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number,
struct lpfc_iocbq *pwqe)
{
- union lpfc_wqe *wqe = &pwqe->wqe;
+ union lpfc_wqe128 *wqe = &pwqe->wqe;
struct lpfc_nvmet_rcv_ctx *ctxp;
struct lpfc_queue *wq;
struct lpfc_sglq *sglq;
generated by cgit v1.2.3-55-g7522 (git 2.39.0) at 2024-07-12 13:01:41 +0200