diff options
Diffstat (limited to 'drivers/staging/ccree/ssi_request_mgr.c')
-rw-r--r-- | drivers/staging/ccree/ssi_request_mgr.c | 393 |
1 files changed, 140 insertions, 253 deletions
diff --git a/drivers/staging/ccree/ssi_request_mgr.c b/drivers/staging/ccree/ssi_request_mgr.c index 8611adf3bb2e..46d9396f9ff9 100644 --- a/drivers/staging/ccree/ssi_request_mgr.c +++ b/drivers/staging/ccree/ssi_request_mgr.c @@ -1,15 +1,15 @@ /* * Copyright (C) 2012-2017 ARM Limited or its affiliates. - * + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. - * + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with this program; if not, see <http://www.gnu.org/licenses/>. */ @@ -35,91 +35,22 @@ #define SSI_MAX_POLL_ITER 10 -#define AXIM_MON_BASE_OFFSET CC_REG_OFFSET(CRY_KERNEL, AXIM_MON_COMP) - -#ifdef CC_CYCLE_COUNT - -#define MONITOR_CNTR_BIT 0 - -/** - * Monitor descriptor. - * Used to measure CC performance. - */ -#define INIT_CC_MONITOR_DESC(desc_p) \ -do { \ - HW_DESC_INIT(desc_p); \ - HW_DESC_SET_DIN_MONITOR_CNTR(desc_p); \ -} while (0) - -/** - * Try adding monitor descriptor BEFORE enqueuing sequence. - */ -#define CC_CYCLE_DESC_HEAD(cc_base_addr, desc_p, lock_p, is_monitored_p) \ -do { \ - if (!test_and_set_bit(MONITOR_CNTR_BIT, (lock_p))) { \ - enqueue_seq((cc_base_addr), (desc_p), 1); \ - *(is_monitored_p) = true; \ - } else { \ - *(is_monitored_p) = false; \ - } \ -} while (0) - -/** - * If CC_CYCLE_DESC_HEAD was successfully added: - * 1. Add memory barrier descriptor to ensure last AXI transaction. - * 2. Add monitor descriptor to sequence tail AFTER enqueuing sequence. - */ -#define CC_CYCLE_DESC_TAIL(cc_base_addr, desc_p, is_monitored) \ -do { \ - if ((is_monitored) == true) { \ - HwDesc_s barrier_desc; \ - HW_DESC_INIT(&barrier_desc); \ - HW_DESC_SET_DIN_NO_DMA(&barrier_desc, 0, 0xfffff0); \ - HW_DESC_SET_DOUT_NO_DMA(&barrier_desc, 0, 0, 1); \ - enqueue_seq((cc_base_addr), &barrier_desc, 1); \ - enqueue_seq((cc_base_addr), (desc_p), 1); \ - } \ -} while (0) - -/** - * Try reading CC monitor counter value upon sequence complete. - * Can only succeed if the lock_p is taken by the owner of the given request. - */ -#define END_CC_MONITOR_COUNT(cc_base_addr, stat_op_type, stat_phase, monitor_null_cycles, lock_p, is_monitored) \ -do { \ - uint32_t elapsed_cycles; \ - if ((is_monitored) == true) { \ - elapsed_cycles = READ_REGISTER((cc_base_addr) + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_MEASURE_CNTR)); \ - clear_bit(MONITOR_CNTR_BIT, (lock_p)); \ - if (elapsed_cycles > 0) \ - update_cc_stat(stat_op_type, stat_phase, (elapsed_cycles - monitor_null_cycles)); \ - } \ -} while (0) - -#else /*CC_CYCLE_COUNT*/ - -#define INIT_CC_MONITOR_DESC(desc_p) do { } while (0) -#define CC_CYCLE_DESC_HEAD(cc_base_addr, desc_p, lock_p, is_monitored_p) do { } while (0) -#define CC_CYCLE_DESC_TAIL(cc_base_addr, desc_p, is_monitored) do { } while (0) -#define END_CC_MONITOR_COUNT(cc_base_addr, stat_op_type, stat_phase, monitor_null_cycles, lock_p, is_monitored) do { } while (0) -#endif /*CC_CYCLE_COUNT*/ - - struct ssi_request_mgr_handle { /* Request manager resources */ unsigned int hw_queue_size; /* HW capability */ unsigned int min_free_hw_slots; unsigned int max_used_sw_slots; struct ssi_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE]; - uint32_t req_queue_head; - uint32_t req_queue_tail; - uint32_t axi_completed; - uint32_t q_free_slots; + u32 req_queue_head; + u32 req_queue_tail; + u32 axi_completed; + u32 q_free_slots; spinlock_t hw_lock; - HwDesc_s compl_desc; - uint8_t *dummy_comp_buff; + struct cc_hw_desc compl_desc; + u8 *dummy_comp_buff; dma_addr_t dummy_comp_buff_dma; - HwDesc_s monitor_desc; + struct cc_hw_desc monitor_desc; + volatile unsigned long monitor_lock; #ifdef COMP_IN_WQ struct workqueue_struct *workq; @@ -127,7 +58,7 @@ struct ssi_request_mgr_handle { #else struct tasklet_struct comptask; #endif -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) bool is_runtime_suspended; #endif }; @@ -141,18 +72,17 @@ void request_mgr_fini(struct ssi_drvdata *drvdata) { struct ssi_request_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; - if (req_mgr_h == NULL) + if (!req_mgr_h) return; /* Not allocated */ if (req_mgr_h->dummy_comp_buff_dma != 0) { - SSI_RESTORE_DMA_ADDR_TO_48BIT(req_mgr_h->dummy_comp_buff_dma); dma_free_coherent(&drvdata->plat_dev->dev, - sizeof(uint32_t), req_mgr_h->dummy_comp_buff, + sizeof(u32), req_mgr_h->dummy_comp_buff, req_mgr_h->dummy_comp_buff_dma); } SSI_LOG_DEBUG("max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size - - req_mgr_h->min_free_hw_slots) ); + req_mgr_h->min_free_hw_slots)); SSI_LOG_DEBUG("max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); #ifdef COMP_IN_WQ @@ -169,15 +99,11 @@ void request_mgr_fini(struct ssi_drvdata *drvdata) int request_mgr_init(struct ssi_drvdata *drvdata) { -#ifdef CC_CYCLE_COUNT - HwDesc_s monitor_desc[2]; - struct ssi_crypto_req monitor_req = {0}; -#endif struct ssi_request_mgr_handle *req_mgr_h; int rc = 0; - req_mgr_h = kzalloc(sizeof(struct ssi_request_mgr_handle),GFP_KERNEL); - if (req_mgr_h == NULL) { + req_mgr_h = kzalloc(sizeof(struct ssi_request_mgr_handle), GFP_KERNEL); + if (!req_mgr_h) { rc = -ENOMEM; goto req_mgr_init_err; } @@ -188,7 +114,7 @@ int request_mgr_init(struct ssi_drvdata *drvdata) #ifdef COMP_IN_WQ SSI_LOG_DEBUG("Initializing completion workqueue\n"); req_mgr_h->workq = create_singlethread_workqueue("arm_cc7x_wq"); - if (unlikely(req_mgr_h->workq == NULL)) { + if (unlikely(!req_mgr_h->workq)) { SSI_LOG_ERR("Failed creating work queue\n"); rc = -ENOMEM; goto req_mgr_init_err; @@ -210,45 +136,23 @@ int request_mgr_init(struct ssi_drvdata *drvdata) req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size; req_mgr_h->max_used_sw_slots = 0; - /* Allocate DMA word for "dummy" completion descriptor use */ req_mgr_h->dummy_comp_buff = dma_alloc_coherent(&drvdata->plat_dev->dev, - sizeof(uint32_t), &req_mgr_h->dummy_comp_buff_dma, GFP_KERNEL); + sizeof(u32), &req_mgr_h->dummy_comp_buff_dma, GFP_KERNEL); if (!req_mgr_h->dummy_comp_buff) { SSI_LOG_ERR("Not enough memory to allocate DMA (%zu) dropped " - "buffer\n", sizeof(uint32_t)); + "buffer\n", sizeof(u32)); rc = -ENOMEM; goto req_mgr_init_err; } - SSI_UPDATE_DMA_ADDR_TO_48BIT(req_mgr_h->dummy_comp_buff_dma, - sizeof(uint32_t)); /* Init. "dummy" completion descriptor */ - HW_DESC_INIT(&req_mgr_h->compl_desc); - HW_DESC_SET_DIN_CONST(&req_mgr_h->compl_desc, 0, sizeof(uint32_t)); - HW_DESC_SET_DOUT_DLLI(&req_mgr_h->compl_desc, - req_mgr_h->dummy_comp_buff_dma, - sizeof(uint32_t), NS_BIT, 1); - HW_DESC_SET_FLOW_MODE(&req_mgr_h->compl_desc, BYPASS); - HW_DESC_SET_QUEUE_LAST_IND(&req_mgr_h->compl_desc); - -#ifdef CC_CYCLE_COUNT - /* For CC-HW cycle performance trace */ - INIT_CC_MONITOR_DESC(&req_mgr_h->monitor_desc); - set_bit(MONITOR_CNTR_BIT, &req_mgr_h->monitor_lock); - monitor_desc[0] = req_mgr_h->monitor_desc; - monitor_desc[1] = req_mgr_h->monitor_desc; - - rc = send_request(drvdata, &monitor_req, monitor_desc, 2, 0); - if (unlikely(rc != 0)) - goto req_mgr_init_err; - - drvdata->monitor_null_cycles = READ_REGISTER(drvdata->cc_base + - CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_MEASURE_CNTR)); - SSI_LOG_ERR("Calibration time=0x%08x\n", drvdata->monitor_null_cycles); - - clear_bit(MONITOR_CNTR_BIT, &req_mgr_h->monitor_lock); -#endif + hw_desc_init(&req_mgr_h->compl_desc); + set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32)); + set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma, + sizeof(u32), NS_BIT, 1); + set_flow_mode(&req_mgr_h->compl_desc, BYPASS); + set_queue_last_ind(&req_mgr_h->compl_desc); return 0; @@ -259,18 +163,18 @@ req_mgr_init_err: static inline void enqueue_seq( void __iomem *cc_base, - HwDesc_s seq[], unsigned int seq_len) + struct cc_hw_desc seq[], unsigned int seq_len) { int i; for (i = 0; i < seq_len; i++) { - writel_relaxed(seq[i].word[0], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); - writel_relaxed(seq[i].word[1], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); - writel_relaxed(seq[i].word[2], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); - writel_relaxed(seq[i].word[3], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); - writel_relaxed(seq[i].word[4], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); + writel_relaxed(seq[i].word[0], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); + writel_relaxed(seq[i].word[1], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); + writel_relaxed(seq[i].word[2], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); + writel_relaxed(seq[i].word[3], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); + writel_relaxed(seq[i].word[4], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); wmb(); - writel_relaxed(seq[i].word[5], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); + writel_relaxed(seq[i].word[5], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0))); #ifdef DX_DUMP_DESCS SSI_LOG_DEBUG("desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", i, seq[i].word[0], seq[i].word[1], seq[i].word[2], seq[i].word[3], seq[i].word[4], seq[i].word[5]); @@ -279,62 +183,63 @@ static inline void enqueue_seq( } /*! - * Completion will take place if and only if user requested completion - * by setting "is_dout = 0" in send_request(). - * - * \param dev + * Completion will take place if and only if user requested completion + * by setting "is_dout = 0" in send_request(). + * + * \param dev * \param dx_compl_h The completion event to signal */ static void request_mgr_complete(struct device *dev, void *dx_compl_h, void __iomem *cc_base) { struct completion *this_compl = dx_compl_h; + complete(this_compl); } - static inline int request_mgr_queues_status_check( struct ssi_request_mgr_handle *req_mgr_h, void __iomem *cc_base, unsigned int total_seq_len) { unsigned long poll_queue; - - /* SW queue is checked only once as it will not - be chaned during the poll becasue the spinlock_bh - is held by the thread */ + + /* SW queue is checked only once as it will not + * be chaned during the poll becasue the spinlock_bh + * is held by the thread + */ if (unlikely(((req_mgr_h->req_queue_head + 1) & - (MAX_REQUEST_QUEUE_SIZE - 1)) == + (MAX_REQUEST_QUEUE_SIZE - 1)) == req_mgr_h->req_queue_tail)) { - SSI_LOG_ERR("SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", + SSI_LOG_ERR("SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE); return -EBUSY; } - if ((likely(req_mgr_h->q_free_slots >= total_seq_len)) ) { + if ((likely(req_mgr_h->q_free_slots >= total_seq_len))) return 0; - } + /* Wait for space in HW queue. Poll constant num of iterations. */ - for (poll_queue =0; poll_queue < SSI_MAX_POLL_ITER ; poll_queue ++) { - req_mgr_h->q_free_slots = + for (poll_queue = 0; poll_queue < SSI_MAX_POLL_ITER ; poll_queue++) { + req_mgr_h->q_free_slots = CC_HAL_READ_REGISTER( CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_CONTENT)); - if (unlikely(req_mgr_h->q_free_slots < + if (unlikely(req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)) { req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots; } - if (likely (req_mgr_h->q_free_slots >= total_seq_len)) { + if (likely(req_mgr_h->q_free_slots >= total_seq_len)) { /* If there is enough place return */ return 0; } - SSI_LOG_DEBUG("HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", + SSI_LOG_DEBUG("HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", req_mgr_h->q_free_slots, total_seq_len); } /* No room in the HW queue try again later */ SSI_LOG_DEBUG("HW FIFO full, timeout. req_queue_head=%d " - "sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", + "sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE, req_mgr_h->q_free_slots, @@ -344,38 +249,37 @@ static inline int request_mgr_queues_status_check( /*! * Enqueue caller request to crypto hardware. - * - * \param drvdata + * + * \param drvdata * \param ssi_req The request to enqueue * \param desc The crypto sequence * \param len The crypto sequence length - * \param is_dout If "true": completion is handled by the caller - * If "false": this function adds a dummy descriptor completion - * and waits upon completion signal. - * + * \param is_dout If "true": completion is handled by the caller + * If "false": this function adds a dummy descriptor completion + * and waits upon completion signal. + * * \return int Returns -EINPROGRESS if "is_dout=true"; "0" if "is_dout=false" */ int send_request( struct ssi_drvdata *drvdata, struct ssi_crypto_req *ssi_req, - HwDesc_s *desc, unsigned int len, bool is_dout) + struct cc_hw_desc *desc, unsigned int len, bool is_dout) { void __iomem *cc_base = drvdata->cc_base; struct ssi_request_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; unsigned int used_sw_slots; unsigned int iv_seq_len = 0; unsigned int total_seq_len = len; /*initial sequence length*/ - HwDesc_s iv_seq[SSI_IVPOOL_SEQ_LEN]; + struct cc_hw_desc iv_seq[SSI_IVPOOL_SEQ_LEN]; int rc; unsigned int max_required_seq_len = (total_seq_len + ((ssi_req->ivgen_dma_addr_len == 0) ? 0 : - SSI_IVPOOL_SEQ_LEN ) + - ((is_dout == 0 )? 1 : 0)); - DECL_CYCLE_COUNT_RESOURCES; + SSI_IVPOOL_SEQ_LEN) + + ((is_dout == 0) ? 1 : 0)); -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) rc = ssi_power_mgr_runtime_get(&drvdata->plat_dev->dev); if (rc != 0) { - SSI_LOG_ERR("ssi_power_mgr_runtime_get returned %x\n",rc); + SSI_LOG_ERR("ssi_power_mgr_runtime_get returned %x\n", rc); return rc; } #endif @@ -384,21 +288,23 @@ int send_request( spin_lock_bh(&req_mgr_h->hw_lock); /* Check if there is enough place in the SW/HW queues - in case iv gen add the max size and in case of no dout add 1 - for the internal completion descriptor */ + * in case iv gen add the max size and in case of no dout add 1 + * for the internal completion descriptor + */ rc = request_mgr_queues_status_check(req_mgr_h, cc_base, max_required_seq_len); - if (likely(rc == 0 )) + if (likely(rc == 0)) /* There is enough place in the queue */ break; /* something wrong release the spinlock*/ spin_unlock_bh(&req_mgr_h->hw_lock); if (rc != -EAGAIN) { - /* Any error other than HW queue full - (SW queue is full) */ -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) + /* Any error other than HW queue full + * (SW queue is full) + */ +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) ssi_power_mgr_runtime_put_suspend(&drvdata->plat_dev->dev); #endif return rc; @@ -409,7 +315,8 @@ int send_request( } while (1); /* Additional completion descriptor is needed incase caller did not - enabled any DLLI/MLLI DOUT bit in the given sequence */ + * enabled any DLLI/MLLI DOUT bit in the given sequence + */ if (!is_dout) { init_completion(&ssi_req->seq_compl); ssi_req->user_cb = request_mgr_complete; @@ -432,7 +339,7 @@ int send_request( if (unlikely(rc != 0)) { SSI_LOG_ERR("Failed to generate IV (rc=%d)\n", rc); spin_unlock_bh(&req_mgr_h->hw_lock); -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) ssi_power_mgr_runtime_put_suspend(&drvdata->plat_dev->dev); #endif return rc; @@ -440,18 +347,13 @@ int send_request( total_seq_len += iv_seq_len; } - - used_sw_slots = ((req_mgr_h->req_queue_head - req_mgr_h->req_queue_tail) & (MAX_REQUEST_QUEUE_SIZE-1)); - if (unlikely(used_sw_slots > req_mgr_h->max_used_sw_slots)) { + + used_sw_slots = ((req_mgr_h->req_queue_head - req_mgr_h->req_queue_tail) & (MAX_REQUEST_QUEUE_SIZE - 1)); + if (unlikely(used_sw_slots > req_mgr_h->max_used_sw_slots)) req_mgr_h->max_used_sw_slots = used_sw_slots; - } - - CC_CYCLE_DESC_HEAD(cc_base, &req_mgr_h->monitor_desc, - &req_mgr_h->monitor_lock, &ssi_req->is_monitored_p); /* Enqueue request - must be locked with HW lock*/ req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *ssi_req; - START_CYCLE_COUNT_AT(req_mgr_h->req_queue[req_mgr_h->req_queue_head].submit_cycle); req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); /* TODO: Use circ_buf.h ? */ @@ -462,13 +364,9 @@ int send_request( #endif /* STAT_PHASE_4: Push sequence */ - START_CYCLE_COUNT(); enqueue_seq(cc_base, iv_seq, iv_seq_len); enqueue_seq(cc_base, desc, len); enqueue_seq(cc_base, &req_mgr_h->compl_desc, (is_dout ? 0 : 1)); - END_CYCLE_COUNT(ssi_req->op_type, STAT_PHASE_4); - - CC_CYCLE_DESC_TAIL(cc_base, &req_mgr_h->monitor_desc, ssi_req->is_monitored_p); if (unlikely(req_mgr_h->q_free_slots < total_seq_len)) { /*This means that there was a problem with the resume*/ @@ -481,28 +379,29 @@ int send_request( if (!is_dout) { /* Wait upon sequence completion. - * Return "0" -Operation done successfully. */ - return wait_for_completion_interruptible(&ssi_req->seq_compl); + * Return "0" -Operation done successfully. + */ + wait_for_completion(&ssi_req->seq_compl); + return 0; } else { /* Operation still in process */ return -EINPROGRESS; } } - /*! * Enqueue caller request to crypto hardware during init process. * assume this function is not called in middle of a flow, * since we set QUEUE_LAST_IND flag in the last descriptor. - * - * \param drvdata + * + * \param drvdata * \param desc The crypto sequence * \param len The crypto sequence length - * + * * \return int Returns "0" upon success */ int send_request_init( - struct ssi_drvdata *drvdata, HwDesc_s *desc, unsigned int len) + struct ssi_drvdata *drvdata, struct cc_hw_desc *desc, unsigned int len) { void __iomem *cc_base = drvdata->cc_base; struct ssi_request_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; @@ -511,10 +410,10 @@ int send_request_init( /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. */ rc = request_mgr_queues_status_check(req_mgr_h, cc_base, total_seq_len); - if (unlikely(rc != 0 )) { + if (unlikely(rc != 0)) return rc; - } - HW_DESC_SET_QUEUE_LAST_IND(&desc[len-1]); + + set_queue_last_ind(&desc[(len - 1)]); enqueue_seq(cc_base, desc, len); @@ -526,10 +425,9 @@ int send_request_init( return 0; } - void complete_request(struct ssi_drvdata *drvdata) { - struct ssi_request_mgr_handle *request_mgr_handle = + struct ssi_request_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; #ifdef COMP_IN_WQ queue_delayed_work(request_mgr_handle->workq, &request_mgr_handle->compwork, 0); @@ -552,14 +450,13 @@ static void proc_completions(struct ssi_drvdata *drvdata) { struct ssi_crypto_req *ssi_req; struct platform_device *plat_dev = drvdata->plat_dev; - struct ssi_request_mgr_handle * request_mgr_handle = + struct ssi_request_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) int rc = 0; #endif - DECL_CYCLE_COUNT_RESOURCES; - while(request_mgr_handle->axi_completed) { + while (request_mgr_handle->axi_completed) { request_mgr_handle->axi_completed--; /* Dequeue request */ @@ -569,9 +466,6 @@ static void proc_completions(struct ssi_drvdata *drvdata) } ssi_req = &request_mgr_handle->req_queue[request_mgr_handle->req_queue_tail]; - END_CYCLE_COUNT_AT(ssi_req->submit_cycle, ssi_req->op_type, STAT_PHASE_5); /* Seq. Comp. */ - END_CC_MONITOR_COUNT(drvdata->cc_base, ssi_req->op_type, STAT_PHASE_6, - drvdata->monitor_null_cycles, &request_mgr_handle->monitor_lock, ssi_req->is_monitored_p); #ifdef FLUSH_CACHE_ALL flush_cache_all(); @@ -580,116 +474,109 @@ static void proc_completions(struct ssi_drvdata *drvdata) #ifdef COMPLETION_DELAY /* Delay */ { - uint32_t axi_err; + u32 axi_err; int i; + SSI_LOG_INFO("Delay\n"); - for (i=0;i<1000000;i++) { + for (i = 0; i < 1000000; i++) axi_err = READ_REGISTER(drvdata->cc_base + CC_REG_OFFSET(CRY_KERNEL, AXIM_MON_ERR)); - } } #endif /* COMPLETION_DELAY */ - if (likely(ssi_req->user_cb != NULL)) { - START_CYCLE_COUNT(); + if (likely(ssi_req->user_cb)) ssi_req->user_cb(&plat_dev->dev, ssi_req->user_arg, drvdata->cc_base); - END_CYCLE_COUNT(STAT_OP_TYPE_GENERIC, STAT_PHASE_3); - } request_mgr_handle->req_queue_tail = (request_mgr_handle->req_queue_tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); SSI_LOG_DEBUG("Dequeue request tail=%u\n", request_mgr_handle->req_queue_tail); SSI_LOG_DEBUG("Request completed. axi_completed=%d\n", request_mgr_handle->axi_completed); -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) rc = ssi_power_mgr_runtime_put_suspend(&plat_dev->dev); - if (rc != 0) { - SSI_LOG_ERR("Failed to set runtime suspension %d\n",rc); - } + if (rc != 0) + SSI_LOG_ERR("Failed to set runtime suspension %d\n", rc); #endif } } +static inline u32 cc_axi_comp_count(void __iomem *cc_base) +{ + /* The CC_HAL_READ_REGISTER macro implictly requires and uses + * a base MMIO register address variable named cc_base. + */ + return FIELD_GET(AXIM_MON_COMP_VALUE, + CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET)); +} + /* Deferred service handler, run as interrupt-fired tasklet */ static void comp_handler(unsigned long devarg) { struct ssi_drvdata *drvdata = (struct ssi_drvdata *)devarg; void __iomem *cc_base = drvdata->cc_base; - struct ssi_request_mgr_handle * request_mgr_handle = + struct ssi_request_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; - uint32_t irq; - - DECL_CYCLE_COUNT_RESOURCES; - - START_CYCLE_COUNT(); + u32 irq; irq = (drvdata->irq & SSI_COMP_IRQ_MASK); if (irq & SSI_COMP_IRQ_MASK) { /* To avoid the interrupt from firing as we unmask it, we clear it now */ CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_ICR), SSI_COMP_IRQ_MASK); - + /* Avoid race with above clear: Test completion counter once more */ - request_mgr_handle->axi_completed += CC_REG_FLD_GET(CRY_KERNEL, AXIM_MON_COMP, VALUE, - CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET)); - - /* ISR-to-Tasklet latency */ - if (request_mgr_handle->axi_completed) { - /* Only if actually reflects ISR-to-completion-handling latency, i.e., - not duplicate as a result of interrupt after AXIM_MON_ERR clear, before end of loop */ - END_CYCLE_COUNT_AT(drvdata->isr_exit_cycles, STAT_OP_TYPE_GENERIC, STAT_PHASE_1); - } - + request_mgr_handle->axi_completed += + cc_axi_comp_count(cc_base); + while (request_mgr_handle->axi_completed) { do { proc_completions(drvdata); - /* At this point (after proc_completions()), request_mgr_handle->axi_completed is always 0. - The following assignment was changed to = (previously was +=) to conform KW restrictions. */ - request_mgr_handle->axi_completed = CC_REG_FLD_GET(CRY_KERNEL, AXIM_MON_COMP, VALUE, - CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET)); + /* At this point (after proc_completions()), + * request_mgr_handle->axi_completed is 0. + */ + request_mgr_handle->axi_completed = + cc_axi_comp_count(cc_base); } while (request_mgr_handle->axi_completed > 0); - + /* To avoid the interrupt from firing as we unmask it, we clear it now */ CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_ICR), SSI_COMP_IRQ_MASK); - + /* Avoid race with above clear: Test completion counter once more */ - request_mgr_handle->axi_completed += CC_REG_FLD_GET(CRY_KERNEL, AXIM_MON_COMP, VALUE, - CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET)); + request_mgr_handle->axi_completed += + cc_axi_comp_count(cc_base); } - } /* after verifing that there is nothing to do, Unmask AXI completion interrupt */ - CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR), + CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR), CC_HAL_READ_REGISTER( CC_REG_OFFSET(HOST_RGF, HOST_IMR)) & ~irq); - END_CYCLE_COUNT(STAT_OP_TYPE_GENERIC, STAT_PHASE_2); } /* -resume the queue configuration - no need to take the lock as this happens inside -the spin lock protection -*/ -#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP) + * resume the queue configuration - no need to take the lock as this happens inside + * the spin lock protection + */ +#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP) int ssi_request_mgr_runtime_resume_queue(struct ssi_drvdata *drvdata) { - struct ssi_request_mgr_handle * request_mgr_handle = drvdata->request_mgr_handle; + struct ssi_request_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; spin_lock_bh(&request_mgr_handle->hw_lock); request_mgr_handle->is_runtime_suspended = false; spin_unlock_bh(&request_mgr_handle->hw_lock); - return 0 ; + return 0; } /* -suspend the queue configuration. Since it is used for the runtime suspend -only verify that the queue can be suspended. -*/ + * suspend the queue configuration. Since it is used for the runtime suspend + * only verify that the queue can be suspended. + */ int ssi_request_mgr_runtime_suspend_queue(struct ssi_drvdata *drvdata) { - struct ssi_request_mgr_handle * request_mgr_handle = + struct ssi_request_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; - + /* lock the send_request */ spin_lock_bh(&request_mgr_handle->hw_lock); - if (request_mgr_handle->req_queue_head != + if (request_mgr_handle->req_queue_head != request_mgr_handle->req_queue_tail) { spin_unlock_bh(&request_mgr_handle->hw_lock); return -EBUSY; @@ -702,10 +589,10 @@ int ssi_request_mgr_runtime_suspend_queue(struct ssi_drvdata *drvdata) bool ssi_request_mgr_is_queue_runtime_suspend(struct ssi_drvdata *drvdata) { - struct ssi_request_mgr_handle * request_mgr_handle = + struct ssi_request_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; - return request_mgr_handle->is_runtime_suspended; + return request_mgr_handle->is_runtime_suspended; } #endif |