summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--block/bfq-iosched.c143
1 files changed, 23 insertions, 120 deletions
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index b7e3c8653414..30bb8f905733 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -538,8 +538,6 @@ struct bfq_data {
/* bfq_queue in service */
struct bfq_queue *in_service_queue;
- /* bfq_io_cq (bic) associated with the @in_service_queue */
- struct bfq_io_cq *in_service_bic;
/* on-disk position of the last served request */
sector_t last_position;
@@ -704,15 +702,6 @@ struct bfq_data {
struct bfq_io_cq *bio_bic;
/* bfqq associated with the task issuing current bio for merging */
struct bfq_queue *bio_bfqq;
-
- /*
- * io context to put right after bfqd->lock is released. This
- * filed is used to perform put_io_context, when needed, to
- * after the scheduler lock has been released, and thus
- * prevent an ioc->lock from being possibly taken while the
- * scheduler lock is being held.
- */
- struct io_context *ioc_to_put;
};
enum bfqq_state_flags {
@@ -1148,34 +1137,6 @@ static void bfq_schedule_dispatch(struct bfq_data *bfqd)
}
}
-/*
- * Next two functions release bfqd->lock and put the io context
- * pointed by bfqd->ioc_to_put. This delayed put is used to not risk
- * to take an ioc->lock while the scheduler lock is being held.
- */
-static void bfq_unlock_put_ioc(struct bfq_data *bfqd)
-{
- struct io_context *ioc_to_put = bfqd->ioc_to_put;
-
- bfqd->ioc_to_put = NULL;
- spin_unlock_irq(&bfqd->lock);
-
- if (ioc_to_put)
- put_io_context(ioc_to_put);
-}
-
-static void bfq_unlock_put_ioc_restore(struct bfq_data *bfqd,
- unsigned long flags)
-{
- struct io_context *ioc_to_put = bfqd->ioc_to_put;
-
- bfqd->ioc_to_put = NULL;
- spin_unlock_irqrestore(&bfqd->lock, flags);
-
- if (ioc_to_put)
- put_io_context(ioc_to_put);
-}
-
/**
* bfq_gt - compare two timestamps.
* @a: first ts.
@@ -2684,18 +2645,6 @@ static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd)
struct bfq_entity *in_serv_entity = &in_serv_bfqq->entity;
struct bfq_entity *entity = in_serv_entity;
- if (bfqd->in_service_bic) {
- /*
- * Schedule the release of a reference to
- * bfqd->in_service_bic->icq.ioc to right after the
- * scheduler lock is released. This ioc is not
- * released immediately, to not risk to possibly take
- * an ioc->lock while holding the scheduler lock.
- */
- bfqd->ioc_to_put = bfqd->in_service_bic->icq.ioc;
- bfqd->in_service_bic = NULL;
- }
-
bfq_clear_bfqq_wait_request(in_serv_bfqq);
hrtimer_try_to_cancel(&bfqd->idle_slice_timer);
bfqd->in_service_queue = NULL;
@@ -3495,7 +3444,7 @@ static void bfq_pd_offline(struct blkg_policy_data *pd)
__bfq_deactivate_entity(entity, false);
bfq_put_async_queues(bfqd, bfqg);
- bfq_unlock_put_ioc_restore(bfqd, flags);
+ spin_unlock_irqrestore(&bfqd->lock, flags);
/*
* @blkg is going offline and will be ignored by
* blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
@@ -5472,20 +5421,18 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
* first time that the requests of some process are redirected to
* it.
*
- * We redirect bfqq to new_bfqq and not the opposite, because we
- * are in the context of the process owning bfqq, hence we have
- * the io_cq of this process. So we can immediately configure this
- * io_cq to redirect the requests of the process to new_bfqq.
+ * We redirect bfqq to new_bfqq and not the opposite, because
+ * we are in the context of the process owning bfqq, thus we
+ * have the io_cq of this process. So we can immediately
+ * configure this io_cq to redirect the requests of the
+ * process to new_bfqq. In contrast, the io_cq of new_bfqq is
+ * not available any more (new_bfqq->bic == NULL).
*
- * NOTE, even if new_bfqq coincides with the in-service queue, the
- * io_cq of new_bfqq is not available, because, if the in-service
- * queue is shared, bfqd->in_service_bic may not point to the
- * io_cq of the in-service queue.
- * Redirecting the requests of the process owning bfqq to the
- * currently in-service queue is in any case the best option, as
- * we feed the in-service queue with new requests close to the
- * last request served and, by doing so, hopefully increase the
- * throughput.
+ * Anyway, even in case new_bfqq coincides with the in-service
+ * queue, redirecting requests the in-service queue is the
+ * best option, as we feed the in-service queue with new
+ * requests close to the last request served and, by doing so,
+ * are likely to increase the throughput.
*/
bfqq->new_bfqq = new_bfqq;
new_bfqq->ref += process_refs;
@@ -5577,8 +5524,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
in_service_bfqq = bfqd->in_service_queue;
- if (!in_service_bfqq || in_service_bfqq == bfqq ||
- !bfqd->in_service_bic || wr_from_too_long(in_service_bfqq) ||
+ if (!in_service_bfqq || in_service_bfqq == bfqq
+ || wr_from_too_long(in_service_bfqq) ||
unlikely(in_service_bfqq == &bfqd->oom_bfqq))
goto check_scheduled;
@@ -5629,16 +5576,6 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;
}
-static void bfq_get_bic_reference(struct bfq_queue *bfqq)
-{
- /*
- * If bfqq->bic has a non-NULL value, the bic to which it belongs
- * is about to begin using a shared bfq_queue.
- */
- if (bfqq->bic)
- atomic_long_inc(&bfqq->bic->icq.ioc->refcount);
-}
-
static void
bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
@@ -5683,12 +5620,6 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
bfqd->wr_busy_queues);
/*
- * Grab a reference to the bic, to prevent it from being destroyed
- * before being possibly touched by a bfq_split_bfqq().
- */
- bfq_get_bic_reference(bfqq);
- bfq_get_bic_reference(new_bfqq);
- /*
* Merge queues (that is, let bic redirect its requests to new_bfqq)
*/
bic_set_bfqq(bic, new_bfqq, 1);
@@ -5853,14 +5784,8 @@ static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd)
static void bfq_arm_slice_timer(struct bfq_data *bfqd)
{
struct bfq_queue *bfqq = bfqd->in_service_queue;
- struct bfq_io_cq *bic;
u32 sl;
- /* Processes have exited, don't wait. */
- bic = bfqd->in_service_bic;
- if (!bic || atomic_read(&bic->icq.ioc->active_ref) == 0)
- return;
-
bfq_mark_bfqq_wait_request(bfqq);
/*
@@ -7147,11 +7072,6 @@ static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd,
*/
bfq_update_wr_data(bfqd, bfqq);
- if (!bfqd->in_service_bic) {
- atomic_long_inc(&RQ_BIC(rq)->icq.ioc->refcount);
- bfqd->in_service_bic = RQ_BIC(rq);
- }
-
/*
* Expire bfqq, pretending that its budget expired, if bfqq
* belongs to CLASS_IDLE and other queues are waiting for
@@ -7272,7 +7192,7 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
spin_lock_irq(&bfqd->lock);
rq = __bfq_dispatch_request(hctx);
- bfq_unlock_put_ioc(bfqd);
+ spin_unlock_irq(&bfqd->lock);
return rq;
}
@@ -7360,20 +7280,9 @@ static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync)
unsigned long flags;
spin_lock_irqsave(&bfqd->lock, flags);
- /*
- * If the bic is using a shared queue, put the
- * reference taken on the io_context when the bic
- * started using a shared bfq_queue. This put cannot
- * make ioc->ref_count reach 0, then no ioc->lock
- * risks to be taken (leading to possible deadlock
- * scenarios).
- */
- if (is_sync && bfq_bfqq_coop(bfqq))
- put_io_context(bic->icq.ioc);
-
bfq_exit_bfqq(bfqd, bfqq);
bic_set_bfqq(bic, NULL, is_sync);
- bfq_unlock_put_ioc_restore(bfqd, flags);
+ spin_unlock_irqrestore(&bfqd->lock, flags);
}
}
@@ -7808,7 +7717,7 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
}
}
- bfq_unlock_put_ioc(bfqd);
+ spin_unlock_irq(&bfqd->lock);
}
static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx,
@@ -7962,7 +7871,7 @@ static void bfq_put_rq_private(struct request_queue *q, struct request *rq)
bfq_completed_request(bfqq, bfqd);
bfq_put_rq_priv_body(bfqq);
- bfq_unlock_put_ioc_restore(bfqd, flags);
+ spin_unlock_irqrestore(&bfqd->lock, flags);
} else {
/*
* Request rq may be still/already in the scheduler,
@@ -8055,6 +7964,7 @@ static int bfq_get_rq_private(struct request_queue *q, struct request *rq,
const int is_sync = rq_is_sync(rq);
struct bfq_queue *bfqq;
bool new_queue = false;
+ bool split = false;
spin_lock_irq(&bfqd->lock);
@@ -8078,14 +7988,7 @@ static int bfq_get_rq_private(struct request_queue *q, struct request *rq,
bic->saved_in_large_burst = true;
bfqq = bfq_split_bfqq(bic, bfqq);
- /*
- * A reference to bic->icq.ioc needs to be
- * released after a queue split. Do not do it
- * immediately, to not risk to possibly take
- * an ioc->lock while holding the scheduler
- * lock.
- */
- bfqd->ioc_to_put = bic->icq.ioc;
+ split = true;
if (!bfqq)
bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,
@@ -8110,7 +8013,7 @@ static int bfq_get_rq_private(struct request_queue *q, struct request *rq,
*/
if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) {
bfqq->bic = bic;
- if (bfqd->ioc_to_put) { /* if true, there has been a split */
+ if (split) {
/*
* The queue has just been split from a shared
* queue: restore the idle window and the
@@ -8123,7 +8026,7 @@ static int bfq_get_rq_private(struct request_queue *q, struct request *rq,
if (unlikely(bfq_bfqq_just_created(bfqq)))
bfq_handle_burst(bfqd, bfqq);
- bfq_unlock_put_ioc(bfqd);
+ spin_unlock_irq(&bfqd->lock);
return 0;
@@ -8168,7 +8071,7 @@ static void bfq_idle_slice_timer_body(struct bfq_queue *bfqq)
bfq_bfqq_expire(bfqd, bfqq, true, reason);
schedule_dispatch:
- bfq_unlock_put_ioc_restore(bfqd, flags);
+ spin_unlock_irqrestore(&bfqd->lock, flags);
bfq_schedule_dispatch(bfqd);
}