From 974271c485a4d8bb801decc616748f90aafb07ec Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 12 Jul 2012 14:46:37 -0700 Subject: workqueue: don't use WQ_HIGHPRI for unbound workqueues Unbound wqs aren't concurrency-managed and try to execute work items as soon as possible. This is currently achieved by implicitly setting %WQ_HIGHPRI on all unbound workqueues; however, WQ_HIGHPRI implementation is about to be restructured and this usage won't be valid anymore. Add an explicit chain-wakeup path for unbound workqueues in process_one_work() instead of piggy backing on %WQ_HIGHPRI. Signed-off-by: Tejun Heo --- kernel/workqueue.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 9a3128dc67df..27637c284cb9 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -580,6 +580,10 @@ static bool __need_more_worker(struct global_cwq *gcwq) /* * Need to wake up a worker? Called from anything but currently * running workers. + * + * Note that, because unbound workers never contribute to nr_running, this + * function will always return %true for unbound gcwq as long as the + * worklist isn't empty. */ static bool need_more_worker(struct global_cwq *gcwq) { @@ -1867,6 +1871,13 @@ __acquires(&gcwq->lock) if (unlikely(cpu_intensive)) worker_set_flags(worker, WORKER_CPU_INTENSIVE, true); + /* + * Unbound gcwq isn't concurrency managed and work items should be + * executed ASAP. Wake up another worker if necessary. + */ + if ((worker->flags & WORKER_UNBOUND) && need_more_worker(gcwq)) + wake_up_worker(gcwq); + spin_unlock_irq(&gcwq->lock); work_clear_pending(work); @@ -2984,13 +2995,6 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, if (flags & WQ_MEM_RECLAIM) flags |= WQ_RESCUER; - /* - * Unbound workqueues aren't concurrency managed and should be - * dispatched to workers immediately. - */ - if (flags & WQ_UNBOUND) - flags |= WQ_HIGHPRI; - max_active = max_active ?: WQ_DFL_ACTIVE; max_active = wq_clamp_max_active(max_active, flags, wq->name); -- cgit v1.2.3-55-g7522 From bd7bdd43dcb81bb08240b9401b36a104f77dc135 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 12 Jul 2012 14:46:37 -0700 Subject: workqueue: factor out worker_pool from global_cwq Move worklist and all worker management fields from global_cwq into the new struct worker_pool. worker_pool points back to the containing gcwq. worker and cpu_workqueue_struct are updated to point to worker_pool instead of gcwq too. This change is mechanical and doesn't introduce any functional difference other than rearranging of fields and an added level of indirection in some places. This is to prepare for multiple pools per gcwq. v2: Comment typo fixes as suggested by Namhyung. Signed-off-by: Tejun Heo Cc: Namhyung Kim --- include/trace/events/workqueue.h | 2 +- kernel/workqueue.c | 216 +++++++++++++++++++++------------------ 2 files changed, 118 insertions(+), 100 deletions(-) (limited to 'kernel') diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h index 4018f5058f27..f28d1b65f178 100644 --- a/include/trace/events/workqueue.h +++ b/include/trace/events/workqueue.h @@ -54,7 +54,7 @@ TRACE_EVENT(workqueue_queue_work, __entry->function = work->func; __entry->workqueue = cwq->wq; __entry->req_cpu = req_cpu; - __entry->cpu = cwq->gcwq->cpu; + __entry->cpu = cwq->pool->gcwq->cpu; ), TP_printk("work struct=%p function=%pf workqueue=%p req_cpu=%u cpu=%u", diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 27637c284cb9..61f154467026 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -115,6 +115,7 @@ enum { */ struct global_cwq; +struct worker_pool; /* * The poor guys doing the actual heavy lifting. All on-duty workers @@ -131,7 +132,7 @@ struct worker { struct cpu_workqueue_struct *current_cwq; /* L: current_work's cwq */ struct list_head scheduled; /* L: scheduled works */ struct task_struct *task; /* I: worker task */ - struct global_cwq *gcwq; /* I: the associated gcwq */ + struct worker_pool *pool; /* I: the associated pool */ /* 64 bytes boundary on 64bit, 32 on 32bit */ unsigned long last_active; /* L: last active timestamp */ unsigned int flags; /* X: flags */ @@ -139,6 +140,21 @@ struct worker { struct work_struct rebind_work; /* L: rebind worker to cpu */ }; +struct worker_pool { + struct global_cwq *gcwq; /* I: the owning gcwq */ + + struct list_head worklist; /* L: list of pending works */ + int nr_workers; /* L: total number of workers */ + int nr_idle; /* L: currently idle ones */ + + struct list_head idle_list; /* X: list of idle workers */ + struct timer_list idle_timer; /* L: worker idle timeout */ + struct timer_list mayday_timer; /* L: SOS timer for workers */ + + struct ida worker_ida; /* L: for worker IDs */ + struct worker *first_idle; /* L: first idle worker */ +}; + /* * Global per-cpu workqueue. There's one and only one for each cpu * and all works are queued and processed here regardless of their @@ -146,27 +162,18 @@ struct worker { */ struct global_cwq { spinlock_t lock; /* the gcwq lock */ - struct list_head worklist; /* L: list of pending works */ unsigned int cpu; /* I: the associated cpu */ unsigned int flags; /* L: GCWQ_* flags */ - int nr_workers; /* L: total number of workers */ - int nr_idle; /* L: currently idle ones */ - - /* workers are chained either in the idle_list or busy_hash */ - struct list_head idle_list; /* X: list of idle workers */ + /* workers are chained either in busy_hash or pool idle_list */ struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE]; /* L: hash of busy workers */ - struct timer_list idle_timer; /* L: worker idle timeout */ - struct timer_list mayday_timer; /* L: SOS timer for dworkers */ - - struct ida worker_ida; /* L: for worker IDs */ + struct worker_pool pool; /* the worker pools */ struct task_struct *trustee; /* L: for gcwq shutdown */ unsigned int trustee_state; /* L: trustee state */ wait_queue_head_t trustee_wait; /* trustee wait */ - struct worker *first_idle; /* L: first idle worker */ } ____cacheline_aligned_in_smp; /* @@ -175,7 +182,7 @@ struct global_cwq { * aligned at two's power of the number of flag bits. */ struct cpu_workqueue_struct { - struct global_cwq *gcwq; /* I: the associated gcwq */ + struct worker_pool *pool; /* I: the associated pool */ struct workqueue_struct *wq; /* I: the owning workqueue */ int work_color; /* L: current color */ int flush_color; /* L: flushing color */ @@ -555,7 +562,7 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) if (data & WORK_STRUCT_CWQ) return ((struct cpu_workqueue_struct *) - (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq; + (data & WORK_STRUCT_WQ_DATA_MASK))->pool->gcwq; cpu = data >> WORK_STRUCT_FLAG_BITS; if (cpu == WORK_CPU_NONE) @@ -587,13 +594,13 @@ static bool __need_more_worker(struct global_cwq *gcwq) */ static bool need_more_worker(struct global_cwq *gcwq) { - return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq); + return !list_empty(&gcwq->pool.worklist) && __need_more_worker(gcwq); } /* Can I start working? Called from busy but !running workers. */ static bool may_start_working(struct global_cwq *gcwq) { - return gcwq->nr_idle; + return gcwq->pool.nr_idle; } /* Do I need to keep working? Called from currently running workers. */ @@ -601,7 +608,7 @@ static bool keep_working(struct global_cwq *gcwq) { atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); - return !list_empty(&gcwq->worklist) && + return !list_empty(&gcwq->pool.worklist) && (atomic_read(nr_running) <= 1 || gcwq->flags & GCWQ_HIGHPRI_PENDING); } @@ -622,8 +629,8 @@ static bool need_to_manage_workers(struct global_cwq *gcwq) static bool too_many_workers(struct global_cwq *gcwq) { bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS; - int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */ - int nr_busy = gcwq->nr_workers - nr_idle; + int nr_idle = gcwq->pool.nr_idle + managing; /* manager is considered idle */ + int nr_busy = gcwq->pool.nr_workers - nr_idle; return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; } @@ -635,10 +642,10 @@ static bool too_many_workers(struct global_cwq *gcwq) /* Return the first worker. Safe with preemption disabled */ static struct worker *first_worker(struct global_cwq *gcwq) { - if (unlikely(list_empty(&gcwq->idle_list))) + if (unlikely(list_empty(&gcwq->pool.idle_list))) return NULL; - return list_first_entry(&gcwq->idle_list, struct worker, entry); + return list_first_entry(&gcwq->pool.idle_list, struct worker, entry); } /** @@ -696,7 +703,8 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, unsigned int cpu) { struct worker *worker = kthread_data(task), *to_wakeup = NULL; - struct global_cwq *gcwq = get_gcwq(cpu); + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; atomic_t *nr_running = get_gcwq_nr_running(cpu); if (worker->flags & WORKER_NOT_RUNNING) @@ -716,7 +724,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, * could be manipulating idle_list, so dereferencing idle_list * without gcwq lock is safe. */ - if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) + if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist)) to_wakeup = first_worker(gcwq); return to_wakeup ? to_wakeup->task : NULL; } @@ -737,7 +745,8 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, static inline void worker_set_flags(struct worker *worker, unsigned int flags, bool wakeup) { - struct global_cwq *gcwq = worker->gcwq; + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; WARN_ON_ONCE(worker->task != current); @@ -752,7 +761,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, if (wakeup) { if (atomic_dec_and_test(nr_running) && - !list_empty(&gcwq->worklist)) + !list_empty(&pool->worklist)) wake_up_worker(gcwq); } else atomic_dec(nr_running); @@ -773,7 +782,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, */ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) { - struct global_cwq *gcwq = worker->gcwq; + struct global_cwq *gcwq = worker->pool->gcwq; unsigned int oflags = worker->flags; WARN_ON_ONCE(worker->task != current); @@ -894,9 +903,9 @@ static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq, struct work_struct *twork; if (likely(!(cwq->wq->flags & WQ_HIGHPRI))) - return &gcwq->worklist; + return &gcwq->pool.worklist; - list_for_each_entry(twork, &gcwq->worklist, entry) { + list_for_each_entry(twork, &gcwq->pool.worklist, entry) { struct cpu_workqueue_struct *tcwq = get_work_cwq(twork); if (!(tcwq->wq->flags & WQ_HIGHPRI)) @@ -924,7 +933,7 @@ static void insert_work(struct cpu_workqueue_struct *cwq, struct work_struct *work, struct list_head *head, unsigned int extra_flags) { - struct global_cwq *gcwq = cwq->gcwq; + struct global_cwq *gcwq = cwq->pool->gcwq; /* we own @work, set data and link */ set_work_cwq(work, cwq, extra_flags); @@ -1196,7 +1205,8 @@ EXPORT_SYMBOL_GPL(queue_delayed_work_on); */ static void worker_enter_idle(struct worker *worker) { - struct global_cwq *gcwq = worker->gcwq; + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; BUG_ON(worker->flags & WORKER_IDLE); BUG_ON(!list_empty(&worker->entry) && @@ -1204,15 +1214,15 @@ static void worker_enter_idle(struct worker *worker) /* can't use worker_set_flags(), also called from start_worker() */ worker->flags |= WORKER_IDLE; - gcwq->nr_idle++; + pool->nr_idle++; worker->last_active = jiffies; /* idle_list is LIFO */ - list_add(&worker->entry, &gcwq->idle_list); + list_add(&worker->entry, &pool->idle_list); if (likely(!(worker->flags & WORKER_ROGUE))) { - if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer)) - mod_timer(&gcwq->idle_timer, + if (too_many_workers(gcwq) && !timer_pending(&pool->idle_timer)) + mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); } else wake_up_all(&gcwq->trustee_wait); @@ -1223,7 +1233,7 @@ static void worker_enter_idle(struct worker *worker) * warning may trigger spuriously. Check iff trustee is idle. */ WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE && - gcwq->nr_workers == gcwq->nr_idle && + pool->nr_workers == pool->nr_idle && atomic_read(get_gcwq_nr_running(gcwq->cpu))); } @@ -1238,11 +1248,11 @@ static void worker_enter_idle(struct worker *worker) */ static void worker_leave_idle(struct worker *worker) { - struct global_cwq *gcwq = worker->gcwq; + struct worker_pool *pool = worker->pool; BUG_ON(!(worker->flags & WORKER_IDLE)); worker_clr_flags(worker, WORKER_IDLE); - gcwq->nr_idle--; + pool->nr_idle--; list_del_init(&worker->entry); } @@ -1279,7 +1289,7 @@ static void worker_leave_idle(struct worker *worker) static bool worker_maybe_bind_and_lock(struct worker *worker) __acquires(&gcwq->lock) { - struct global_cwq *gcwq = worker->gcwq; + struct global_cwq *gcwq = worker->pool->gcwq; struct task_struct *task = worker->task; while (true) { @@ -1321,7 +1331,7 @@ __acquires(&gcwq->lock) static void worker_rebind_fn(struct work_struct *work) { struct worker *worker = container_of(work, struct worker, rebind_work); - struct global_cwq *gcwq = worker->gcwq; + struct global_cwq *gcwq = worker->pool->gcwq; if (worker_maybe_bind_and_lock(worker)) worker_clr_flags(worker, WORKER_REBIND); @@ -1362,13 +1372,14 @@ static struct worker *alloc_worker(void) static struct worker *create_worker(struct global_cwq *gcwq, bool bind) { bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; + struct worker_pool *pool = &gcwq->pool; struct worker *worker = NULL; int id = -1; spin_lock_irq(&gcwq->lock); - while (ida_get_new(&gcwq->worker_ida, &id)) { + while (ida_get_new(&pool->worker_ida, &id)) { spin_unlock_irq(&gcwq->lock); - if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL)) + if (!ida_pre_get(&pool->worker_ida, GFP_KERNEL)) goto fail; spin_lock_irq(&gcwq->lock); } @@ -1378,7 +1389,7 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) if (!worker) goto fail; - worker->gcwq = gcwq; + worker->pool = pool; worker->id = id; if (!on_unbound_cpu) @@ -1409,7 +1420,7 @@ static struct worker *create_worker(struct global_cwq *gcwq, bool bind) fail: if (id >= 0) { spin_lock_irq(&gcwq->lock); - ida_remove(&gcwq->worker_ida, id); + ida_remove(&pool->worker_ida, id); spin_unlock_irq(&gcwq->lock); } kfree(worker); @@ -1428,7 +1439,7 @@ fail: static void start_worker(struct worker *worker) { worker->flags |= WORKER_STARTED; - worker->gcwq->nr_workers++; + worker->pool->nr_workers++; worker_enter_idle(worker); wake_up_process(worker->task); } @@ -1444,7 +1455,8 @@ static void start_worker(struct worker *worker) */ static void destroy_worker(struct worker *worker) { - struct global_cwq *gcwq = worker->gcwq; + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; int id = worker->id; /* sanity check frenzy */ @@ -1452,9 +1464,9 @@ static void destroy_worker(struct worker *worker) BUG_ON(!list_empty(&worker->scheduled)); if (worker->flags & WORKER_STARTED) - gcwq->nr_workers--; + pool->nr_workers--; if (worker->flags & WORKER_IDLE) - gcwq->nr_idle--; + pool->nr_idle--; list_del_init(&worker->entry); worker->flags |= WORKER_DIE; @@ -1465,7 +1477,7 @@ static void destroy_worker(struct worker *worker) kfree(worker); spin_lock_irq(&gcwq->lock); - ida_remove(&gcwq->worker_ida, id); + ida_remove(&pool->worker_ida, id); } static void idle_worker_timeout(unsigned long __gcwq) @@ -1479,11 +1491,12 @@ static void idle_worker_timeout(unsigned long __gcwq) unsigned long expires; /* idle_list is kept in LIFO order, check the last one */ - worker = list_entry(gcwq->idle_list.prev, struct worker, entry); + worker = list_entry(gcwq->pool.idle_list.prev, struct worker, + entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) - mod_timer(&gcwq->idle_timer, expires); + mod_timer(&gcwq->pool.idle_timer, expires); else { /* it's been idle for too long, wake up manager */ gcwq->flags |= GCWQ_MANAGE_WORKERS; @@ -1504,7 +1517,7 @@ static bool send_mayday(struct work_struct *work) return false; /* mayday mayday mayday */ - cpu = cwq->gcwq->cpu; + cpu = cwq->pool->gcwq->cpu; /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */ if (cpu == WORK_CPU_UNBOUND) cpu = 0; @@ -1527,13 +1540,13 @@ static void gcwq_mayday_timeout(unsigned long __gcwq) * allocation deadlock. Send distress signals to * rescuers. */ - list_for_each_entry(work, &gcwq->worklist, entry) + list_for_each_entry(work, &gcwq->pool.worklist, entry) send_mayday(work); } spin_unlock_irq(&gcwq->lock); - mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL); + mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INTERVAL); } /** @@ -1568,14 +1581,14 @@ restart: spin_unlock_irq(&gcwq->lock); /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ - mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); + mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); while (true) { struct worker *worker; worker = create_worker(gcwq, true); if (worker) { - del_timer_sync(&gcwq->mayday_timer); + del_timer_sync(&gcwq->pool.mayday_timer); spin_lock_irq(&gcwq->lock); start_worker(worker); BUG_ON(need_to_create_worker(gcwq)); @@ -1592,7 +1605,7 @@ restart: break; } - del_timer_sync(&gcwq->mayday_timer); + del_timer_sync(&gcwq->pool.mayday_timer); spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) goto restart; @@ -1622,11 +1635,12 @@ static bool maybe_destroy_workers(struct global_cwq *gcwq) struct worker *worker; unsigned long expires; - worker = list_entry(gcwq->idle_list.prev, struct worker, entry); + worker = list_entry(gcwq->pool.idle_list.prev, struct worker, + entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) { - mod_timer(&gcwq->idle_timer, expires); + mod_timer(&gcwq->pool.idle_timer, expires); break; } @@ -1659,7 +1673,7 @@ static bool maybe_destroy_workers(struct global_cwq *gcwq) */ static bool manage_workers(struct worker *worker) { - struct global_cwq *gcwq = worker->gcwq; + struct global_cwq *gcwq = worker->pool->gcwq; bool ret = false; if (gcwq->flags & GCWQ_MANAGING_WORKERS) @@ -1732,7 +1746,7 @@ static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) { struct work_struct *work = list_first_entry(&cwq->delayed_works, struct work_struct, entry); - struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq); + struct list_head *pos = gcwq_determine_ins_pos(cwq->pool->gcwq, cwq); trace_workqueue_activate_work(work); move_linked_works(work, pos, NULL); @@ -1808,7 +1822,8 @@ __releases(&gcwq->lock) __acquires(&gcwq->lock) { struct cpu_workqueue_struct *cwq = get_work_cwq(work); - struct global_cwq *gcwq = cwq->gcwq; + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; struct hlist_head *bwh = busy_worker_head(gcwq, work); bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE; work_func_t f = work->func; @@ -1854,10 +1869,10 @@ __acquires(&gcwq->lock) * wake up another worker; otherwise, clear HIGHPRI_PENDING. */ if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) { - struct work_struct *nwork = list_first_entry(&gcwq->worklist, - struct work_struct, entry); + struct work_struct *nwork = list_first_entry(&pool->worklist, + struct work_struct, entry); - if (!list_empty(&gcwq->worklist) && + if (!list_empty(&pool->worklist) && get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) wake_up_worker(gcwq); else @@ -1950,7 +1965,8 @@ static void process_scheduled_works(struct worker *worker) static int worker_thread(void *__worker) { struct worker *worker = __worker; - struct global_cwq *gcwq = worker->gcwq; + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; /* tell the scheduler that this is a workqueue worker */ worker->task->flags |= PF_WQ_WORKER; @@ -1990,7 +2006,7 @@ recheck: do { struct work_struct *work = - list_first_entry(&gcwq->worklist, + list_first_entry(&pool->worklist, struct work_struct, entry); if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) { @@ -2064,14 +2080,15 @@ repeat: for_each_mayday_cpu(cpu, wq->mayday_mask) { unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu; struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq); - struct global_cwq *gcwq = cwq->gcwq; + struct worker_pool *pool = cwq->pool; + struct global_cwq *gcwq = pool->gcwq; struct work_struct *work, *n; __set_current_state(TASK_RUNNING); mayday_clear_cpu(cpu, wq->mayday_mask); /* migrate to the target cpu if possible */ - rescuer->gcwq = gcwq; + rescuer->pool = pool; worker_maybe_bind_and_lock(rescuer); /* @@ -2079,7 +2096,7 @@ repeat: * process'em. */ BUG_ON(!list_empty(&rescuer->scheduled)); - list_for_each_entry_safe(work, n, &gcwq->worklist, entry) + list_for_each_entry_safe(work, n, &pool->worklist, entry) if (get_work_cwq(work) == cwq) move_linked_works(work, scheduled, &n); @@ -2216,7 +2233,7 @@ static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); - struct global_cwq *gcwq = cwq->gcwq; + struct global_cwq *gcwq = cwq->pool->gcwq; spin_lock_irq(&gcwq->lock); @@ -2432,9 +2449,9 @@ reflush: struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); bool drained; - spin_lock_irq(&cwq->gcwq->lock); + spin_lock_irq(&cwq->pool->gcwq->lock); drained = !cwq->nr_active && list_empty(&cwq->delayed_works); - spin_unlock_irq(&cwq->gcwq->lock); + spin_unlock_irq(&cwq->pool->gcwq->lock); if (drained) continue; @@ -2474,7 +2491,7 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, */ smp_rmb(); cwq = get_work_cwq(work); - if (unlikely(!cwq || gcwq != cwq->gcwq)) + if (unlikely(!cwq || gcwq != cwq->pool->gcwq)) goto already_gone; } else if (wait_executing) { worker = find_worker_executing_work(gcwq, work); @@ -3017,7 +3034,7 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, struct global_cwq *gcwq = get_gcwq(cpu); BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK); - cwq->gcwq = gcwq; + cwq->pool = &gcwq->pool; cwq->wq = wq; cwq->flush_color = -1; cwq->max_active = max_active; @@ -3344,7 +3361,7 @@ static int __cpuinit trustee_thread(void *__gcwq) gcwq->flags |= GCWQ_MANAGING_WORKERS; - list_for_each_entry(worker, &gcwq->idle_list, entry) + list_for_each_entry(worker, &gcwq->pool.idle_list, entry) worker->flags |= WORKER_ROGUE; for_each_busy_worker(worker, i, pos, gcwq) @@ -3369,7 +3386,7 @@ static int __cpuinit trustee_thread(void *__gcwq) atomic_set(get_gcwq_nr_running(gcwq->cpu), 0); spin_unlock_irq(&gcwq->lock); - del_timer_sync(&gcwq->idle_timer); + del_timer_sync(&gcwq->pool.idle_timer); spin_lock_irq(&gcwq->lock); /* @@ -3391,17 +3408,17 @@ static int __cpuinit trustee_thread(void *__gcwq) * may be frozen works in freezable cwqs. Don't declare * completion while frozen. */ - while (gcwq->nr_workers != gcwq->nr_idle || + while (gcwq->pool.nr_workers != gcwq->pool.nr_idle || gcwq->flags & GCWQ_FREEZING || gcwq->trustee_state == TRUSTEE_IN_CHARGE) { int nr_works = 0; - list_for_each_entry(work, &gcwq->worklist, entry) { + list_for_each_entry(work, &gcwq->pool.worklist, entry) { send_mayday(work); nr_works++; } - list_for_each_entry(worker, &gcwq->idle_list, entry) { + list_for_each_entry(worker, &gcwq->pool.idle_list, entry) { if (!nr_works--) break; wake_up_process(worker->task); @@ -3428,11 +3445,11 @@ static int __cpuinit trustee_thread(void *__gcwq) * all workers till we're canceled. */ do { - rc = trustee_wait_event(!list_empty(&gcwq->idle_list)); - while (!list_empty(&gcwq->idle_list)) - destroy_worker(list_first_entry(&gcwq->idle_list, + rc = trustee_wait_event(!list_empty(&gcwq->pool.idle_list)); + while (!list_empty(&gcwq->pool.idle_list)) + destroy_worker(list_first_entry(&gcwq->pool.idle_list, struct worker, entry)); - } while (gcwq->nr_workers && rc >= 0); + } while (gcwq->pool.nr_workers && rc >= 0); /* * At this point, either draining has completed and no worker @@ -3441,7 +3458,7 @@ static int __cpuinit trustee_thread(void *__gcwq) * Tell the remaining busy ones to rebind once it finishes the * currently scheduled works by scheduling the rebind_work. */ - WARN_ON(!list_empty(&gcwq->idle_list)); + WARN_ON(!list_empty(&gcwq->pool.idle_list)); for_each_busy_worker(worker, i, pos, gcwq) { struct work_struct *rebind_work = &worker->rebind_work; @@ -3522,7 +3539,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, kthread_bind(new_trustee, cpu); /* fall through */ case CPU_UP_PREPARE: - BUG_ON(gcwq->first_idle); + BUG_ON(gcwq->pool.first_idle); new_worker = create_worker(gcwq, false); if (!new_worker) { if (new_trustee) @@ -3544,8 +3561,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); /* fall through */ case CPU_UP_PREPARE: - BUG_ON(gcwq->first_idle); - gcwq->first_idle = new_worker; + BUG_ON(gcwq->pool.first_idle); + gcwq->pool.first_idle = new_worker; break; case CPU_DYING: @@ -3562,8 +3579,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, gcwq->trustee_state = TRUSTEE_BUTCHER; /* fall through */ case CPU_UP_CANCELED: - destroy_worker(gcwq->first_idle); - gcwq->first_idle = NULL; + destroy_worker(gcwq->pool.first_idle); + gcwq->pool.first_idle = NULL; break; case CPU_DOWN_FAILED: @@ -3581,11 +3598,11 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, * take a look. */ spin_unlock_irq(&gcwq->lock); - kthread_bind(gcwq->first_idle->task, cpu); + kthread_bind(gcwq->pool.first_idle->task, cpu); spin_lock_irq(&gcwq->lock); gcwq->flags |= GCWQ_MANAGE_WORKERS; - start_worker(gcwq->first_idle); - gcwq->first_idle = NULL; + start_worker(gcwq->pool.first_idle); + gcwq->pool.first_idle = NULL; break; } @@ -3794,22 +3811,23 @@ static int __init init_workqueues(void) struct global_cwq *gcwq = get_gcwq(cpu); spin_lock_init(&gcwq->lock); - INIT_LIST_HEAD(&gcwq->worklist); + gcwq->pool.gcwq = gcwq; + INIT_LIST_HEAD(&gcwq->pool.worklist); gcwq->cpu = cpu; gcwq->flags |= GCWQ_DISASSOCIATED; - INIT_LIST_HEAD(&gcwq->idle_list); + INIT_LIST_HEAD(&gcwq->pool.idle_list); for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) INIT_HLIST_HEAD(&gcwq->busy_hash[i]); - init_timer_deferrable(&gcwq->idle_timer); - gcwq->idle_timer.function = idle_worker_timeout; - gcwq->idle_timer.data = (unsigned long)gcwq; + init_timer_deferrable(&gcwq->pool.idle_timer); + gcwq->pool.idle_timer.function = idle_worker_timeout; + gcwq->pool.idle_timer.data = (unsigned long)gcwq; - setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout, + setup_timer(&gcwq->pool.mayday_timer, gcwq_mayday_timeout, (unsigned long)gcwq); - ida_init(&gcwq->worker_ida); + ida_init(&gcwq->pool.worker_ida); gcwq->trustee_state = TRUSTEE_DONE; init_waitqueue_head(&gcwq->trustee_wait); -- cgit v1.2.3-55-g7522 From 63d95a9150ee3bbd4117fcd609dee40313b454d9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 12 Jul 2012 14:46:37 -0700 Subject: workqueue: use @pool instead of @gcwq or @cpu where applicable Modify all functions which deal with per-pool properties to pass around @pool instead of @gcwq or @cpu. The changes in this patch are mechanical and don't caues any functional difference. This is to prepare for multiple pools per gcwq. Signed-off-by: Tejun Heo --- kernel/workqueue.c | 218 +++++++++++++++++++++++++++-------------------------- 1 file changed, 111 insertions(+), 107 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 61f154467026..2d82f7b193a0 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -471,8 +471,10 @@ static struct global_cwq *get_gcwq(unsigned int cpu) return &unbound_global_cwq; } -static atomic_t *get_gcwq_nr_running(unsigned int cpu) +static atomic_t *get_pool_nr_running(struct worker_pool *pool) { + int cpu = pool->gcwq->cpu; + if (cpu != WORK_CPU_UNBOUND) return &per_cpu(gcwq_nr_running, cpu); else @@ -578,10 +580,10 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) * assume that they're being called with gcwq->lock held. */ -static bool __need_more_worker(struct global_cwq *gcwq) +static bool __need_more_worker(struct worker_pool *pool) { - return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) || - gcwq->flags & GCWQ_HIGHPRI_PENDING; + return !atomic_read(get_pool_nr_running(pool)) || + pool->gcwq->flags & GCWQ_HIGHPRI_PENDING; } /* @@ -592,45 +594,46 @@ static bool __need_more_worker(struct global_cwq *gcwq) * function will always return %true for unbound gcwq as long as the * worklist isn't empty. */ -static bool need_more_worker(struct global_cwq *gcwq) +static bool need_more_worker(struct worker_pool *pool) { - return !list_empty(&gcwq->pool.worklist) && __need_more_worker(gcwq); + return !list_empty(&pool->worklist) && __need_more_worker(pool); } /* Can I start working? Called from busy but !running workers. */ -static bool may_start_working(struct global_cwq *gcwq) +static bool may_start_working(struct worker_pool *pool) { - return gcwq->pool.nr_idle; + return pool->nr_idle; } /* Do I need to keep working? Called from currently running workers. */ -static bool keep_working(struct global_cwq *gcwq) +static bool keep_working(struct worker_pool *pool) { - atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); + atomic_t *nr_running = get_pool_nr_running(pool); - return !list_empty(&gcwq->pool.worklist) && + return !list_empty(&pool->worklist) && (atomic_read(nr_running) <= 1 || - gcwq->flags & GCWQ_HIGHPRI_PENDING); + pool->gcwq->flags & GCWQ_HIGHPRI_PENDING); } /* Do we need a new worker? Called from manager. */ -static bool need_to_create_worker(struct global_cwq *gcwq) +static bool need_to_create_worker(struct worker_pool *pool) { - return need_more_worker(gcwq) && !may_start_working(gcwq); + return need_more_worker(pool) && !may_start_working(pool); } /* Do I need to be the manager? */ -static bool need_to_manage_workers(struct global_cwq *gcwq) +static bool need_to_manage_workers(struct worker_pool *pool) { - return need_to_create_worker(gcwq) || gcwq->flags & GCWQ_MANAGE_WORKERS; + return need_to_create_worker(pool) || + pool->gcwq->flags & GCWQ_MANAGE_WORKERS; } /* Do we have too many workers and should some go away? */ -static bool too_many_workers(struct global_cwq *gcwq) +static bool too_many_workers(struct worker_pool *pool) { - bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS; - int nr_idle = gcwq->pool.nr_idle + managing; /* manager is considered idle */ - int nr_busy = gcwq->pool.nr_workers - nr_idle; + bool managing = pool->gcwq->flags & GCWQ_MANAGING_WORKERS; + int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ + int nr_busy = pool->nr_workers - nr_idle; return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; } @@ -640,26 +643,26 @@ static bool too_many_workers(struct global_cwq *gcwq) */ /* Return the first worker. Safe with preemption disabled */ -static struct worker *first_worker(struct global_cwq *gcwq) +static struct worker *first_worker(struct worker_pool *pool) { - if (unlikely(list_empty(&gcwq->pool.idle_list))) + if (unlikely(list_empty(&pool->idle_list))) return NULL; - return list_first_entry(&gcwq->pool.idle_list, struct worker, entry); + return list_first_entry(&pool->idle_list, struct worker, entry); } /** * wake_up_worker - wake up an idle worker - * @gcwq: gcwq to wake worker for + * @pool: worker pool to wake worker from * - * Wake up the first idle worker of @gcwq. + * Wake up the first idle worker of @pool. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ -static void wake_up_worker(struct global_cwq *gcwq) +static void wake_up_worker(struct worker_pool *pool) { - struct worker *worker = first_worker(gcwq); + struct worker *worker = first_worker(pool); if (likely(worker)) wake_up_process(worker->task); @@ -681,7 +684,7 @@ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu) struct worker *worker = kthread_data(task); if (!(worker->flags & WORKER_NOT_RUNNING)) - atomic_inc(get_gcwq_nr_running(cpu)); + atomic_inc(get_pool_nr_running(worker->pool)); } /** @@ -704,8 +707,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, { struct worker *worker = kthread_data(task), *to_wakeup = NULL; struct worker_pool *pool = worker->pool; - struct global_cwq *gcwq = pool->gcwq; - atomic_t *nr_running = get_gcwq_nr_running(cpu); + atomic_t *nr_running = get_pool_nr_running(pool); if (worker->flags & WORKER_NOT_RUNNING) return NULL; @@ -725,7 +727,7 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, * without gcwq lock is safe. */ if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist)) - to_wakeup = first_worker(gcwq); + to_wakeup = first_worker(pool); return to_wakeup ? to_wakeup->task : NULL; } @@ -746,7 +748,6 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, bool wakeup) { struct worker_pool *pool = worker->pool; - struct global_cwq *gcwq = pool->gcwq; WARN_ON_ONCE(worker->task != current); @@ -757,12 +758,12 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, */ if ((flags & WORKER_NOT_RUNNING) && !(worker->flags & WORKER_NOT_RUNNING)) { - atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); + atomic_t *nr_running = get_pool_nr_running(pool); if (wakeup) { if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist)) - wake_up_worker(gcwq); + wake_up_worker(pool); } else atomic_dec(nr_running); } @@ -782,7 +783,7 @@ static inline void worker_set_flags(struct worker *worker, unsigned int flags, */ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) { - struct global_cwq *gcwq = worker->pool->gcwq; + struct worker_pool *pool = worker->pool; unsigned int oflags = worker->flags; WARN_ON_ONCE(worker->task != current); @@ -796,7 +797,7 @@ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) */ if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING)) if (!(worker->flags & WORKER_NOT_RUNNING)) - atomic_inc(get_gcwq_nr_running(gcwq->cpu)); + atomic_inc(get_pool_nr_running(pool)); } /** @@ -880,15 +881,15 @@ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, } /** - * gcwq_determine_ins_pos - find insertion position - * @gcwq: gcwq of interest + * pool_determine_ins_pos - find insertion position + * @pool: pool of interest * @cwq: cwq a work is being queued for * - * A work for @cwq is about to be queued on @gcwq, determine insertion + * A work for @cwq is about to be queued on @pool, determine insertion * position for the work. If @cwq is for HIGHPRI wq, the work is * queued at the head of the queue but in FIFO order with respect to * other HIGHPRI works; otherwise, at the end of the queue. This - * function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that + * function also sets GCWQ_HIGHPRI_PENDING flag to hint @pool that * there are HIGHPRI works pending. * * CONTEXT: @@ -897,22 +898,22 @@ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, * RETURNS: * Pointer to inserstion position. */ -static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq, +static inline struct list_head *pool_determine_ins_pos(struct worker_pool *pool, struct cpu_workqueue_struct *cwq) { struct work_struct *twork; if (likely(!(cwq->wq->flags & WQ_HIGHPRI))) - return &gcwq->pool.worklist; + return &pool->worklist; - list_for_each_entry(twork, &gcwq->pool.worklist, entry) { + list_for_each_entry(twork, &pool->worklist, entry) { struct cpu_workqueue_struct *tcwq = get_work_cwq(twork); if (!(tcwq->wq->flags & WQ_HIGHPRI)) break; } - gcwq->flags |= GCWQ_HIGHPRI_PENDING; + pool->gcwq->flags |= GCWQ_HIGHPRI_PENDING; return &twork->entry; } @@ -933,7 +934,7 @@ static void insert_work(struct cpu_workqueue_struct *cwq, struct work_struct *work, struct list_head *head, unsigned int extra_flags) { - struct global_cwq *gcwq = cwq->pool->gcwq; + struct worker_pool *pool = cwq->pool; /* we own @work, set data and link */ set_work_cwq(work, cwq, extra_flags); @@ -953,8 +954,8 @@ static void insert_work(struct cpu_workqueue_struct *cwq, */ smp_mb(); - if (__need_more_worker(gcwq)) - wake_up_worker(gcwq); + if (__need_more_worker(pool)) + wake_up_worker(pool); } /* @@ -1056,7 +1057,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, if (likely(cwq->nr_active < cwq->max_active)) { trace_workqueue_activate_work(work); cwq->nr_active++; - worklist = gcwq_determine_ins_pos(gcwq, cwq); + worklist = pool_determine_ins_pos(cwq->pool, cwq); } else { work_flags |= WORK_STRUCT_DELAYED; worklist = &cwq->delayed_works; @@ -1221,7 +1222,7 @@ static void worker_enter_idle(struct worker *worker) list_add(&worker->entry, &pool->idle_list); if (likely(!(worker->flags & WORKER_ROGUE))) { - if (too_many_workers(gcwq) && !timer_pending(&pool->idle_timer)) + if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); } else @@ -1234,7 +1235,7 @@ static void worker_enter_idle(struct worker *worker) */ WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE && pool->nr_workers == pool->nr_idle && - atomic_read(get_gcwq_nr_running(gcwq->cpu))); + atomic_read(get_pool_nr_running(pool))); } /** @@ -1356,10 +1357,10 @@ static struct worker *alloc_worker(void) /** * create_worker - create a new workqueue worker - * @gcwq: gcwq the new worker will belong to + * @pool: pool the new worker will belong to * @bind: whether to set affinity to @cpu or not * - * Create a new worker which is bound to @gcwq. The returned worker + * Create a new worker which is bound to @pool. The returned worker * can be started by calling start_worker() or destroyed using * destroy_worker(). * @@ -1369,10 +1370,10 @@ static struct worker *alloc_worker(void) * RETURNS: * Pointer to the newly created worker. */ -static struct worker *create_worker(struct global_cwq *gcwq, bool bind) +static struct worker *create_worker(struct worker_pool *pool, bool bind) { + struct global_cwq *gcwq = pool->gcwq; bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; - struct worker_pool *pool = &gcwq->pool; struct worker *worker = NULL; int id = -1; @@ -1480,27 +1481,27 @@ static void destroy_worker(struct worker *worker) ida_remove(&pool->worker_ida, id); } -static void idle_worker_timeout(unsigned long __gcwq) +static void idle_worker_timeout(unsigned long __pool) { - struct global_cwq *gcwq = (void *)__gcwq; + struct worker_pool *pool = (void *)__pool; + struct global_cwq *gcwq = pool->gcwq; spin_lock_irq(&gcwq->lock); - if (too_many_workers(gcwq)) { + if (too_many_workers(pool)) { struct worker *worker; unsigned long expires; /* idle_list is kept in LIFO order, check the last one */ - worker = list_entry(gcwq->pool.idle_list.prev, struct worker, - entry); + worker = list_entry(pool->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) - mod_timer(&gcwq->pool.idle_timer, expires); + mod_timer(&pool->idle_timer, expires); else { /* it's been idle for too long, wake up manager */ gcwq->flags |= GCWQ_MANAGE_WORKERS; - wake_up_worker(gcwq); + wake_up_worker(pool); } } @@ -1526,37 +1527,38 @@ static bool send_mayday(struct work_struct *work) return true; } -static void gcwq_mayday_timeout(unsigned long __gcwq) +static void gcwq_mayday_timeout(unsigned long __pool) { - struct global_cwq *gcwq = (void *)__gcwq; + struct worker_pool *pool = (void *)__pool; + struct global_cwq *gcwq = pool->gcwq; struct work_struct *work; spin_lock_irq(&gcwq->lock); - if (need_to_create_worker(gcwq)) { + if (need_to_create_worker(pool)) { /* * We've been trying to create a new worker but * haven't been successful. We might be hitting an * allocation deadlock. Send distress signals to * rescuers. */ - list_for_each_entry(work, &gcwq->pool.worklist, entry) + list_for_each_entry(work, &pool->worklist, entry) send_mayday(work); } spin_unlock_irq(&gcwq->lock); - mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INTERVAL); + mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL); } /** * maybe_create_worker - create a new worker if necessary - * @gcwq: gcwq to create a new worker for + * @pool: pool to create a new worker for * - * Create a new worker for @gcwq if necessary. @gcwq is guaranteed to + * Create a new worker for @pool if necessary. @pool is guaranteed to * have at least one idle worker on return from this function. If * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is - * sent to all rescuers with works scheduled on @gcwq to resolve + * sent to all rescuers with works scheduled on @pool to resolve * possible allocation deadlock. * * On return, need_to_create_worker() is guaranteed to be false and @@ -1571,52 +1573,54 @@ static void gcwq_mayday_timeout(unsigned long __gcwq) * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */ -static bool maybe_create_worker(struct global_cwq *gcwq) +static bool maybe_create_worker(struct worker_pool *pool) __releases(&gcwq->lock) __acquires(&gcwq->lock) { - if (!need_to_create_worker(gcwq)) + struct global_cwq *gcwq = pool->gcwq; + + if (!need_to_create_worker(pool)) return false; restart: spin_unlock_irq(&gcwq->lock); /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ - mod_timer(&gcwq->pool.mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); + mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); while (true) { struct worker *worker; - worker = create_worker(gcwq, true); + worker = create_worker(pool, true); if (worker) { - del_timer_sync(&gcwq->pool.mayday_timer); + del_timer_sync(&pool->mayday_timer); spin_lock_irq(&gcwq->lock); start_worker(worker); - BUG_ON(need_to_create_worker(gcwq)); + BUG_ON(need_to_create_worker(pool)); return true; } - if (!need_to_create_worker(gcwq)) + if (!need_to_create_worker(pool)) break; __set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(CREATE_COOLDOWN); - if (!need_to_create_worker(gcwq)) + if (!need_to_create_worker(pool)) break; } - del_timer_sync(&gcwq->pool.mayday_timer); + del_timer_sync(&pool->mayday_timer); spin_lock_irq(&gcwq->lock); - if (need_to_create_worker(gcwq)) + if (need_to_create_worker(pool)) goto restart; return true; } /** * maybe_destroy_worker - destroy workers which have been idle for a while - * @gcwq: gcwq to destroy workers for + * @pool: pool to destroy workers for * - * Destroy @gcwq workers which have been idle for longer than + * Destroy @pool workers which have been idle for longer than * IDLE_WORKER_TIMEOUT. * * LOCKING: @@ -1627,20 +1631,19 @@ restart: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */ -static bool maybe_destroy_workers(struct global_cwq *gcwq) +static bool maybe_destroy_workers(struct worker_pool *pool) { bool ret = false; - while (too_many_workers(gcwq)) { + while (too_many_workers(pool)) { struct worker *worker; unsigned long expires; - worker = list_entry(gcwq->pool.idle_list.prev, struct worker, - entry); + worker = list_entry(pool->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) { - mod_timer(&gcwq->pool.idle_timer, expires); + mod_timer(&pool->idle_timer, expires); break; } @@ -1673,7 +1676,8 @@ static bool maybe_destroy_workers(struct global_cwq *gcwq) */ static bool manage_workers(struct worker *worker) { - struct global_cwq *gcwq = worker->pool->gcwq; + struct worker_pool *pool = worker->pool; + struct global_cwq *gcwq = pool->gcwq; bool ret = false; if (gcwq->flags & GCWQ_MANAGING_WORKERS) @@ -1686,8 +1690,8 @@ static bool manage_workers(struct worker *worker) * Destroy and then create so that may_start_working() is true * on return. */ - ret |= maybe_destroy_workers(gcwq); - ret |= maybe_create_worker(gcwq); + ret |= maybe_destroy_workers(pool); + ret |= maybe_create_worker(pool); gcwq->flags &= ~GCWQ_MANAGING_WORKERS; @@ -1746,7 +1750,7 @@ static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) { struct work_struct *work = list_first_entry(&cwq->delayed_works, struct work_struct, entry); - struct list_head *pos = gcwq_determine_ins_pos(cwq->pool->gcwq, cwq); + struct list_head *pos = pool_determine_ins_pos(cwq->pool, cwq); trace_workqueue_activate_work(work); move_linked_works(work, pos, NULL); @@ -1874,7 +1878,7 @@ __acquires(&gcwq->lock) if (!list_empty(&pool->worklist) && get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) - wake_up_worker(gcwq); + wake_up_worker(pool); else gcwq->flags &= ~GCWQ_HIGHPRI_PENDING; } @@ -1890,8 +1894,8 @@ __acquires(&gcwq->lock) * Unbound gcwq isn't concurrency managed and work items should be * executed ASAP. Wake up another worker if necessary. */ - if ((worker->flags & WORKER_UNBOUND) && need_more_worker(gcwq)) - wake_up_worker(gcwq); + if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool)) + wake_up_worker(pool); spin_unlock_irq(&gcwq->lock); @@ -1983,11 +1987,11 @@ woke_up: worker_leave_idle(worker); recheck: /* no more worker necessary? */ - if (!need_more_worker(gcwq)) + if (!need_more_worker(pool)) goto sleep; /* do we need to manage? */ - if (unlikely(!may_start_working(gcwq)) && manage_workers(worker)) + if (unlikely(!may_start_working(pool)) && manage_workers(worker)) goto recheck; /* @@ -2018,11 +2022,11 @@ recheck: move_linked_works(work, &worker->scheduled, NULL); process_scheduled_works(worker); } - } while (keep_working(gcwq)); + } while (keep_working(pool)); worker_set_flags(worker, WORKER_PREP, false); sleep: - if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker)) + if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker)) goto recheck; /* @@ -2107,8 +2111,8 @@ repeat: * regular worker; otherwise, we end up with 0 concurrency * and stalling the execution. */ - if (keep_working(gcwq)) - wake_up_worker(gcwq); + if (keep_working(pool)) + wake_up_worker(pool); spin_unlock_irq(&gcwq->lock); } @@ -3383,7 +3387,7 @@ static int __cpuinit trustee_thread(void *__gcwq) * keep_working() are always true as long as the worklist is * not empty. */ - atomic_set(get_gcwq_nr_running(gcwq->cpu), 0); + atomic_set(get_pool_nr_running(&gcwq->pool), 0); spin_unlock_irq(&gcwq->lock); del_timer_sync(&gcwq->pool.idle_timer); @@ -3424,9 +3428,9 @@ static int __cpuinit trustee_thread(void *__gcwq) wake_up_process(worker->task); } - if (need_to_create_worker(gcwq)) { + if (need_to_create_worker(&gcwq->pool)) { spin_unlock_irq(&gcwq->lock); - worker = create_worker(gcwq, false); + worker = create_worker(&gcwq->pool, false); spin_lock_irq(&gcwq->lock); if (worker) { worker->flags |= WORKER_ROGUE; @@ -3540,7 +3544,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, /* fall through */ case CPU_UP_PREPARE: BUG_ON(gcwq->pool.first_idle); - new_worker = create_worker(gcwq, false); + new_worker = create_worker(&gcwq->pool, false); if (!new_worker) { if (new_trustee) kthread_stop(new_trustee); @@ -3788,7 +3792,7 @@ void thaw_workqueues(void) cwq_activate_first_delayed(cwq); } - wake_up_worker(gcwq); + wake_up_worker(&gcwq->pool); spin_unlock_irq(&gcwq->lock); } @@ -3822,10 +3826,10 @@ static int __init init_workqueues(void) init_timer_deferrable(&gcwq->pool.idle_timer); gcwq->pool.idle_timer.function = idle_worker_timeout; - gcwq->pool.idle_timer.data = (unsigned long)gcwq; + gcwq->pool.idle_timer.data = (unsigned long)&gcwq->pool; setup_timer(&gcwq->pool.mayday_timer, gcwq_mayday_timeout, - (unsigned long)gcwq); + (unsigned long)&gcwq->pool); ida_init(&gcwq->pool.worker_ida); @@ -3840,7 +3844,7 @@ static int __init init_workqueues(void) if (cpu != WORK_CPU_UNBOUND) gcwq->flags &= ~GCWQ_DISASSOCIATED; - worker = create_worker(gcwq, true); + worker = create_worker(&gcwq->pool, true); BUG_ON(!worker); spin_lock_irq(&gcwq->lock); start_worker(worker); -- cgit v1.2.3-55-g7522 From 11ebea50dbc1ade5994b2c838a096078d4c02399 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 12 Jul 2012 14:46:37 -0700 Subject: workqueue: separate out worker_pool flags GCWQ_MANAGE_WORKERS, GCWQ_MANAGING_WORKERS and GCWQ_HIGHPRI_PENDING are per-pool properties. Add worker_pool->flags and make the above three flags per-pool flags. The changes in this patch are mechanical and don't caues any functional difference. This is to prepare for multiple pools per gcwq. Signed-off-by: Tejun Heo --- kernel/workqueue.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 2d82f7b193a0..7a98bae635fa 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -46,11 +46,13 @@ enum { /* global_cwq flags */ - GCWQ_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ - GCWQ_MANAGING_WORKERS = 1 << 1, /* managing workers */ - GCWQ_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */ - GCWQ_FREEZING = 1 << 3, /* freeze in progress */ - GCWQ_HIGHPRI_PENDING = 1 << 4, /* highpri works on queue */ + GCWQ_DISASSOCIATED = 1 << 0, /* cpu can't serve workers */ + GCWQ_FREEZING = 1 << 1, /* freeze in progress */ + + /* pool flags */ + POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ + POOL_MANAGING_WORKERS = 1 << 1, /* managing workers */ + POOL_HIGHPRI_PENDING = 1 << 2, /* highpri works on queue */ /* worker flags */ WORKER_STARTED = 1 << 0, /* started */ @@ -142,6 +144,7 @@ struct worker { struct worker_pool { struct global_cwq *gcwq; /* I: the owning gcwq */ + unsigned int flags; /* X: flags */ struct list_head worklist; /* L: list of pending works */ int nr_workers; /* L: total number of workers */ @@ -583,7 +586,7 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) static bool __need_more_worker(struct worker_pool *pool) { return !atomic_read(get_pool_nr_running(pool)) || - pool->gcwq->flags & GCWQ_HIGHPRI_PENDING; + (pool->flags & POOL_HIGHPRI_PENDING); } /* @@ -612,7 +615,7 @@ static bool keep_working(struct worker_pool *pool) return !list_empty(&pool->worklist) && (atomic_read(nr_running) <= 1 || - pool->gcwq->flags & GCWQ_HIGHPRI_PENDING); + (pool->flags & POOL_HIGHPRI_PENDING)); } /* Do we need a new worker? Called from manager. */ @@ -625,13 +628,13 @@ static bool need_to_create_worker(struct worker_pool *pool) static bool need_to_manage_workers(struct worker_pool *pool) { return need_to_create_worker(pool) || - pool->gcwq->flags & GCWQ_MANAGE_WORKERS; + (pool->flags & POOL_MANAGE_WORKERS); } /* Do we have too many workers and should some go away? */ static bool too_many_workers(struct worker_pool *pool) { - bool managing = pool->gcwq->flags & GCWQ_MANAGING_WORKERS; + bool managing = pool->flags & POOL_MANAGING_WORKERS; int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ int nr_busy = pool->nr_workers - nr_idle; @@ -889,7 +892,7 @@ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, * position for the work. If @cwq is for HIGHPRI wq, the work is * queued at the head of the queue but in FIFO order with respect to * other HIGHPRI works; otherwise, at the end of the queue. This - * function also sets GCWQ_HIGHPRI_PENDING flag to hint @pool that + * function also sets POOL_HIGHPRI_PENDING flag to hint @pool that * there are HIGHPRI works pending. * * CONTEXT: @@ -913,7 +916,7 @@ static inline struct list_head *pool_determine_ins_pos(struct worker_pool *pool, break; } - pool->gcwq->flags |= GCWQ_HIGHPRI_PENDING; + pool->flags |= POOL_HIGHPRI_PENDING; return &twork->entry; } @@ -1500,7 +1503,7 @@ static void idle_worker_timeout(unsigned long __pool) mod_timer(&pool->idle_timer, expires); else { /* it's been idle for too long, wake up manager */ - gcwq->flags |= GCWQ_MANAGE_WORKERS; + pool->flags |= POOL_MANAGE_WORKERS; wake_up_worker(pool); } } @@ -1680,11 +1683,11 @@ static bool manage_workers(struct worker *worker) struct global_cwq *gcwq = pool->gcwq; bool ret = false; - if (gcwq->flags & GCWQ_MANAGING_WORKERS) + if (pool->flags & POOL_MANAGING_WORKERS) return ret; - gcwq->flags &= ~GCWQ_MANAGE_WORKERS; - gcwq->flags |= GCWQ_MANAGING_WORKERS; + pool->flags &= ~POOL_MANAGE_WORKERS; + pool->flags |= POOL_MANAGING_WORKERS; /* * Destroy and then create so that may_start_working() is true @@ -1693,7 +1696,7 @@ static bool manage_workers(struct worker *worker) ret |= maybe_destroy_workers(pool); ret |= maybe_create_worker(pool); - gcwq->flags &= ~GCWQ_MANAGING_WORKERS; + pool->flags &= ~POOL_MANAGING_WORKERS; /* * The trustee might be waiting to take over the manager @@ -1872,7 +1875,7 @@ __acquires(&gcwq->lock) * If HIGHPRI_PENDING, check the next work, and, if HIGHPRI, * wake up another worker; otherwise, clear HIGHPRI_PENDING. */ - if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) { + if (unlikely(pool->flags & POOL_HIGHPRI_PENDING)) { struct work_struct *nwork = list_first_entry(&pool->worklist, struct work_struct, entry); @@ -1880,7 +1883,7 @@ __acquires(&gcwq->lock) get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) wake_up_worker(pool); else - gcwq->flags &= ~GCWQ_HIGHPRI_PENDING; + pool->flags &= ~POOL_HIGHPRI_PENDING; } /* @@ -3360,10 +3363,10 @@ static int __cpuinit trustee_thread(void *__gcwq) * cancelled. */ BUG_ON(gcwq->cpu != smp_processor_id()); - rc = trustee_wait_event(!(gcwq->flags & GCWQ_MANAGING_WORKERS)); + rc = trustee_wait_event(!(gcwq->pool.flags & POOL_MANAGING_WORKERS)); BUG_ON(rc < 0); - gcwq->flags |= GCWQ_MANAGING_WORKERS; + gcwq->pool.flags |= POOL_MANAGING_WORKERS; list_for_each_entry(worker, &gcwq->pool.idle_list, entry) worker->flags |= WORKER_ROGUE; @@ -3487,7 +3490,7 @@ static int __cpuinit trustee_thread(void *__gcwq) } /* relinquish manager role */ - gcwq->flags &= ~GCWQ_MANAGING_WORKERS; + gcwq->pool.flags &= ~POOL_MANAGING_WORKERS; /* notify completion */ gcwq->trustee = NULL; @@ -3604,7 +3607,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, spin_unlock_irq(&gcwq->lock); kthread_bind(gcwq->pool.first_idle->task, cpu); spin_lock_irq(&gcwq->lock); - gcwq->flags |= GCWQ_MANAGE_WORKERS; + gcwq->pool.flags |= POOL_MANAGE_WORKERS; start_worker(gcwq->pool.first_idle); gcwq->pool.first_idle = NULL; break; -- cgit v1.2.3-55-g7522 From 4ce62e9e30cacc26885cab133ad1de358dd79f21 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 13 Jul 2012 22:16:44 -0700 Subject: workqueue: introduce NR_WORKER_POOLS and for_each_worker_pool() Introduce NR_WORKER_POOLS and for_each_worker_pool() and convert code paths which need to manipulate all pools in a gcwq to use them. NR_WORKER_POOLS is currently one and for_each_worker_pool() iterates over only @gcwq->pool. Note that nr_running is per-pool property and converted to an array with NR_WORKER_POOLS elements and renamed to pool_nr_running. Note that get_pool_nr_running() currently assumes 0 index. The next patch will make use of non-zero index. The changes in this patch are mechanical and don't caues any functional difference. This is to prepare for multiple pools per gcwq. v2: nr_running indexing bug in get_pool_nr_running() fixed. v3: Pointer to array is stupid. Don't use it in get_pool_nr_running() as suggested by Linus. Signed-off-by: Tejun Heo Cc: Tony Luck Cc: Fengguang Wu Cc: Linus Torvalds --- kernel/workqueue.c | 223 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 153 insertions(+), 70 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 7a98bae635fa..b0daaea44eaa 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -74,6 +74,8 @@ enum { TRUSTEE_RELEASE = 3, /* release workers */ TRUSTEE_DONE = 4, /* trustee is done */ + NR_WORKER_POOLS = 1, /* # worker pools per gcwq */ + BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */ BUSY_WORKER_HASH_SIZE = 1 << BUSY_WORKER_HASH_ORDER, BUSY_WORKER_HASH_MASK = BUSY_WORKER_HASH_SIZE - 1, @@ -274,6 +276,9 @@ EXPORT_SYMBOL_GPL(system_nrt_freezable_wq); #define CREATE_TRACE_POINTS #include +#define for_each_worker_pool(pool, gcwq) \ + for ((pool) = &(gcwq)->pool; (pool); (pool) = NULL) + #define for_each_busy_worker(worker, i, pos, gcwq) \ for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \ hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry) @@ -454,7 +459,7 @@ static bool workqueue_freezing; /* W: have wqs started freezing? */ * try_to_wake_up(). Put it in a separate cacheline. */ static DEFINE_PER_CPU(struct global_cwq, global_cwq); -static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); +static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, pool_nr_running[NR_WORKER_POOLS]); /* * Global cpu workqueue and nr_running counter for unbound gcwq. The @@ -462,7 +467,9 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); * workers have WORKER_UNBOUND set. */ static struct global_cwq unbound_global_cwq; -static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0); /* always 0 */ +static atomic_t unbound_pool_nr_running[NR_WORKER_POOLS] = { + [0 ... NR_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */ +}; static int worker_thread(void *__worker); @@ -477,11 +484,12 @@ static struct global_cwq *get_gcwq(unsigned int cpu) static atomic_t *get_pool_nr_running(struct worker_pool *pool) { int cpu = pool->gcwq->cpu; + int idx = 0; if (cpu != WORK_CPU_UNBOUND) - return &per_cpu(gcwq_nr_running, cpu); + return &per_cpu(pool_nr_running, cpu)[idx]; else - return &unbound_gcwq_nr_running; + return &unbound_pool_nr_running[idx]; } static struct cpu_workqueue_struct *get_cwq(unsigned int cpu, @@ -3345,9 +3353,30 @@ EXPORT_SYMBOL_GPL(work_busy); __ret1 < 0 ? -1 : 0; \ }) +static bool gcwq_is_managing_workers(struct global_cwq *gcwq) +{ + struct worker_pool *pool; + + for_each_worker_pool(pool, gcwq) + if (pool->flags & POOL_MANAGING_WORKERS) + return true; + return false; +} + +static bool gcwq_has_idle_workers(struct global_cwq *gcwq) +{ + struct worker_pool *pool; + + for_each_worker_pool(pool, gcwq) + if (!list_empty(&pool->idle_list)) + return true; + return false; +} + static int __cpuinit trustee_thread(void *__gcwq) { struct global_cwq *gcwq = __gcwq; + struct worker_pool *pool; struct worker *worker; struct work_struct *work; struct hlist_node *pos; @@ -3363,13 +3392,15 @@ static int __cpuinit trustee_thread(void *__gcwq) * cancelled. */ BUG_ON(gcwq->cpu != smp_processor_id()); - rc = trustee_wait_event(!(gcwq->pool.flags & POOL_MANAGING_WORKERS)); + rc = trustee_wait_event(!gcwq_is_managing_workers(gcwq)); BUG_ON(rc < 0); - gcwq->pool.flags |= POOL_MANAGING_WORKERS; + for_each_worker_pool(pool, gcwq) { + pool->flags |= POOL_MANAGING_WORKERS; - list_for_each_entry(worker, &gcwq->pool.idle_list, entry) - worker->flags |= WORKER_ROGUE; + list_for_each_entry(worker, &pool->idle_list, entry) + worker->flags |= WORKER_ROGUE; + } for_each_busy_worker(worker, i, pos, gcwq) worker->flags |= WORKER_ROGUE; @@ -3390,10 +3421,12 @@ static int __cpuinit trustee_thread(void *__gcwq) * keep_working() are always true as long as the worklist is * not empty. */ - atomic_set(get_pool_nr_running(&gcwq->pool), 0); + for_each_worker_pool(pool, gcwq) + atomic_set(get_pool_nr_running(pool), 0); spin_unlock_irq(&gcwq->lock); - del_timer_sync(&gcwq->pool.idle_timer); + for_each_worker_pool(pool, gcwq) + del_timer_sync(&pool->idle_timer); spin_lock_irq(&gcwq->lock); /* @@ -3415,29 +3448,38 @@ static int __cpuinit trustee_thread(void *__gcwq) * may be frozen works in freezable cwqs. Don't declare * completion while frozen. */ - while (gcwq->pool.nr_workers != gcwq->pool.nr_idle || - gcwq->flags & GCWQ_FREEZING || - gcwq->trustee_state == TRUSTEE_IN_CHARGE) { - int nr_works = 0; + while (true) { + bool busy = false; - list_for_each_entry(work, &gcwq->pool.worklist, entry) { - send_mayday(work); - nr_works++; - } + for_each_worker_pool(pool, gcwq) + busy |= pool->nr_workers != pool->nr_idle; - list_for_each_entry(worker, &gcwq->pool.idle_list, entry) { - if (!nr_works--) - break; - wake_up_process(worker->task); - } + if (!busy && !(gcwq->flags & GCWQ_FREEZING) && + gcwq->trustee_state != TRUSTEE_IN_CHARGE) + break; - if (need_to_create_worker(&gcwq->pool)) { - spin_unlock_irq(&gcwq->lock); - worker = create_worker(&gcwq->pool, false); - spin_lock_irq(&gcwq->lock); - if (worker) { - worker->flags |= WORKER_ROGUE; - start_worker(worker); + for_each_worker_pool(pool, gcwq) { + int nr_works = 0; + + list_for_each_entry(work, &pool->worklist, entry) { + send_mayday(work); + nr_works++; + } + + list_for_each_entry(worker, &pool->idle_list, entry) { + if (!nr_works--) + break; + wake_up_process(worker->task); + } + + if (need_to_create_worker(pool)) { + spin_unlock_irq(&gcwq->lock); + worker = create_worker(pool, false); + spin_lock_irq(&gcwq->lock); + if (worker) { + worker->flags |= WORKER_ROGUE; + start_worker(worker); + } } } @@ -3452,11 +3494,18 @@ static int __cpuinit trustee_thread(void *__gcwq) * all workers till we're canceled. */ do { - rc = trustee_wait_event(!list_empty(&gcwq->pool.idle_list)); - while (!list_empty(&gcwq->pool.idle_list)) - destroy_worker(list_first_entry(&gcwq->pool.idle_list, - struct worker, entry)); - } while (gcwq->pool.nr_workers && rc >= 0); + rc = trustee_wait_event(gcwq_has_idle_workers(gcwq)); + + i = 0; + for_each_worker_pool(pool, gcwq) { + while (!list_empty(&pool->idle_list)) { + worker = list_first_entry(&pool->idle_list, + struct worker, entry); + destroy_worker(worker); + } + i |= pool->nr_workers; + } + } while (i && rc >= 0); /* * At this point, either draining has completed and no worker @@ -3465,7 +3514,8 @@ static int __cpuinit trustee_thread(void *__gcwq) * Tell the remaining busy ones to rebind once it finishes the * currently scheduled works by scheduling the rebind_work. */ - WARN_ON(!list_empty(&gcwq->pool.idle_list)); + for_each_worker_pool(pool, gcwq) + WARN_ON(!list_empty(&pool->idle_list)); for_each_busy_worker(worker, i, pos, gcwq) { struct work_struct *rebind_work = &worker->rebind_work; @@ -3490,7 +3540,8 @@ static int __cpuinit trustee_thread(void *__gcwq) } /* relinquish manager role */ - gcwq->pool.flags &= ~POOL_MANAGING_WORKERS; + for_each_worker_pool(pool, gcwq) + pool->flags &= ~POOL_MANAGING_WORKERS; /* notify completion */ gcwq->trustee = NULL; @@ -3532,8 +3583,10 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, unsigned int cpu = (unsigned long)hcpu; struct global_cwq *gcwq = get_gcwq(cpu); struct task_struct *new_trustee = NULL; - struct worker *uninitialized_var(new_worker); + struct worker *new_workers[NR_WORKER_POOLS] = { }; + struct worker_pool *pool; unsigned long flags; + int i; action &= ~CPU_TASKS_FROZEN; @@ -3546,12 +3599,12 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, kthread_bind(new_trustee, cpu); /* fall through */ case CPU_UP_PREPARE: - BUG_ON(gcwq->pool.first_idle); - new_worker = create_worker(&gcwq->pool, false); - if (!new_worker) { - if (new_trustee) - kthread_stop(new_trustee); - return NOTIFY_BAD; + i = 0; + for_each_worker_pool(pool, gcwq) { + BUG_ON(pool->first_idle); + new_workers[i] = create_worker(pool, false); + if (!new_workers[i++]) + goto err_destroy; } } @@ -3568,8 +3621,11 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); /* fall through */ case CPU_UP_PREPARE: - BUG_ON(gcwq->pool.first_idle); - gcwq->pool.first_idle = new_worker; + i = 0; + for_each_worker_pool(pool, gcwq) { + BUG_ON(pool->first_idle); + pool->first_idle = new_workers[i++]; + } break; case CPU_DYING: @@ -3586,8 +3642,10 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, gcwq->trustee_state = TRUSTEE_BUTCHER; /* fall through */ case CPU_UP_CANCELED: - destroy_worker(gcwq->pool.first_idle); - gcwq->pool.first_idle = NULL; + for_each_worker_pool(pool, gcwq) { + destroy_worker(pool->first_idle); + pool->first_idle = NULL; + } break; case CPU_DOWN_FAILED: @@ -3604,18 +3662,32 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, * Put the first_idle in and request a real manager to * take a look. */ - spin_unlock_irq(&gcwq->lock); - kthread_bind(gcwq->pool.first_idle->task, cpu); - spin_lock_irq(&gcwq->lock); - gcwq->pool.flags |= POOL_MANAGE_WORKERS; - start_worker(gcwq->pool.first_idle); - gcwq->pool.first_idle = NULL; + for_each_worker_pool(pool, gcwq) { + spin_unlock_irq(&gcwq->lock); + kthread_bind(pool->first_idle->task, cpu); + spin_lock_irq(&gcwq->lock); + pool->flags |= POOL_MANAGE_WORKERS; + start_worker(pool->first_idle); + pool->first_idle = NULL; + } break; } spin_unlock_irqrestore(&gcwq->lock, flags); return notifier_from_errno(0); + +err_destroy: + if (new_trustee) + kthread_stop(new_trustee); + + spin_lock_irqsave(&gcwq->lock, flags); + for (i = 0; i < NR_WORKER_POOLS; i++) + if (new_workers[i]) + destroy_worker(new_workers[i]); + spin_unlock_irqrestore(&gcwq->lock, flags); + + return NOTIFY_BAD; } #ifdef CONFIG_SMP @@ -3774,6 +3846,7 @@ void thaw_workqueues(void) for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); + struct worker_pool *pool; struct workqueue_struct *wq; spin_lock_irq(&gcwq->lock); @@ -3795,7 +3868,8 @@ void thaw_workqueues(void) cwq_activate_first_delayed(cwq); } - wake_up_worker(&gcwq->pool); + for_each_worker_pool(pool, gcwq) + wake_up_worker(pool); spin_unlock_irq(&gcwq->lock); } @@ -3816,25 +3890,29 @@ static int __init init_workqueues(void) /* initialize gcwqs */ for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); + struct worker_pool *pool; spin_lock_init(&gcwq->lock); - gcwq->pool.gcwq = gcwq; - INIT_LIST_HEAD(&gcwq->pool.worklist); gcwq->cpu = cpu; gcwq->flags |= GCWQ_DISASSOCIATED; - INIT_LIST_HEAD(&gcwq->pool.idle_list); for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) INIT_HLIST_HEAD(&gcwq->busy_hash[i]); - init_timer_deferrable(&gcwq->pool.idle_timer); - gcwq->pool.idle_timer.function = idle_worker_timeout; - gcwq->pool.idle_timer.data = (unsigned long)&gcwq->pool; + for_each_worker_pool(pool, gcwq) { + pool->gcwq = gcwq; + INIT_LIST_HEAD(&pool->worklist); + INIT_LIST_HEAD(&pool->idle_list); - setup_timer(&gcwq->pool.mayday_timer, gcwq_mayday_timeout, - (unsigned long)&gcwq->pool); + init_timer_deferrable(&pool->idle_timer); + pool->idle_timer.function = idle_worker_timeout; + pool->idle_timer.data = (unsigned long)pool; - ida_init(&gcwq->pool.worker_ida); + setup_timer(&pool->mayday_timer, gcwq_mayday_timeout, + (unsigned long)pool); + + ida_init(&pool->worker_ida); + } gcwq->trustee_state = TRUSTEE_DONE; init_waitqueue_head(&gcwq->trustee_wait); @@ -3843,15 +3921,20 @@ static int __init init_workqueues(void) /* create the initial worker */ for_each_online_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); - struct worker *worker; + struct worker_pool *pool; if (cpu != WORK_CPU_UNBOUND) gcwq->flags &= ~GCWQ_DISASSOCIATED; - worker = create_worker(&gcwq->pool, true); - BUG_ON(!worker); - spin_lock_irq(&gcwq->lock); - start_worker(worker); - spin_unlock_irq(&gcwq->lock); + + for_each_worker_pool(pool, gcwq) { + struct worker *worker; + + worker = create_worker(pool, true); + BUG_ON(!worker); + spin_lock_irq(&gcwq->lock); + start_worker(worker); + spin_unlock_irq(&gcwq->lock); + } } system_wq = alloc_workqueue("events", 0, 0); -- cgit v1.2.3-55-g7522 From 3270476a6c0ce322354df8679652f060d66526dc Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 13 Jul 2012 22:16:45 -0700 Subject: workqueue: reimplement WQ_HIGHPRI using a separate worker_pool WQ_HIGHPRI was implemented by queueing highpri work items at the head of the global worklist. Other than queueing at the head, they weren't handled differently; unfortunately, this could lead to execution latency of a few seconds on heavily loaded systems. Now that workqueue code has been updated to deal with multiple worker_pools per global_cwq, this patch reimplements WQ_HIGHPRI using a separate worker_pool. NR_WORKER_POOLS is bumped to two and gcwq->pools[0] is used for normal pri work items and ->pools[1] for highpri. Highpri workers get -20 nice level and has 'H' suffix in their names. Note that this change increases the number of kworkers per cpu. POOL_HIGHPRI_PENDING, pool_determine_ins_pos() and highpri chain wakeup code in process_one_work() are no longer used and removed. This allows proper prioritization of highpri work items and removes high execution latency of highpri work items. v2: nr_running indexing bug in get_pool_nr_running() fixed. v3: Refreshed for the get_pool_nr_running() update in the previous patch. Signed-off-by: Tejun Heo Reported-by: Josh Hunt LKML-Reference: Cc: Tony Luck Cc: Fengguang Wu --- Documentation/workqueue.txt | 103 ++++++++++++++++---------------------------- kernel/workqueue.c | 100 ++++++++++++------------------------------ 2 files changed, 65 insertions(+), 138 deletions(-) (limited to 'kernel') diff --git a/Documentation/workqueue.txt b/Documentation/workqueue.txt index a0b577de918f..a6ab4b62d926 100644 --- a/Documentation/workqueue.txt +++ b/Documentation/workqueue.txt @@ -89,25 +89,28 @@ called thread-pools. The cmwq design differentiates between the user-facing workqueues that subsystems and drivers queue work items on and the backend mechanism -which manages thread-pool and processes the queued work items. +which manages thread-pools and processes the queued work items. The backend is called gcwq. There is one gcwq for each possible CPU -and one gcwq to serve work items queued on unbound workqueues. +and one gcwq to serve work items queued on unbound workqueues. Each +gcwq has two thread-pools - one for normal work items and the other +for high priority ones. Subsystems and drivers can create and queue work items through special workqueue API functions as they see fit. They can influence some aspects of the way the work items are executed by setting flags on the workqueue they are putting the work item on. These flags include -things like CPU locality, reentrancy, concurrency limits and more. To -get a detailed overview refer to the API description of +things like CPU locality, reentrancy, concurrency limits, priority and +more. To get a detailed overview refer to the API description of alloc_workqueue() below. -When a work item is queued to a workqueue, the target gcwq is -determined according to the queue parameters and workqueue attributes -and appended on the shared worklist of the gcwq. For example, unless -specifically overridden, a work item of a bound workqueue will be -queued on the worklist of exactly that gcwq that is associated to the -CPU the issuer is running on. +When a work item is queued to a workqueue, the target gcwq and +thread-pool is determined according to the queue parameters and +workqueue attributes and appended on the shared worklist of the +thread-pool. For example, unless specifically overridden, a work item +of a bound workqueue will be queued on the worklist of either normal +or highpri thread-pool of the gcwq that is associated to the CPU the +issuer is running on. For any worker pool implementation, managing the concurrency level (how many execution contexts are active) is an important issue. cmwq @@ -115,26 +118,26 @@ tries to keep the concurrency at a minimal but sufficient level. Minimal to save resources and sufficient in that the system is used at its full capacity. -Each gcwq bound to an actual CPU implements concurrency management by -hooking into the scheduler. The gcwq is notified whenever an active -worker wakes up or sleeps and keeps track of the number of the -currently runnable workers. Generally, work items are not expected to -hog a CPU and consume many cycles. That means maintaining just enough -concurrency to prevent work processing from stalling should be -optimal. As long as there are one or more runnable workers on the -CPU, the gcwq doesn't start execution of a new work, but, when the -last running worker goes to sleep, it immediately schedules a new -worker so that the CPU doesn't sit idle while there are pending work -items. This allows using a minimal number of workers without losing -execution bandwidth. +Each thread-pool bound to an actual CPU implements concurrency +management by hooking into the scheduler. The thread-pool is notified +whenever an active worker wakes up or sleeps and keeps track of the +number of the currently runnable workers. Generally, work items are +not expected to hog a CPU and consume many cycles. That means +maintaining just enough concurrency to prevent work processing from +stalling should be optimal. As long as there are one or more runnable +workers on the CPU, the thread-pool doesn't start execution of a new +work, but, when the last running worker goes to sleep, it immediately +schedules a new worker so that the CPU doesn't sit idle while there +are pending work items. This allows using a minimal number of workers +without losing execution bandwidth. Keeping idle workers around doesn't cost other than the memory space for kthreads, so cmwq holds onto idle ones for a while before killing them. For an unbound wq, the above concurrency management doesn't apply and -the gcwq for the pseudo unbound CPU tries to start executing all work -items as soon as possible. The responsibility of regulating +the thread-pools for the pseudo unbound CPU try to start executing all +work items as soon as possible. The responsibility of regulating concurrency level is on the users. There is also a flag to mark a bound wq to ignore the concurrency management. Please refer to the API section for details. @@ -205,31 +208,22 @@ resources, scheduled and executed. WQ_HIGHPRI - Work items of a highpri wq are queued at the head of the - worklist of the target gcwq and start execution regardless of - the current concurrency level. In other words, highpri work - items will always start execution as soon as execution - resource is available. + Work items of a highpri wq are queued to the highpri + thread-pool of the target gcwq. Highpri thread-pools are + served by worker threads with elevated nice level. - Ordering among highpri work items is preserved - a highpri - work item queued after another highpri work item will start - execution after the earlier highpri work item starts. - - Although highpri work items are not held back by other - runnable work items, they still contribute to the concurrency - level. Highpri work items in runnable state will prevent - non-highpri work items from starting execution. - - This flag is meaningless for unbound wq. + Note that normal and highpri thread-pools don't interact with + each other. Each maintain its separate pool of workers and + implements concurrency management among its workers. WQ_CPU_INTENSIVE Work items of a CPU intensive wq do not contribute to the concurrency level. In other words, runnable CPU intensive - work items will not prevent other work items from starting - execution. This is useful for bound work items which are - expected to hog CPU cycles so that their execution is - regulated by the system scheduler. + work items will not prevent other work items in the same + thread-pool from starting execution. This is useful for bound + work items which are expected to hog CPU cycles so that their + execution is regulated by the system scheduler. Although CPU intensive work items don't contribute to the concurrency level, start of their executions is still @@ -239,14 +233,6 @@ resources, scheduled and executed. This flag is meaningless for unbound wq. - WQ_HIGHPRI | WQ_CPU_INTENSIVE - - This combination makes the wq avoid interaction with - concurrency management completely and behave as a simple - per-CPU execution context provider. Work items queued on a - highpri CPU-intensive wq start execution as soon as resources - are available and don't affect execution of other work items. - @max_active: @max_active determines the maximum number of execution contexts per @@ -328,20 +314,7 @@ If @max_active == 2, 35 w2 wakes up and finishes Now, let's assume w1 and w2 are queued to a different wq q1 which has -WQ_HIGHPRI set, - - TIME IN MSECS EVENT - 0 w1 and w2 start and burn CPU - 5 w1 sleeps - 10 w2 sleeps - 10 w0 starts and burns CPU - 15 w0 sleeps - 15 w1 wakes up and finishes - 20 w2 wakes up and finishes - 25 w0 wakes up and burns CPU - 30 w0 finishes - -If q1 has WQ_CPU_INTENSIVE set, +WQ_CPU_INTENSIVE set, TIME IN MSECS EVENT 0 w0 starts and burns CPU diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b0daaea44eaa..4fa9e3552f1e 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -52,7 +52,6 @@ enum { /* pool flags */ POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ POOL_MANAGING_WORKERS = 1 << 1, /* managing workers */ - POOL_HIGHPRI_PENDING = 1 << 2, /* highpri works on queue */ /* worker flags */ WORKER_STARTED = 1 << 0, /* started */ @@ -74,7 +73,7 @@ enum { TRUSTEE_RELEASE = 3, /* release workers */ TRUSTEE_DONE = 4, /* trustee is done */ - NR_WORKER_POOLS = 1, /* # worker pools per gcwq */ + NR_WORKER_POOLS = 2, /* # worker pools per gcwq */ BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */ BUSY_WORKER_HASH_SIZE = 1 << BUSY_WORKER_HASH_ORDER, @@ -95,6 +94,7 @@ enum { * all cpus. Give -20. */ RESCUER_NICE_LEVEL = -20, + HIGHPRI_NICE_LEVEL = -20, }; /* @@ -174,7 +174,7 @@ struct global_cwq { struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE]; /* L: hash of busy workers */ - struct worker_pool pool; /* the worker pools */ + struct worker_pool pools[2]; /* normal and highpri pools */ struct task_struct *trustee; /* L: for gcwq shutdown */ unsigned int trustee_state; /* L: trustee state */ @@ -277,7 +277,8 @@ EXPORT_SYMBOL_GPL(system_nrt_freezable_wq); #include #define for_each_worker_pool(pool, gcwq) \ - for ((pool) = &(gcwq)->pool; (pool); (pool) = NULL) + for ((pool) = &(gcwq)->pools[0]; \ + (pool) < &(gcwq)->pools[NR_WORKER_POOLS]; (pool)++) #define for_each_busy_worker(worker, i, pos, gcwq) \ for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \ @@ -473,6 +474,11 @@ static atomic_t unbound_pool_nr_running[NR_WORKER_POOLS] = { static int worker_thread(void *__worker); +static int worker_pool_pri(struct worker_pool *pool) +{ + return pool - pool->gcwq->pools; +} + static struct global_cwq *get_gcwq(unsigned int cpu) { if (cpu != WORK_CPU_UNBOUND) @@ -484,7 +490,7 @@ static struct global_cwq *get_gcwq(unsigned int cpu) static atomic_t *get_pool_nr_running(struct worker_pool *pool) { int cpu = pool->gcwq->cpu; - int idx = 0; + int idx = worker_pool_pri(pool); if (cpu != WORK_CPU_UNBOUND) return &per_cpu(pool_nr_running, cpu)[idx]; @@ -586,15 +592,14 @@ static struct global_cwq *get_work_gcwq(struct work_struct *work) } /* - * Policy functions. These define the policies on how the global - * worker pool is managed. Unless noted otherwise, these functions - * assume that they're being called with gcwq->lock held. + * Policy functions. These define the policies on how the global worker + * pools are managed. Unless noted otherwise, these functions assume that + * they're being called with gcwq->lock held. */ static bool __need_more_worker(struct worker_pool *pool) { - return !atomic_read(get_pool_nr_running(pool)) || - (pool->flags & POOL_HIGHPRI_PENDING); + return !atomic_read(get_pool_nr_running(pool)); } /* @@ -621,9 +626,7 @@ static bool keep_working(struct worker_pool *pool) { atomic_t *nr_running = get_pool_nr_running(pool); - return !list_empty(&pool->worklist) && - (atomic_read(nr_running) <= 1 || - (pool->flags & POOL_HIGHPRI_PENDING)); + return !list_empty(&pool->worklist) && atomic_read(nr_running) <= 1; } /* Do we need a new worker? Called from manager. */ @@ -891,43 +894,6 @@ static struct worker *find_worker_executing_work(struct global_cwq *gcwq, work); } -/** - * pool_determine_ins_pos - find insertion position - * @pool: pool of interest - * @cwq: cwq a work is being queued for - * - * A work for @cwq is about to be queued on @pool, determine insertion - * position for the work. If @cwq is for HIGHPRI wq, the work is - * queued at the head of the queue but in FIFO order with respect to - * other HIGHPRI works; otherwise, at the end of the queue. This - * function also sets POOL_HIGHPRI_PENDING flag to hint @pool that - * there are HIGHPRI works pending. - * - * CONTEXT: - * spin_lock_irq(gcwq->lock). - * - * RETURNS: - * Pointer to inserstion position. - */ -static inline struct list_head *pool_determine_ins_pos(struct worker_pool *pool, - struct cpu_workqueue_struct *cwq) -{ - struct work_struct *twork; - - if (likely(!(cwq->wq->flags & WQ_HIGHPRI))) - return &pool->worklist; - - list_for_each_entry(twork, &pool->worklist, entry) { - struct cpu_workqueue_struct *tcwq = get_work_cwq(twork); - - if (!(tcwq->wq->flags & WQ_HIGHPRI)) - break; - } - - pool->flags |= POOL_HIGHPRI_PENDING; - return &twork->entry; -} - /** * insert_work - insert a work into gcwq * @cwq: cwq @work belongs to @@ -1068,7 +1034,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, if (likely(cwq->nr_active < cwq->max_active)) { trace_workqueue_activate_work(work); cwq->nr_active++; - worklist = pool_determine_ins_pos(cwq->pool, cwq); + worklist = &cwq->pool->worklist; } else { work_flags |= WORK_STRUCT_DELAYED; worklist = &cwq->delayed_works; @@ -1385,6 +1351,7 @@ static struct worker *create_worker(struct worker_pool *pool, bool bind) { struct global_cwq *gcwq = pool->gcwq; bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; + const char *pri = worker_pool_pri(pool) ? "H" : ""; struct worker *worker = NULL; int id = -1; @@ -1406,15 +1373,17 @@ static struct worker *create_worker(struct worker_pool *pool, bool bind) if (!on_unbound_cpu) worker->task = kthread_create_on_node(worker_thread, - worker, - cpu_to_node(gcwq->cpu), - "kworker/%u:%d", gcwq->cpu, id); + worker, cpu_to_node(gcwq->cpu), + "kworker/%u:%d%s", gcwq->cpu, id, pri); else worker->task = kthread_create(worker_thread, worker, - "kworker/u:%d", id); + "kworker/u:%d%s", id, pri); if (IS_ERR(worker->task)) goto fail; + if (worker_pool_pri(pool)) + set_user_nice(worker->task, HIGHPRI_NICE_LEVEL); + /* * A rogue worker will become a regular one if CPU comes * online later on. Make sure every worker has @@ -1761,10 +1730,9 @@ static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) { struct work_struct *work = list_first_entry(&cwq->delayed_works, struct work_struct, entry); - struct list_head *pos = pool_determine_ins_pos(cwq->pool, cwq); trace_workqueue_activate_work(work); - move_linked_works(work, pos, NULL); + move_linked_works(work, &cwq->pool->worklist, NULL); __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work)); cwq->nr_active++; } @@ -1879,21 +1847,6 @@ __acquires(&gcwq->lock) set_work_cpu(work, gcwq->cpu); list_del_init(&work->entry); - /* - * If HIGHPRI_PENDING, check the next work, and, if HIGHPRI, - * wake up another worker; otherwise, clear HIGHPRI_PENDING. - */ - if (unlikely(pool->flags & POOL_HIGHPRI_PENDING)) { - struct work_struct *nwork = list_first_entry(&pool->worklist, - struct work_struct, entry); - - if (!list_empty(&pool->worklist) && - get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) - wake_up_worker(pool); - else - pool->flags &= ~POOL_HIGHPRI_PENDING; - } - /* * CPU intensive works don't participate in concurrency * management. They're the scheduler's responsibility. @@ -3047,9 +3000,10 @@ struct workqueue_struct *__alloc_workqueue_key(const char *fmt, for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); struct global_cwq *gcwq = get_gcwq(cpu); + int pool_idx = (bool)(flags & WQ_HIGHPRI); BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK); - cwq->pool = &gcwq->pool; + cwq->pool = &gcwq->pools[pool_idx]; cwq->wq = wq; cwq->flush_color = -1; cwq->max_active = max_active; -- cgit v1.2.3-55-g7522 From 6575820221f7a4dd6eadecf7bf83cdd154335eda Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:26 -0700 Subject: workqueue: perform cpu down operations from low priority cpu_notifier() Currently, all workqueue cpu hotplug operations run off CPU_PRI_WORKQUEUE which is higher than normal notifiers. This is to ensure that workqueue is up and running while bringing up a CPU before other notifiers try to use workqueue on the CPU. Per-cpu workqueues are supposed to remain working and bound to the CPU for normal CPU_DOWN_PREPARE notifiers. This holds mostly true even with workqueue offlining running with higher priority because workqueue CPU_DOWN_PREPARE only creates a bound trustee thread which runs the per-cpu workqueue without concurrency management without explicitly detaching the existing workers. However, if the trustee needs to create new workers, it creates unbound workers which may wander off to other CPUs while CPU_DOWN_PREPARE notifiers are in progress. Furthermore, if the CPU down is cancelled, the per-CPU workqueue may end up with workers which aren't bound to the CPU. While reliably reproducible with a convoluted artificial test-case involving scheduling and flushing CPU burning work items from CPU down notifiers, this isn't very likely to happen in the wild, and, even when it happens, the effects are likely to be hidden by the following successful CPU down. Fix it by using different priorities for up and down notifiers - high priority for up operations and low priority for down operations. Workqueue cpu hotplug operations will soon go through further cleanup. Signed-off-by: Tejun Heo Cc: stable@vger.kernel.org Acked-by: "Rafael J. Wysocki" --- include/linux/cpu.h | 5 +++-- kernel/workqueue.c | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 2e9b9ebbeb78..ce7a074f2519 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -73,8 +73,9 @@ enum { /* migration should happen before other stuff but after perf */ CPU_PRI_PERF = 20, CPU_PRI_MIGRATION = 10, - /* prepare workqueues for other notifiers */ - CPU_PRI_WORKQUEUE = 5, + /* bring up workqueues before normal notifiers and down after */ + CPU_PRI_WORKQUEUE_UP = 5, + CPU_PRI_WORKQUEUE_DOWN = -5, }; #define CPU_ONLINE 0x0002 /* CPU (unsigned)v is up */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 4fa9e3552f1e..f59b7fd26e26 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3644,6 +3644,41 @@ err_destroy: return NOTIFY_BAD; } +/* + * Workqueues should be brought up before normal priority CPU notifiers. + * This will be registered high priority CPU notifier. + */ +static int __devinit workqueue_cpu_up_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + switch (action & ~CPU_TASKS_FROZEN) { + case CPU_UP_PREPARE: + case CPU_UP_CANCELED: + case CPU_DOWN_FAILED: + case CPU_ONLINE: + return workqueue_cpu_callback(nfb, action, hcpu); + } + return NOTIFY_OK; +} + +/* + * Workqueues should be brought down after normal priority CPU notifiers. + * This will be registered as low priority CPU notifier. + */ +static int __devinit workqueue_cpu_down_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + switch (action & ~CPU_TASKS_FROZEN) { + case CPU_DOWN_PREPARE: + case CPU_DYING: + case CPU_POST_DEAD: + return workqueue_cpu_callback(nfb, action, hcpu); + } + return NOTIFY_OK; +} + #ifdef CONFIG_SMP struct work_for_cpu { @@ -3839,7 +3874,8 @@ static int __init init_workqueues(void) unsigned int cpu; int i; - cpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); + cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP); + cpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN); /* initialize gcwqs */ for_each_gcwq_cpu(cpu) { -- cgit v1.2.3-55-g7522 From f2d5a0ee06c1813f985bb9386f3ccc0d0315720f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:26 -0700 Subject: workqueue: drop CPU_DYING notifier operation Workqueue used CPU_DYING notification to mark GCWQ_DISASSOCIATED. This was necessary because workqueue's CPU_DOWN_PREPARE happened before other DOWN_PREPARE notifiers and workqueue needed to stay associated across the rest of DOWN_PREPARE. After the previous patch, workqueue's DOWN_PREPARE happens after others and can set GCWQ_DISASSOCIATED directly. Drop CPU_DYING and let the trustee set GCWQ_DISASSOCIATED after disabling concurrency management. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f59b7fd26e26..1405fb98c0b1 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1250,11 +1250,11 @@ static void worker_leave_idle(struct worker *worker) * verbatim as it's best effort and blocking and gcwq may be * [dis]associated in the meantime. * - * This function tries set_cpus_allowed() and locks gcwq and verifies - * the binding against GCWQ_DISASSOCIATED which is set during - * CPU_DYING and cleared during CPU_ONLINE, so if the worker enters - * idle state or fetches works without dropping lock, it can guarantee - * the scheduling requirement described in the first paragraph. + * This function tries set_cpus_allowed() and locks gcwq and verifies the + * binding against %GCWQ_DISASSOCIATED which is set during + * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker + * enters idle state or fetches works without dropping lock, it can + * guarantee the scheduling requirement described in the first paragraph. * * CONTEXT: * Might sleep. Called without any lock but returns with gcwq->lock @@ -3349,6 +3349,12 @@ static int __cpuinit trustee_thread(void *__gcwq) rc = trustee_wait_event(!gcwq_is_managing_workers(gcwq)); BUG_ON(rc < 0); + /* + * We've claimed all manager positions. Make all workers unbound + * and set DISASSOCIATED. Before this, all workers except for the + * ones which are still executing works from before the last CPU + * down must be on the cpu. After this, they may become diasporas. + */ for_each_worker_pool(pool, gcwq) { pool->flags |= POOL_MANAGING_WORKERS; @@ -3359,6 +3365,8 @@ static int __cpuinit trustee_thread(void *__gcwq) for_each_busy_worker(worker, i, pos, gcwq) worker->flags |= WORKER_ROGUE; + gcwq->flags |= GCWQ_DISASSOCIATED; + /* * Call schedule() so that we cross rq->lock and thus can * guarantee sched callbacks see the rogue flag. This is @@ -3582,16 +3590,6 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, } break; - case CPU_DYING: - /* - * Before this, the trustee and all workers except for - * the ones which are still executing works from - * before the last CPU down must be on the cpu. After - * this, they'll all be diasporas. - */ - gcwq->flags |= GCWQ_DISASSOCIATED; - break; - case CPU_POST_DEAD: gcwq->trustee_state = TRUSTEE_BUTCHER; /* fall through */ @@ -3672,7 +3670,6 @@ static int __devinit workqueue_cpu_down_callback(struct notifier_block *nfb, { switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_PREPARE: - case CPU_DYING: case CPU_POST_DEAD: return workqueue_cpu_callback(nfb, action, hcpu); } -- cgit v1.2.3-55-g7522 From 403c821d452c03be4ced571ac91339a9d3631b17 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:27 -0700 Subject: workqueue: ROGUE workers are UNBOUND workers Currently, WORKER_UNBOUND is used to mark workers for the unbound global_cwq and WORKER_ROGUE is used to mark workers for disassociated per-cpu global_cwqs. Both are used to make the marked worker skip concurrency management and the only place they make any difference is in worker_enter_idle() where WORKER_ROGUE is used to skip scheduling idle timer, which can easily be replaced with trustee state testing. This patch replaces WORKER_ROGUE with WORKER_UNBOUND and drops WORKER_ROGUE. This is to prepare for removing trustee and handling disassociated global_cwqs as unbound. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 1405fb98c0b1..af512927c607 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -58,13 +58,12 @@ enum { WORKER_DIE = 1 << 1, /* die die die */ WORKER_IDLE = 1 << 2, /* is idle */ WORKER_PREP = 1 << 3, /* preparing to run works */ - WORKER_ROGUE = 1 << 4, /* not bound to any cpu */ WORKER_REBIND = 1 << 5, /* mom is home, come back */ WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ WORKER_UNBOUND = 1 << 7, /* worker is unbound */ - WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND | - WORKER_CPU_INTENSIVE | WORKER_UNBOUND, + WORKER_NOT_RUNNING = WORKER_PREP | WORKER_REBIND | WORKER_UNBOUND | + WORKER_CPU_INTENSIVE, /* gcwq->trustee_state */ TRUSTEE_START = 0, /* start */ @@ -1198,7 +1197,7 @@ static void worker_enter_idle(struct worker *worker) /* idle_list is LIFO */ list_add(&worker->entry, &pool->idle_list); - if (likely(!(worker->flags & WORKER_ROGUE))) { + if (likely(gcwq->trustee_state != TRUSTEE_DONE)) { if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); @@ -1207,7 +1206,7 @@ static void worker_enter_idle(struct worker *worker) /* * Sanity check nr_running. Because trustee releases gcwq->lock - * between setting %WORKER_ROGUE and zapping nr_running, the + * between setting %WORKER_UNBOUND and zapping nr_running, the * warning may trigger spuriously. Check iff trustee is idle. */ WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE && @@ -1301,10 +1300,10 @@ __acquires(&gcwq->lock) } /* - * Function for worker->rebind_work used to rebind rogue busy workers - * to the associated cpu which is coming back online. This is - * scheduled by cpu up but can race with other cpu hotplug operations - * and may be executed twice without intervening cpu down. + * Function for worker->rebind_work used to rebind unbound busy workers to + * the associated cpu which is coming back online. This is scheduled by + * cpu up but can race with other cpu hotplug operations and may be + * executed twice without intervening cpu down. */ static void worker_rebind_fn(struct work_struct *work) { @@ -1385,9 +1384,8 @@ static struct worker *create_worker(struct worker_pool *pool, bool bind) set_user_nice(worker->task, HIGHPRI_NICE_LEVEL); /* - * A rogue worker will become a regular one if CPU comes - * online later on. Make sure every worker has - * PF_THREAD_BOUND set. + * An unbound worker will become a regular one if CPU comes online + * later on. Make sure every worker has PF_THREAD_BOUND set. */ if (bind && !on_unbound_cpu) kthread_bind(worker->task, gcwq->cpu); @@ -3215,11 +3213,10 @@ EXPORT_SYMBOL_GPL(work_busy); * gcwqs serve mix of short, long and very long running works making * blocked draining impractical. * - * This is solved by allowing a gcwq to be detached from CPU, running - * it with unbound (rogue) workers and allowing it to be reattached - * later if the cpu comes back online. A separate thread is created - * to govern a gcwq in such state and is called the trustee of the - * gcwq. + * This is solved by allowing a gcwq to be detached from CPU, running it + * with unbound workers and allowing it to be reattached later if the cpu + * comes back online. A separate thread is created to govern a gcwq in + * such state and is called the trustee of the gcwq. * * Trustee states and their descriptions. * @@ -3359,19 +3356,18 @@ static int __cpuinit trustee_thread(void *__gcwq) pool->flags |= POOL_MANAGING_WORKERS; list_for_each_entry(worker, &pool->idle_list, entry) - worker->flags |= WORKER_ROGUE; + worker->flags |= WORKER_UNBOUND; } for_each_busy_worker(worker, i, pos, gcwq) - worker->flags |= WORKER_ROGUE; + worker->flags |= WORKER_UNBOUND; gcwq->flags |= GCWQ_DISASSOCIATED; /* - * Call schedule() so that we cross rq->lock and thus can - * guarantee sched callbacks see the rogue flag. This is - * necessary as scheduler callbacks may be invoked from other - * cpus. + * Call schedule() so that we cross rq->lock and thus can guarantee + * sched callbacks see the unbound flag. This is necessary as + * scheduler callbacks may be invoked from other cpus. */ spin_unlock_irq(&gcwq->lock); schedule(); @@ -3439,7 +3435,7 @@ static int __cpuinit trustee_thread(void *__gcwq) worker = create_worker(pool, false); spin_lock_irq(&gcwq->lock); if (worker) { - worker->flags |= WORKER_ROGUE; + worker->flags |= WORKER_UNBOUND; start_worker(worker); } } @@ -3488,7 +3484,7 @@ static int __cpuinit trustee_thread(void *__gcwq) * rebinding is scheduled. */ worker->flags |= WORKER_REBIND; - worker->flags &= ~WORKER_ROGUE; + worker->flags &= ~WORKER_UNBOUND; /* queue rebind_work, wq doesn't matter, use the default one */ if (test_and_set_bit(WORK_STRUCT_PENDING_BIT, -- cgit v1.2.3-55-g7522 From 6037315269d62bf967286ae2670fdd6b6acedab9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:27 -0700 Subject: workqueue: use mutex for global_cwq manager exclusion POOL_MANAGING_WORKERS is used to ensure that at most one worker takes the manager role at any given time on a given global_cwq. Trustee later hitched on it to assume manager adding blocking wait for the bit. As trustee already needed a custom wait mechanism, waiting for MANAGING_WORKERS was rolled into the same mechanism. Trustee is scheduled to be removed. This patch separates out MANAGING_WORKERS wait into per-pool mutex. Workers use mutex_trylock() to test for manager role and trustee uses mutex_lock() to claim manager roles. gcwq_claim/release_management() helpers are added to grab and release manager roles of all pools on a global_cwq. gcwq_claim_management() always grabs pool manager mutexes in ascending pool index order and uses pool index as lockdep subclass. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 65 ++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 39 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index af512927c607..f7a00697d150 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -51,7 +51,6 @@ enum { /* pool flags */ POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ - POOL_MANAGING_WORKERS = 1 << 1, /* managing workers */ /* worker flags */ WORKER_STARTED = 1 << 0, /* started */ @@ -155,6 +154,7 @@ struct worker_pool { struct timer_list idle_timer; /* L: worker idle timeout */ struct timer_list mayday_timer; /* L: SOS timer for workers */ + struct mutex manager_mutex; /* mutex manager should hold */ struct ida worker_ida; /* L: for worker IDs */ struct worker *first_idle; /* L: first idle worker */ }; @@ -644,7 +644,7 @@ static bool need_to_manage_workers(struct worker_pool *pool) /* Do we have too many workers and should some go away? */ static bool too_many_workers(struct worker_pool *pool) { - bool managing = pool->flags & POOL_MANAGING_WORKERS; + bool managing = mutex_is_locked(&pool->manager_mutex); int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ int nr_busy = pool->nr_workers - nr_idle; @@ -1655,14 +1655,12 @@ static bool maybe_destroy_workers(struct worker_pool *pool) static bool manage_workers(struct worker *worker) { struct worker_pool *pool = worker->pool; - struct global_cwq *gcwq = pool->gcwq; bool ret = false; - if (pool->flags & POOL_MANAGING_WORKERS) + if (!mutex_trylock(&pool->manager_mutex)) return ret; pool->flags &= ~POOL_MANAGE_WORKERS; - pool->flags |= POOL_MANAGING_WORKERS; /* * Destroy and then create so that may_start_working() is true @@ -1671,15 +1669,7 @@ static bool manage_workers(struct worker *worker) ret |= maybe_destroy_workers(pool); ret |= maybe_create_worker(pool); - pool->flags &= ~POOL_MANAGING_WORKERS; - - /* - * The trustee might be waiting to take over the manager - * position, tell it we're done. - */ - if (unlikely(gcwq->trustee)) - wake_up_all(&gcwq->trustee_wait); - + mutex_unlock(&pool->manager_mutex); return ret; } @@ -3255,6 +3245,24 @@ EXPORT_SYMBOL_GPL(work_busy); * ----------------> RELEASE -------------- */ +/* claim manager positions of all pools */ +static void gcwq_claim_management(struct global_cwq *gcwq) +{ + struct worker_pool *pool; + + for_each_worker_pool(pool, gcwq) + mutex_lock_nested(&pool->manager_mutex, pool - gcwq->pools); +} + +/* release manager positions */ +static void gcwq_release_management(struct global_cwq *gcwq) +{ + struct worker_pool *pool; + + for_each_worker_pool(pool, gcwq) + mutex_unlock(&pool->manager_mutex); +} + /** * trustee_wait_event_timeout - timed event wait for trustee * @cond: condition to wait for @@ -3304,16 +3312,6 @@ EXPORT_SYMBOL_GPL(work_busy); __ret1 < 0 ? -1 : 0; \ }) -static bool gcwq_is_managing_workers(struct global_cwq *gcwq) -{ - struct worker_pool *pool; - - for_each_worker_pool(pool, gcwq) - if (pool->flags & POOL_MANAGING_WORKERS) - return true; - return false; -} - static bool gcwq_has_idle_workers(struct global_cwq *gcwq) { struct worker_pool *pool; @@ -3336,15 +3334,8 @@ static int __cpuinit trustee_thread(void *__gcwq) BUG_ON(gcwq->cpu != smp_processor_id()); + gcwq_claim_management(gcwq); spin_lock_irq(&gcwq->lock); - /* - * Claim the manager position and make all workers rogue. - * Trustee must be bound to the target cpu and can't be - * cancelled. - */ - BUG_ON(gcwq->cpu != smp_processor_id()); - rc = trustee_wait_event(!gcwq_is_managing_workers(gcwq)); - BUG_ON(rc < 0); /* * We've claimed all manager positions. Make all workers unbound @@ -3352,12 +3343,9 @@ static int __cpuinit trustee_thread(void *__gcwq) * ones which are still executing works from before the last CPU * down must be on the cpu. After this, they may become diasporas. */ - for_each_worker_pool(pool, gcwq) { - pool->flags |= POOL_MANAGING_WORKERS; - + for_each_worker_pool(pool, gcwq) list_for_each_entry(worker, &pool->idle_list, entry) worker->flags |= WORKER_UNBOUND; - } for_each_busy_worker(worker, i, pos, gcwq) worker->flags |= WORKER_UNBOUND; @@ -3497,9 +3485,7 @@ static int __cpuinit trustee_thread(void *__gcwq) work_color_to_flags(WORK_NO_COLOR)); } - /* relinquish manager role */ - for_each_worker_pool(pool, gcwq) - pool->flags &= ~POOL_MANAGING_WORKERS; + gcwq_release_management(gcwq); /* notify completion */ gcwq->trustee = NULL; @@ -3894,6 +3880,7 @@ static int __init init_workqueues(void) setup_timer(&pool->mayday_timer, gcwq_mayday_timeout, (unsigned long)pool); + mutex_init(&pool->manager_mutex); ida_init(&pool->worker_ida); } -- cgit v1.2.3-55-g7522 From bc2ae0f5bb2f39e6db06a62f9d353e4601a332a1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:27 -0700 Subject: workqueue: drop @bind from create_worker() Currently, create_worker()'s callers are responsible for deciding whether the newly created worker should be bound to the associated CPU and create_worker() sets WORKER_UNBOUND only for the workers for the unbound global_cwq. Creation during normal operation is always via maybe_create_worker() and @bind is true. For workers created during hotplug, @bind is false. Normal operation path is planned to be used even while the CPU is going through hotplug operations or offline and this static decision won't work. Drop @bind from create_worker() and decide whether to bind by looking at GCWQ_DISASSOCIATED. create_worker() will also set WORKER_UNBOUND autmatically if disassociated. To avoid flipping GCWQ_DISASSOCIATED while create_worker() is in progress, the flag is now allowed to be changed only while holding all manager_mutexes on the global_cwq. This requires that GCWQ_DISASSOCIATED is not cleared behind trustee's back. CPU_ONLINE no longer clears DISASSOCIATED before flushing trustee, which clears DISASSOCIATED before rebinding remaining workers if asked to release. For cases where trustee isn't around, CPU_ONLINE clears DISASSOCIATED after flushing trustee. Also, now, first_idle has UNBOUND set on creation which is explicitly cleared by CPU_ONLINE while binding it. These convolutions will soon be removed by further simplification of CPU hotplug path. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 64 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index f7a00697d150..e1d05e51a80a 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -45,7 +45,22 @@ #include "workqueue_sched.h" enum { - /* global_cwq flags */ + /* + * global_cwq flags + * + * A bound gcwq is either associated or disassociated with its CPU. + * While associated (!DISASSOCIATED), all workers are bound to the + * CPU and none has %WORKER_UNBOUND set and concurrency management + * is in effect. + * + * While DISASSOCIATED, the cpu may be offline and all workers have + * %WORKER_UNBOUND set and concurrency management disabled, and may + * be executing on any CPU. The gcwq behaves as an unbound one. + * + * Note that DISASSOCIATED can be flipped only while holding + * managership of all pools on the gcwq to avoid changing binding + * state while create_worker() is in progress. + */ GCWQ_DISASSOCIATED = 1 << 0, /* cpu can't serve workers */ GCWQ_FREEZING = 1 << 1, /* freeze in progress */ @@ -1334,7 +1349,6 @@ static struct worker *alloc_worker(void) /** * create_worker - create a new workqueue worker * @pool: pool the new worker will belong to - * @bind: whether to set affinity to @cpu or not * * Create a new worker which is bound to @pool. The returned worker * can be started by calling start_worker() or destroyed using @@ -1346,10 +1360,9 @@ static struct worker *alloc_worker(void) * RETURNS: * Pointer to the newly created worker. */ -static struct worker *create_worker(struct worker_pool *pool, bool bind) +static struct worker *create_worker(struct worker_pool *pool) { struct global_cwq *gcwq = pool->gcwq; - bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; const char *pri = worker_pool_pri(pool) ? "H" : ""; struct worker *worker = NULL; int id = -1; @@ -1370,7 +1383,7 @@ static struct worker *create_worker(struct worker_pool *pool, bool bind) worker->pool = pool; worker->id = id; - if (!on_unbound_cpu) + if (gcwq->cpu != WORK_CPU_UNBOUND) worker->task = kthread_create_on_node(worker_thread, worker, cpu_to_node(gcwq->cpu), "kworker/%u:%d%s", gcwq->cpu, id, pri); @@ -1384,15 +1397,19 @@ static struct worker *create_worker(struct worker_pool *pool, bool bind) set_user_nice(worker->task, HIGHPRI_NICE_LEVEL); /* - * An unbound worker will become a regular one if CPU comes online - * later on. Make sure every worker has PF_THREAD_BOUND set. + * Determine CPU binding of the new worker depending on + * %GCWQ_DISASSOCIATED. The caller is responsible for ensuring the + * flag remains stable across this function. See the comments + * above the flag definition for details. + * + * As an unbound worker may later become a regular one if CPU comes + * online, make sure every worker has %PF_THREAD_BOUND set. */ - if (bind && !on_unbound_cpu) + if (!(gcwq->flags & GCWQ_DISASSOCIATED)) { kthread_bind(worker->task, gcwq->cpu); - else { + } else { worker->task->flags |= PF_THREAD_BOUND; - if (on_unbound_cpu) - worker->flags |= WORKER_UNBOUND; + worker->flags |= WORKER_UNBOUND; } return worker; @@ -1568,7 +1585,7 @@ restart: while (true) { struct worker *worker; - worker = create_worker(pool, true); + worker = create_worker(pool); if (worker) { del_timer_sync(&pool->mayday_timer); spin_lock_irq(&gcwq->lock); @@ -3420,12 +3437,10 @@ static int __cpuinit trustee_thread(void *__gcwq) if (need_to_create_worker(pool)) { spin_unlock_irq(&gcwq->lock); - worker = create_worker(pool, false); + worker = create_worker(pool); spin_lock_irq(&gcwq->lock); - if (worker) { - worker->flags |= WORKER_UNBOUND; + if (worker) start_worker(worker); - } } } @@ -3463,6 +3478,10 @@ static int __cpuinit trustee_thread(void *__gcwq) for_each_worker_pool(pool, gcwq) WARN_ON(!list_empty(&pool->idle_list)); + /* if we're reassociating, clear DISASSOCIATED */ + if (gcwq->trustee_state == TRUSTEE_RELEASE) + gcwq->flags &= ~GCWQ_DISASSOCIATED; + for_each_busy_worker(worker, i, pos, gcwq) { struct work_struct *rebind_work = &worker->rebind_work; @@ -3546,7 +3565,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, i = 0; for_each_worker_pool(pool, gcwq) { BUG_ON(pool->first_idle); - new_workers[i] = create_worker(pool, false); + new_workers[i] = create_worker(pool); if (!new_workers[i++]) goto err_destroy; } @@ -3584,13 +3603,19 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, case CPU_DOWN_FAILED: case CPU_ONLINE: - gcwq->flags &= ~GCWQ_DISASSOCIATED; if (gcwq->trustee_state != TRUSTEE_DONE) { gcwq->trustee_state = TRUSTEE_RELEASE; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_DONE); } + /* + * Either DISASSOCIATED is already cleared or no worker is + * left on the gcwq. Safe to clear DISASSOCIATED without + * claiming managers. + */ + gcwq->flags &= ~GCWQ_DISASSOCIATED; + /* * Trustee is done and there might be no worker left. * Put the first_idle in and request a real manager to @@ -3601,6 +3626,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, kthread_bind(pool->first_idle->task, cpu); spin_lock_irq(&gcwq->lock); pool->flags |= POOL_MANAGE_WORKERS; + pool->first_idle->flags &= ~WORKER_UNBOUND; start_worker(pool->first_idle); pool->first_idle = NULL; } @@ -3899,7 +3925,7 @@ static int __init init_workqueues(void) for_each_worker_pool(pool, gcwq) { struct worker *worker; - worker = create_worker(pool, true); + worker = create_worker(pool); BUG_ON(!worker); spin_lock_irq(&gcwq->lock); start_worker(worker); -- cgit v1.2.3-55-g7522 From 25511a477657884d2164f338341fa89652610507 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:27 -0700 Subject: workqueue: reimplement CPU online rebinding to handle idle workers Currently, if there are left workers when a CPU is being brough back online, the trustee kills all idle workers and scheduled rebind_work so that they re-bind to the CPU after the currently executing work is finished. This works for busy workers because concurrency management doesn't try to wake up them from scheduler callbacks, which require the target task to be on the local run queue. The busy worker bumps concurrency counter appropriately as it clears WORKER_UNBOUND from the rebind work item and it's bound to the CPU before returning to the idle state. To reduce CPU on/offlining overhead (as many embedded systems use it for powersaving) and simplify the code path, workqueue is planned to be modified to retain idle workers across CPU on/offlining. This patch reimplements CPU online rebinding such that it can also handle idle workers. As noted earlier, due to the local wakeup requirement, rebinding idle workers is tricky. All idle workers must be re-bound before scheduler callbacks are enabled. This is achieved by interlocking idle re-binding. Idle workers are requested to re-bind and then hold until all idle re-binding is complete so that no bound worker starts executing work item. Only after all idle workers are re-bound and parked, CPU_ONLINE proceeds to release them and queue rebind work item to busy workers thus guaranteeing scheduler callbacks aren't invoked until all idle workers are ready. worker_rebind_fn() is renamed to busy_worker_rebind_fn() and idle_worker_rebind() for idle workers is added. Rebinding logic is moved to rebind_workers() and now called from CPU_ONLINE after flushing trustee. While at it, add CPU sanity check in worker_thread(). Note that now a worker may become idle or the manager between trustee release and rebinding during CPU_ONLINE. As the previous patch updated create_worker() so that it can be used by regular manager while unbound and this patch implements idle re-binding, this is safe. This prepares for removal of trustee and keeping idle workers across CPU hotplugs. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 215 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 166 insertions(+), 49 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e1d05e51a80a..6927fecae412 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -133,6 +133,7 @@ enum { struct global_cwq; struct worker_pool; +struct idle_rebind; /* * The poor guys doing the actual heavy lifting. All on-duty workers @@ -154,7 +155,10 @@ struct worker { unsigned long last_active; /* L: last active timestamp */ unsigned int flags; /* X: flags */ int id; /* I: worker id */ - struct work_struct rebind_work; /* L: rebind worker to cpu */ + + /* for rebinding worker to CPU */ + struct idle_rebind *idle_rebind; /* L: for idle worker */ + struct work_struct rebind_work; /* L: for busy worker */ }; struct worker_pool { @@ -190,6 +194,8 @@ struct global_cwq { struct worker_pool pools[2]; /* normal and highpri pools */ + wait_queue_head_t rebind_hold; /* rebind hold wait */ + struct task_struct *trustee; /* L: for gcwq shutdown */ unsigned int trustee_state; /* L: trustee state */ wait_queue_head_t trustee_wait; /* trustee wait */ @@ -1314,13 +1320,37 @@ __acquires(&gcwq->lock) } } +struct idle_rebind { + int cnt; /* # workers to be rebound */ + struct completion done; /* all workers rebound */ +}; + +/* + * Rebind an idle @worker to its CPU. During CPU onlining, this has to + * happen synchronously for idle workers. worker_thread() will test + * %WORKER_REBIND before leaving idle and call this function. + */ +static void idle_worker_rebind(struct worker *worker) +{ + struct global_cwq *gcwq = worker->pool->gcwq; + + /* CPU must be online at this point */ + WARN_ON(!worker_maybe_bind_and_lock(worker)); + if (!--worker->idle_rebind->cnt) + complete(&worker->idle_rebind->done); + spin_unlock_irq(&worker->pool->gcwq->lock); + + /* we did our part, wait for rebind_workers() to finish up */ + wait_event(gcwq->rebind_hold, !(worker->flags & WORKER_REBIND)); +} + /* - * Function for worker->rebind_work used to rebind unbound busy workers to + * Function for @worker->rebind.work used to rebind unbound busy workers to * the associated cpu which is coming back online. This is scheduled by * cpu up but can race with other cpu hotplug operations and may be * executed twice without intervening cpu down. */ -static void worker_rebind_fn(struct work_struct *work) +static void busy_worker_rebind_fn(struct work_struct *work) { struct worker *worker = container_of(work, struct worker, rebind_work); struct global_cwq *gcwq = worker->pool->gcwq; @@ -1331,6 +1361,112 @@ static void worker_rebind_fn(struct work_struct *work) spin_unlock_irq(&gcwq->lock); } +/** + * rebind_workers - rebind all workers of a gcwq to the associated CPU + * @gcwq: gcwq of interest + * + * @gcwq->cpu is coming online. Rebind all workers to the CPU. Rebinding + * is different for idle and busy ones. + * + * The idle ones should be rebound synchronously and idle rebinding should + * be complete before any worker starts executing work items with + * concurrency management enabled; otherwise, scheduler may oops trying to + * wake up non-local idle worker from wq_worker_sleeping(). + * + * This is achieved by repeatedly requesting rebinding until all idle + * workers are known to have been rebound under @gcwq->lock and holding all + * idle workers from becoming busy until idle rebinding is complete. + * + * Once idle workers are rebound, busy workers can be rebound as they + * finish executing their current work items. Queueing the rebind work at + * the head of their scheduled lists is enough. Note that nr_running will + * be properbly bumped as busy workers rebind. + * + * On return, all workers are guaranteed to either be bound or have rebind + * work item scheduled. + */ +static void rebind_workers(struct global_cwq *gcwq) + __releases(&gcwq->lock) __acquires(&gcwq->lock) +{ + struct idle_rebind idle_rebind; + struct worker_pool *pool; + struct worker *worker; + struct hlist_node *pos; + int i; + + lockdep_assert_held(&gcwq->lock); + + for_each_worker_pool(pool, gcwq) + lockdep_assert_held(&pool->manager_mutex); + + /* + * Rebind idle workers. Interlocked both ways. We wait for + * workers to rebind via @idle_rebind.done. Workers will wait for + * us to finish up by watching %WORKER_REBIND. + */ + init_completion(&idle_rebind.done); +retry: + idle_rebind.cnt = 1; + INIT_COMPLETION(idle_rebind.done); + + /* set REBIND and kick idle ones, we'll wait for these later */ + for_each_worker_pool(pool, gcwq) { + list_for_each_entry(worker, &pool->idle_list, entry) { + if (worker->flags & WORKER_REBIND) + continue; + + /* morph UNBOUND to REBIND */ + worker->flags &= ~WORKER_UNBOUND; + worker->flags |= WORKER_REBIND; + + idle_rebind.cnt++; + worker->idle_rebind = &idle_rebind; + + /* worker_thread() will call idle_worker_rebind() */ + wake_up_process(worker->task); + } + } + + if (--idle_rebind.cnt) { + spin_unlock_irq(&gcwq->lock); + wait_for_completion(&idle_rebind.done); + spin_lock_irq(&gcwq->lock); + /* busy ones might have become idle while waiting, retry */ + goto retry; + } + + /* + * All idle workers are rebound and waiting for %WORKER_REBIND to + * be cleared inside idle_worker_rebind(). Clear and release. + * Clearing %WORKER_REBIND from this foreign context is safe + * because these workers are still guaranteed to be idle. + */ + for_each_worker_pool(pool, gcwq) + list_for_each_entry(worker, &pool->idle_list, entry) + worker->flags &= ~WORKER_REBIND; + + wake_up_all(&gcwq->rebind_hold); + + /* rebind busy workers */ + for_each_busy_worker(worker, i, pos, gcwq) { + struct work_struct *rebind_work = &worker->rebind_work; + + /* morph UNBOUND to REBIND */ + worker->flags &= ~WORKER_UNBOUND; + worker->flags |= WORKER_REBIND; + + if (test_and_set_bit(WORK_STRUCT_PENDING_BIT, + work_data_bits(rebind_work))) + continue; + + /* wq doesn't matter, use the default one */ + debug_work_activate(rebind_work); + insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work, + worker->scheduled.next, + work_color_to_flags(WORK_NO_COLOR)); + } +} + static struct worker *alloc_worker(void) { struct worker *worker; @@ -1339,7 +1475,7 @@ static struct worker *alloc_worker(void) if (worker) { INIT_LIST_HEAD(&worker->entry); INIT_LIST_HEAD(&worker->scheduled); - INIT_WORK(&worker->rebind_work, worker_rebind_fn); + INIT_WORK(&worker->rebind_work, busy_worker_rebind_fn); /* on creation a worker is in !idle && prep state */ worker->flags = WORKER_PREP; } @@ -1829,6 +1965,9 @@ __acquires(&gcwq->lock) lockdep_copy_map(&lockdep_map, &work->lockdep_map); #endif + WARN_ON_ONCE(!(worker->flags & (WORKER_UNBOUND | WORKER_REBIND)) && + raw_smp_processor_id() != gcwq->cpu); + /* * A single work shouldn't be executed concurrently by * multiple workers on a single cpu. Check whether anyone is @@ -1946,11 +2085,20 @@ static int worker_thread(void *__worker) woke_up: spin_lock_irq(&gcwq->lock); - /* DIE can be set only while we're idle, checking here is enough */ - if (worker->flags & WORKER_DIE) { + /* + * DIE can be set only while idle and REBIND set while busy has + * @worker->rebind_work scheduled. Checking here is enough. + */ + if (unlikely(worker->flags & (WORKER_REBIND | WORKER_DIE))) { spin_unlock_irq(&gcwq->lock); - worker->task->flags &= ~PF_WQ_WORKER; - return 0; + + if (worker->flags & WORKER_DIE) { + worker->task->flags &= ~PF_WQ_WORKER; + return 0; + } + + idle_worker_rebind(worker); + goto woke_up; } worker_leave_idle(worker); @@ -3468,42 +3616,6 @@ static int __cpuinit trustee_thread(void *__gcwq) } } while (i && rc >= 0); - /* - * At this point, either draining has completed and no worker - * is left, or cpu down has been canceled or the cpu is being - * brought back up. There shouldn't be any idle one left. - * Tell the remaining busy ones to rebind once it finishes the - * currently scheduled works by scheduling the rebind_work. - */ - for_each_worker_pool(pool, gcwq) - WARN_ON(!list_empty(&pool->idle_list)); - - /* if we're reassociating, clear DISASSOCIATED */ - if (gcwq->trustee_state == TRUSTEE_RELEASE) - gcwq->flags &= ~GCWQ_DISASSOCIATED; - - for_each_busy_worker(worker, i, pos, gcwq) { - struct work_struct *rebind_work = &worker->rebind_work; - - /* - * Rebind_work may race with future cpu hotplug - * operations. Use a separate flag to mark that - * rebinding is scheduled. - */ - worker->flags |= WORKER_REBIND; - worker->flags &= ~WORKER_UNBOUND; - - /* queue rebind_work, wq doesn't matter, use the default one */ - if (test_and_set_bit(WORK_STRUCT_PENDING_BIT, - work_data_bits(rebind_work))) - continue; - - debug_work_activate(rebind_work); - insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work, - worker->scheduled.next, - work_color_to_flags(WORK_NO_COLOR)); - } - gcwq_release_management(gcwq); /* notify completion */ @@ -3609,13 +3721,16 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, wait_trustee_state(gcwq, TRUSTEE_DONE); } - /* - * Either DISASSOCIATED is already cleared or no worker is - * left on the gcwq. Safe to clear DISASSOCIATED without - * claiming managers. - */ + spin_unlock_irq(&gcwq->lock); + gcwq_claim_management(gcwq); + spin_lock_irq(&gcwq->lock); + gcwq->flags &= ~GCWQ_DISASSOCIATED; + rebind_workers(gcwq); + + gcwq_release_management(gcwq); + /* * Trustee is done and there might be no worker left. * Put the first_idle in and request a real manager to @@ -3910,6 +4025,8 @@ static int __init init_workqueues(void) ida_init(&pool->worker_ida); } + init_waitqueue_head(&gcwq->rebind_hold); + gcwq->trustee_state = TRUSTEE_DONE; init_waitqueue_head(&gcwq->trustee_wait); } -- cgit v1.2.3-55-g7522 From 3ce63377305b694f53e7dd0c72907591c5344224 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:27 -0700 Subject: workqueue: don't butcher idle workers on an offline CPU Currently, during CPU offlining, after all pending work items are drained, the trustee butchers all workers. Also, on CPU onlining failure, workqueue_cpu_callback() ensures that the first idle worker is destroyed. Combined, these guarantee that an offline CPU doesn't have any worker for it once all the lingering work items are finished. This guarantee isn't really necessary and makes CPU on/offlining more expensive than needs to be, especially for platforms which use CPU hotplug for powersaving. This patch lets offline CPUs removes idle worker butchering from the trustee and let a CPU which failed onlining keep the created first worker. The first worker is created if the CPU doesn't have any during CPU_DOWN_PREPARE and started right away. If onlining succeeds, the rebind_workers() call in CPU_ONLINE will rebind it like any other workers. If onlining fails, the worker is left alone till the next try. This makes CPU hotplugs cheaper by allowing global_cwqs to keep workers across them and simplifies code. Note that trustee doesn't re-arm idle timer when it's done and thus the disassociated global_cwq will keep all workers until it comes back online. This will be improved by further patches. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 94 ++++++++---------------------------------------------- 1 file changed, 14 insertions(+), 80 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 6927fecae412..acfabb22e2c4 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -175,7 +175,6 @@ struct worker_pool { struct mutex manager_mutex; /* mutex manager should hold */ struct ida worker_ida; /* L: for worker IDs */ - struct worker *first_idle; /* L: first idle worker */ }; /* @@ -3477,16 +3476,6 @@ static void gcwq_release_management(struct global_cwq *gcwq) __ret1 < 0 ? -1 : 0; \ }) -static bool gcwq_has_idle_workers(struct global_cwq *gcwq) -{ - struct worker_pool *pool; - - for_each_worker_pool(pool, gcwq) - if (!list_empty(&pool->idle_list)) - return true; - return false; -} - static int __cpuinit trustee_thread(void *__gcwq) { struct global_cwq *gcwq = __gcwq; @@ -3494,7 +3483,6 @@ static int __cpuinit trustee_thread(void *__gcwq) struct worker *worker; struct work_struct *work; struct hlist_node *pos; - long rc; int i; BUG_ON(gcwq->cpu != smp_processor_id()); @@ -3597,25 +3585,6 @@ static int __cpuinit trustee_thread(void *__gcwq) break; } - /* - * Either all works have been scheduled and cpu is down, or - * cpu down has already been canceled. Wait for and butcher - * all workers till we're canceled. - */ - do { - rc = trustee_wait_event(gcwq_has_idle_workers(gcwq)); - - i = 0; - for_each_worker_pool(pool, gcwq) { - while (!list_empty(&pool->idle_list)) { - worker = list_first_entry(&pool->idle_list, - struct worker, entry); - destroy_worker(worker); - } - i |= pool->nr_workers; - } - } while (i && rc >= 0); - gcwq_release_management(gcwq); /* notify completion */ @@ -3658,10 +3627,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, unsigned int cpu = (unsigned long)hcpu; struct global_cwq *gcwq = get_gcwq(cpu); struct task_struct *new_trustee = NULL; - struct worker *new_workers[NR_WORKER_POOLS] = { }; struct worker_pool *pool; unsigned long flags; - int i; action &= ~CPU_TASKS_FROZEN; @@ -3672,14 +3639,22 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, if (IS_ERR(new_trustee)) return notifier_from_errno(PTR_ERR(new_trustee)); kthread_bind(new_trustee, cpu); - /* fall through */ + break; + case CPU_UP_PREPARE: - i = 0; for_each_worker_pool(pool, gcwq) { - BUG_ON(pool->first_idle); - new_workers[i] = create_worker(pool); - if (!new_workers[i++]) - goto err_destroy; + struct worker *worker; + + if (pool->nr_workers) + continue; + + worker = create_worker(pool); + if (!worker) + return NOTIFY_BAD; + + spin_lock_irq(&gcwq->lock); + start_worker(worker); + spin_unlock_irq(&gcwq->lock); } } @@ -3694,23 +3669,10 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, gcwq->trustee_state = TRUSTEE_START; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); - /* fall through */ - case CPU_UP_PREPARE: - i = 0; - for_each_worker_pool(pool, gcwq) { - BUG_ON(pool->first_idle); - pool->first_idle = new_workers[i++]; - } break; case CPU_POST_DEAD: gcwq->trustee_state = TRUSTEE_BUTCHER; - /* fall through */ - case CPU_UP_CANCELED: - for_each_worker_pool(pool, gcwq) { - destroy_worker(pool->first_idle); - pool->first_idle = NULL; - } break; case CPU_DOWN_FAILED: @@ -3730,39 +3692,12 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, rebind_workers(gcwq); gcwq_release_management(gcwq); - - /* - * Trustee is done and there might be no worker left. - * Put the first_idle in and request a real manager to - * take a look. - */ - for_each_worker_pool(pool, gcwq) { - spin_unlock_irq(&gcwq->lock); - kthread_bind(pool->first_idle->task, cpu); - spin_lock_irq(&gcwq->lock); - pool->flags |= POOL_MANAGE_WORKERS; - pool->first_idle->flags &= ~WORKER_UNBOUND; - start_worker(pool->first_idle); - pool->first_idle = NULL; - } break; } spin_unlock_irqrestore(&gcwq->lock, flags); return notifier_from_errno(0); - -err_destroy: - if (new_trustee) - kthread_stop(new_trustee); - - spin_lock_irqsave(&gcwq->lock, flags); - for (i = 0; i < NR_WORKER_POOLS; i++) - if (new_workers[i]) - destroy_worker(new_workers[i]); - spin_unlock_irqrestore(&gcwq->lock, flags); - - return NOTIFY_BAD; } /* @@ -3775,7 +3710,6 @@ static int __devinit workqueue_cpu_up_callback(struct notifier_block *nfb, { switch (action & ~CPU_TASKS_FROZEN) { case CPU_UP_PREPARE: - case CPU_UP_CANCELED: case CPU_DOWN_FAILED: case CPU_ONLINE: return workqueue_cpu_callback(nfb, action, hcpu); -- cgit v1.2.3-55-g7522 From 628c78e7ea19d5b70d2b6a59030362168cdbe1ad Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:27 -0700 Subject: workqueue: remove CPU offline trustee With the previous changes, a disassociated global_cwq now can run as an unbound one on its own - it can create workers as necessary to drain remaining works after the CPU has been brought down and manage the number of workers using the usual idle timer mechanism making trustee completely redundant except for the actual unbinding operation. This patch removes the trustee and let a disassociated global_cwq manage itself. Unbinding is moved to a work item (for CPU affinity) which is scheduled and flushed from CPU_DONW_PREPARE. This patch moves nr_running clearing outside gcwq and manager locks to simplify the code. As nr_running is unused at the point, this is safe. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 288 +++++++---------------------------------------------- 1 file changed, 36 insertions(+), 252 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index acfabb22e2c4..d1545daa74ad 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -79,13 +79,6 @@ enum { WORKER_NOT_RUNNING = WORKER_PREP | WORKER_REBIND | WORKER_UNBOUND | WORKER_CPU_INTENSIVE, - /* gcwq->trustee_state */ - TRUSTEE_START = 0, /* start */ - TRUSTEE_IN_CHARGE = 1, /* trustee in charge of gcwq */ - TRUSTEE_BUTCHER = 2, /* butcher workers */ - TRUSTEE_RELEASE = 3, /* release workers */ - TRUSTEE_DONE = 4, /* trustee is done */ - NR_WORKER_POOLS = 2, /* # worker pools per gcwq */ BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */ @@ -100,7 +93,6 @@ enum { (min two ticks) */ MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ CREATE_COOLDOWN = HZ, /* time to breath after fail */ - TRUSTEE_COOLDOWN = HZ / 10, /* for trustee draining */ /* * Rescue workers are used only on emergencies and shared by @@ -194,10 +186,6 @@ struct global_cwq { struct worker_pool pools[2]; /* normal and highpri pools */ wait_queue_head_t rebind_hold; /* rebind hold wait */ - - struct task_struct *trustee; /* L: for gcwq shutdown */ - unsigned int trustee_state; /* L: trustee state */ - wait_queue_head_t trustee_wait; /* trustee wait */ } ____cacheline_aligned_in_smp; /* @@ -753,11 +741,11 @@ struct task_struct *wq_worker_sleeping(struct task_struct *task, * worklist not empty test sequence is in insert_work(). * Please read comment there. * - * NOT_RUNNING is clear. This means that trustee is not in - * charge and we're running on the local cpu w/ rq lock held - * and preemption disabled, which in turn means that none else - * could be manipulating idle_list, so dereferencing idle_list - * without gcwq lock is safe. + * NOT_RUNNING is clear. This means that we're bound to and + * running on the local cpu w/ rq lock held and preemption + * disabled, which in turn means that none else could be + * manipulating idle_list, so dereferencing idle_list without gcwq + * lock is safe. */ if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist)) to_wakeup = first_worker(pool); @@ -1217,19 +1205,16 @@ static void worker_enter_idle(struct worker *worker) /* idle_list is LIFO */ list_add(&worker->entry, &pool->idle_list); - if (likely(gcwq->trustee_state != TRUSTEE_DONE)) { - if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) - mod_timer(&pool->idle_timer, - jiffies + IDLE_WORKER_TIMEOUT); - } else - wake_up_all(&gcwq->trustee_wait); + if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) + mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); /* - * Sanity check nr_running. Because trustee releases gcwq->lock - * between setting %WORKER_UNBOUND and zapping nr_running, the - * warning may trigger spuriously. Check iff trustee is idle. + * Sanity check nr_running. Because gcwq_unbind_fn() releases + * gcwq->lock between setting %WORKER_UNBOUND and zapping + * nr_running, the warning may trigger spuriously. Check iff + * unbind is not in progress. */ - WARN_ON_ONCE(gcwq->trustee_state == TRUSTEE_DONE && + WARN_ON_ONCE(!(gcwq->flags & GCWQ_DISASSOCIATED) && pool->nr_workers == pool->nr_idle && atomic_read(get_pool_nr_running(pool))); } @@ -3367,46 +3352,9 @@ EXPORT_SYMBOL_GPL(work_busy); * gcwqs serve mix of short, long and very long running works making * blocked draining impractical. * - * This is solved by allowing a gcwq to be detached from CPU, running it - * with unbound workers and allowing it to be reattached later if the cpu - * comes back online. A separate thread is created to govern a gcwq in - * such state and is called the trustee of the gcwq. - * - * Trustee states and their descriptions. - * - * START Command state used on startup. On CPU_DOWN_PREPARE, a - * new trustee is started with this state. - * - * IN_CHARGE Once started, trustee will enter this state after - * assuming the manager role and making all existing - * workers rogue. DOWN_PREPARE waits for trustee to - * enter this state. After reaching IN_CHARGE, trustee - * tries to execute the pending worklist until it's empty - * and the state is set to BUTCHER, or the state is set - * to RELEASE. - * - * BUTCHER Command state which is set by the cpu callback after - * the cpu has went down. Once this state is set trustee - * knows that there will be no new works on the worklist - * and once the worklist is empty it can proceed to - * killing idle workers. - * - * RELEASE Command state which is set by the cpu callback if the - * cpu down has been canceled or it has come online - * again. After recognizing this state, trustee stops - * trying to drain or butcher and clears ROGUE, rebinds - * all remaining workers back to the cpu and releases - * manager role. - * - * DONE Trustee will enter this state after BUTCHER or RELEASE - * is complete. - * - * trustee CPU draining - * took over down complete - * START -----------> IN_CHARGE -----------> BUTCHER -----------> DONE - * | | ^ - * | CPU is back online v return workers | - * ----------------> RELEASE -------------- + * This is solved by allowing a gcwq to be disassociated from the CPU + * running as an unbound one and allowing it to be reattached later if the + * cpu comes back online. */ /* claim manager positions of all pools */ @@ -3427,61 +3375,11 @@ static void gcwq_release_management(struct global_cwq *gcwq) mutex_unlock(&pool->manager_mutex); } -/** - * trustee_wait_event_timeout - timed event wait for trustee - * @cond: condition to wait for - * @timeout: timeout in jiffies - * - * wait_event_timeout() for trustee to use. Handles locking and - * checks for RELEASE request. - * - * CONTEXT: - * spin_lock_irq(gcwq->lock) which may be released and regrabbed - * multiple times. To be used by trustee. - * - * RETURNS: - * Positive indicating left time if @cond is satisfied, 0 if timed - * out, -1 if canceled. - */ -#define trustee_wait_event_timeout(cond, timeout) ({ \ - long __ret = (timeout); \ - while (!((cond) || (gcwq->trustee_state == TRUSTEE_RELEASE)) && \ - __ret) { \ - spin_unlock_irq(&gcwq->lock); \ - __wait_event_timeout(gcwq->trustee_wait, (cond) || \ - (gcwq->trustee_state == TRUSTEE_RELEASE), \ - __ret); \ - spin_lock_irq(&gcwq->lock); \ - } \ - gcwq->trustee_state == TRUSTEE_RELEASE ? -1 : (__ret); \ -}) - -/** - * trustee_wait_event - event wait for trustee - * @cond: condition to wait for - * - * wait_event() for trustee to use. Automatically handles locking and - * checks for CANCEL request. - * - * CONTEXT: - * spin_lock_irq(gcwq->lock) which may be released and regrabbed - * multiple times. To be used by trustee. - * - * RETURNS: - * 0 if @cond is satisfied, -1 if canceled. - */ -#define trustee_wait_event(cond) ({ \ - long __ret1; \ - __ret1 = trustee_wait_event_timeout(cond, MAX_SCHEDULE_TIMEOUT);\ - __ret1 < 0 ? -1 : 0; \ -}) - -static int __cpuinit trustee_thread(void *__gcwq) +static void gcwq_unbind_fn(struct work_struct *work) { - struct global_cwq *gcwq = __gcwq; + struct global_cwq *gcwq = get_gcwq(smp_processor_id()); struct worker_pool *pool; struct worker *worker; - struct work_struct *work; struct hlist_node *pos; int i; @@ -3505,119 +3403,29 @@ static int __cpuinit trustee_thread(void *__gcwq) gcwq->flags |= GCWQ_DISASSOCIATED; + spin_unlock_irq(&gcwq->lock); + gcwq_release_management(gcwq); + /* * Call schedule() so that we cross rq->lock and thus can guarantee - * sched callbacks see the unbound flag. This is necessary as - * scheduler callbacks may be invoked from other cpus. + * sched callbacks see the %WORKER_UNBOUND flag. This is necessary + * as scheduler callbacks may be invoked from other cpus. */ - spin_unlock_irq(&gcwq->lock); schedule(); - spin_lock_irq(&gcwq->lock); /* - * Sched callbacks are disabled now. Zap nr_running. After - * this, nr_running stays zero and need_more_worker() and - * keep_working() are always true as long as the worklist is - * not empty. + * Sched callbacks are disabled now. Zap nr_running. After this, + * nr_running stays zero and need_more_worker() and keep_working() + * are always true as long as the worklist is not empty. @gcwq now + * behaves as unbound (in terms of concurrency management) gcwq + * which is served by workers tied to the CPU. + * + * On return from this function, the current worker would trigger + * unbound chain execution of pending work items if other workers + * didn't already. */ for_each_worker_pool(pool, gcwq) atomic_set(get_pool_nr_running(pool), 0); - - spin_unlock_irq(&gcwq->lock); - for_each_worker_pool(pool, gcwq) - del_timer_sync(&pool->idle_timer); - spin_lock_irq(&gcwq->lock); - - /* - * We're now in charge. Notify and proceed to drain. We need - * to keep the gcwq running during the whole CPU down - * procedure as other cpu hotunplug callbacks may need to - * flush currently running tasks. - */ - gcwq->trustee_state = TRUSTEE_IN_CHARGE; - wake_up_all(&gcwq->trustee_wait); - - /* - * The original cpu is in the process of dying and may go away - * anytime now. When that happens, we and all workers would - * be migrated to other cpus. Try draining any left work. We - * want to get it over with ASAP - spam rescuers, wake up as - * many idlers as necessary and create new ones till the - * worklist is empty. Note that if the gcwq is frozen, there - * may be frozen works in freezable cwqs. Don't declare - * completion while frozen. - */ - while (true) { - bool busy = false; - - for_each_worker_pool(pool, gcwq) - busy |= pool->nr_workers != pool->nr_idle; - - if (!busy && !(gcwq->flags & GCWQ_FREEZING) && - gcwq->trustee_state != TRUSTEE_IN_CHARGE) - break; - - for_each_worker_pool(pool, gcwq) { - int nr_works = 0; - - list_for_each_entry(work, &pool->worklist, entry) { - send_mayday(work); - nr_works++; - } - - list_for_each_entry(worker, &pool->idle_list, entry) { - if (!nr_works--) - break; - wake_up_process(worker->task); - } - - if (need_to_create_worker(pool)) { - spin_unlock_irq(&gcwq->lock); - worker = create_worker(pool); - spin_lock_irq(&gcwq->lock); - if (worker) - start_worker(worker); - } - } - - /* give a breather */ - if (trustee_wait_event_timeout(false, TRUSTEE_COOLDOWN) < 0) - break; - } - - gcwq_release_management(gcwq); - - /* notify completion */ - gcwq->trustee = NULL; - gcwq->trustee_state = TRUSTEE_DONE; - wake_up_all(&gcwq->trustee_wait); - spin_unlock_irq(&gcwq->lock); - return 0; -} - -/** - * wait_trustee_state - wait for trustee to enter the specified state - * @gcwq: gcwq the trustee of interest belongs to - * @state: target state to wait for - * - * Wait for the trustee to reach @state. DONE is already matched. - * - * CONTEXT: - * spin_lock_irq(gcwq->lock) which may be released and regrabbed - * multiple times. To be used by cpu_callback. - */ -static void __cpuinit wait_trustee_state(struct global_cwq *gcwq, int state) -__releases(&gcwq->lock) -__acquires(&gcwq->lock) -{ - if (!(gcwq->trustee_state == state || - gcwq->trustee_state == TRUSTEE_DONE)) { - spin_unlock_irq(&gcwq->lock); - __wait_event(gcwq->trustee_wait, - gcwq->trustee_state == state || - gcwq->trustee_state == TRUSTEE_DONE); - spin_lock_irq(&gcwq->lock); - } } static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, @@ -3626,19 +3434,18 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, { unsigned int cpu = (unsigned long)hcpu; struct global_cwq *gcwq = get_gcwq(cpu); - struct task_struct *new_trustee = NULL; struct worker_pool *pool; + struct work_struct unbind_work; unsigned long flags; action &= ~CPU_TASKS_FROZEN; switch (action) { case CPU_DOWN_PREPARE: - new_trustee = kthread_create(trustee_thread, gcwq, - "workqueue_trustee/%d\n", cpu); - if (IS_ERR(new_trustee)) - return notifier_from_errno(PTR_ERR(new_trustee)); - kthread_bind(new_trustee, cpu); + /* unbinding should happen on the local CPU */ + INIT_WORK_ONSTACK(&unbind_work, gcwq_unbind_fn); + schedule_work_on(cpu, &unbind_work); + flush_work(&unbind_work); break; case CPU_UP_PREPARE: @@ -3662,27 +3469,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, spin_lock_irqsave(&gcwq->lock, flags); switch (action) { - case CPU_DOWN_PREPARE: - /* initialize trustee and tell it to acquire the gcwq */ - BUG_ON(gcwq->trustee || gcwq->trustee_state != TRUSTEE_DONE); - gcwq->trustee = new_trustee; - gcwq->trustee_state = TRUSTEE_START; - wake_up_process(gcwq->trustee); - wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); - break; - - case CPU_POST_DEAD: - gcwq->trustee_state = TRUSTEE_BUTCHER; - break; - case CPU_DOWN_FAILED: case CPU_ONLINE: - if (gcwq->trustee_state != TRUSTEE_DONE) { - gcwq->trustee_state = TRUSTEE_RELEASE; - wake_up_process(gcwq->trustee); - wait_trustee_state(gcwq, TRUSTEE_DONE); - } - spin_unlock_irq(&gcwq->lock); gcwq_claim_management(gcwq); spin_lock_irq(&gcwq->lock); @@ -3727,7 +3515,6 @@ static int __devinit workqueue_cpu_down_callback(struct notifier_block *nfb, { switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_PREPARE: - case CPU_POST_DEAD: return workqueue_cpu_callback(nfb, action, hcpu); } return NOTIFY_OK; @@ -3960,9 +3747,6 @@ static int __init init_workqueues(void) } init_waitqueue_head(&gcwq->rebind_hold); - - gcwq->trustee_state = TRUSTEE_DONE; - init_waitqueue_head(&gcwq->trustee_wait); } /* create the initial worker */ -- cgit v1.2.3-55-g7522 From 8db25e7891a47e03db6f04344a9c92be16e391bb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jul 2012 12:39:28 -0700 Subject: workqueue: simplify CPU hotplug code With trustee gone, CPU hotplug code can be simplified. * gcwq_claim/release_management() now grab and release gcwq lock too respectively and gained _and_lock and _and_unlock postfixes. * All CPU hotplug logic was implemented in workqueue_cpu_callback() which was called by workqueue_cpu_up/down_callback() for the correct priority. This was because up and down paths shared a lot of logic, which is no longer true. Remove workqueue_cpu_callback() and move all hotplug logic into the two actual callbacks. This patch doesn't make any functional changes. Signed-off-by: Tejun Heo Acked-by: "Rafael J. Wysocki" --- kernel/workqueue.c | 79 +++++++++++++++++------------------------------------- 1 file changed, 25 insertions(+), 54 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index d1545daa74ad..471996a81633 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3358,19 +3358,21 @@ EXPORT_SYMBOL_GPL(work_busy); */ /* claim manager positions of all pools */ -static void gcwq_claim_management(struct global_cwq *gcwq) +static void gcwq_claim_management_and_lock(struct global_cwq *gcwq) { struct worker_pool *pool; for_each_worker_pool(pool, gcwq) mutex_lock_nested(&pool->manager_mutex, pool - gcwq->pools); + spin_lock_irq(&gcwq->lock); } /* release manager positions */ -static void gcwq_release_management(struct global_cwq *gcwq) +static void gcwq_release_management_and_unlock(struct global_cwq *gcwq) { struct worker_pool *pool; + spin_unlock_irq(&gcwq->lock); for_each_worker_pool(pool, gcwq) mutex_unlock(&pool->manager_mutex); } @@ -3385,8 +3387,7 @@ static void gcwq_unbind_fn(struct work_struct *work) BUG_ON(gcwq->cpu != smp_processor_id()); - gcwq_claim_management(gcwq); - spin_lock_irq(&gcwq->lock); + gcwq_claim_management_and_lock(gcwq); /* * We've claimed all manager positions. Make all workers unbound @@ -3403,8 +3404,7 @@ static void gcwq_unbind_fn(struct work_struct *work) gcwq->flags |= GCWQ_DISASSOCIATED; - spin_unlock_irq(&gcwq->lock); - gcwq_release_management(gcwq); + gcwq_release_management_and_unlock(gcwq); /* * Call schedule() so that we cross rq->lock and thus can guarantee @@ -3428,26 +3428,19 @@ static void gcwq_unbind_fn(struct work_struct *work) atomic_set(get_pool_nr_running(pool), 0); } -static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +/* + * Workqueues should be brought up before normal priority CPU notifiers. + * This will be registered high priority CPU notifier. + */ +static int __devinit workqueue_cpu_up_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) { unsigned int cpu = (unsigned long)hcpu; struct global_cwq *gcwq = get_gcwq(cpu); struct worker_pool *pool; - struct work_struct unbind_work; - unsigned long flags; - - action &= ~CPU_TASKS_FROZEN; - - switch (action) { - case CPU_DOWN_PREPARE: - /* unbinding should happen on the local CPU */ - INIT_WORK_ONSTACK(&unbind_work, gcwq_unbind_fn); - schedule_work_on(cpu, &unbind_work); - flush_work(&unbind_work); - break; + switch (action & ~CPU_TASKS_FROZEN) { case CPU_UP_PREPARE: for_each_worker_pool(pool, gcwq) { struct worker *worker; @@ -3463,45 +3456,16 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, start_worker(worker); spin_unlock_irq(&gcwq->lock); } - } - - /* some are called w/ irq disabled, don't disturb irq status */ - spin_lock_irqsave(&gcwq->lock, flags); + break; - switch (action) { case CPU_DOWN_FAILED: case CPU_ONLINE: - spin_unlock_irq(&gcwq->lock); - gcwq_claim_management(gcwq); - spin_lock_irq(&gcwq->lock); - + gcwq_claim_management_and_lock(gcwq); gcwq->flags &= ~GCWQ_DISASSOCIATED; - rebind_workers(gcwq); - - gcwq_release_management(gcwq); + gcwq_release_management_and_unlock(gcwq); break; } - - spin_unlock_irqrestore(&gcwq->lock, flags); - - return notifier_from_errno(0); -} - -/* - * Workqueues should be brought up before normal priority CPU notifiers. - * This will be registered high priority CPU notifier. - */ -static int __devinit workqueue_cpu_up_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) -{ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_UP_PREPARE: - case CPU_DOWN_FAILED: - case CPU_ONLINE: - return workqueue_cpu_callback(nfb, action, hcpu); - } return NOTIFY_OK; } @@ -3513,9 +3477,16 @@ static int __devinit workqueue_cpu_down_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { + unsigned int cpu = (unsigned long)hcpu; + struct work_struct unbind_work; + switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_PREPARE: - return workqueue_cpu_callback(nfb, action, hcpu); + /* unbinding should happen on the local CPU */ + INIT_WORK_ONSTACK(&unbind_work, gcwq_unbind_fn); + schedule_work_on(cpu, &unbind_work); + flush_work(&unbind_work); + break; } return NOTIFY_OK; } -- cgit v1.2.3-55-g7522 From 9a2e03d8ed518a61154f18d83d6466628e519f94 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 19 Jul 2012 13:52:53 -0700 Subject: kthread_worker: reorganize to prepare for flush_kthread_work() reimplementation Make the following two non-functional changes. * Separate out insert_kthread_work() from queue_kthread_work(). * Relocate struct kthread_flush_work and kthread_flush_work_fn() definitions above flush_kthread_work(). v2: Added lockdep_assert_held() in insert_kthread_work() as suggested by Andy Walls. Signed-off-by: Tejun Heo Acked-by: Andy Walls --- kernel/kthread.c | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/kthread.c b/kernel/kthread.c index 3d3de633702e..4bfbff36d447 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -378,6 +378,19 @@ repeat: } EXPORT_SYMBOL_GPL(kthread_worker_fn); +/* insert @work before @pos in @worker */ +static void insert_kthread_work(struct kthread_worker *worker, + struct kthread_work *work, + struct list_head *pos) +{ + lockdep_assert_held(&worker->lock); + + list_add_tail(&work->node, pos); + work->queue_seq++; + if (likely(worker->task)) + wake_up_process(worker->task); +} + /** * queue_kthread_work - queue a kthread_work * @worker: target kthread_worker @@ -395,10 +408,7 @@ bool queue_kthread_work(struct kthread_worker *worker, spin_lock_irqsave(&worker->lock, flags); if (list_empty(&work->node)) { - list_add_tail(&work->node, &worker->work_list); - work->queue_seq++; - if (likely(worker->task)) - wake_up_process(worker->task); + insert_kthread_work(worker, work, &worker->work_list); ret = true; } spin_unlock_irqrestore(&worker->lock, flags); @@ -406,6 +416,18 @@ bool queue_kthread_work(struct kthread_worker *worker, } EXPORT_SYMBOL_GPL(queue_kthread_work); +struct kthread_flush_work { + struct kthread_work work; + struct completion done; +}; + +static void kthread_flush_work_fn(struct kthread_work *work) +{ + struct kthread_flush_work *fwork = + container_of(work, struct kthread_flush_work, work); + complete(&fwork->done); +} + /** * flush_kthread_work - flush a kthread_work * @work: work to flush @@ -436,18 +458,6 @@ void flush_kthread_work(struct kthread_work *work) } EXPORT_SYMBOL_GPL(flush_kthread_work); -struct kthread_flush_work { - struct kthread_work work; - struct completion done; -}; - -static void kthread_flush_work_fn(struct kthread_work *work) -{ - struct kthread_flush_work *fwork = - container_of(work, struct kthread_flush_work, work); - complete(&fwork->done); -} - /** * flush_kthread_worker - flush all current works on a kthread_worker * @worker: worker to flush -- cgit v1.2.3-55-g7522 From 46f3d976213452350f9d10b0c2780c2681f7075b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 19 Jul 2012 13:52:53 -0700 Subject: kthread_worker: reimplement flush_kthread_work() to allow freeing the work item being executed kthread_worker provides minimalistic workqueue-like interface for users which need a dedicated worker thread (e.g. for realtime priority). It has basic queue, flush_work, flush_worker operations which mostly match the workqueue counterparts; however, due to the way flush_work() is implemented, it has a noticeable difference of not allowing work items to be freed while being executed. While the current users of kthread_worker are okay with the current behavior, the restriction does impede some valid use cases. Also, removing this difference isn't difficult and actually makes the code easier to understand. This patch reimplements flush_kthread_work() such that it uses a flush_work item instead of queue/done sequence numbers. Signed-off-by: Tejun Heo --- include/linux/kthread.h | 8 ++------ kernel/kthread.c | 48 +++++++++++++++++++++++++++--------------------- 2 files changed, 29 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 0714b24c0e45..22ccf9dee177 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -49,8 +49,6 @@ extern int tsk_fork_get_node(struct task_struct *tsk); * can be queued and flushed using queue/flush_kthread_work() * respectively. Queued kthread_works are processed by a kthread * running kthread_worker_fn(). - * - * A kthread_work can't be freed while it is executing. */ struct kthread_work; typedef void (*kthread_work_func_t)(struct kthread_work *work); @@ -59,15 +57,14 @@ struct kthread_worker { spinlock_t lock; struct list_head work_list; struct task_struct *task; + struct kthread_work *current_work; }; struct kthread_work { struct list_head node; kthread_work_func_t func; wait_queue_head_t done; - atomic_t flushing; - int queue_seq; - int done_seq; + struct kthread_worker *worker; }; #define KTHREAD_WORKER_INIT(worker) { \ @@ -79,7 +76,6 @@ struct kthread_work { .node = LIST_HEAD_INIT((work).node), \ .func = (fn), \ .done = __WAIT_QUEUE_HEAD_INITIALIZER((work).done), \ - .flushing = ATOMIC_INIT(0), \ } #define DEFINE_KTHREAD_WORKER(worker) \ diff --git a/kernel/kthread.c b/kernel/kthread.c index 4bfbff36d447..b579af57ea10 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -360,16 +360,12 @@ repeat: struct kthread_work, node); list_del_init(&work->node); } + worker->current_work = work; spin_unlock_irq(&worker->lock); if (work) { __set_current_state(TASK_RUNNING); work->func(work); - smp_wmb(); /* wmb worker-b0 paired with flush-b1 */ - work->done_seq = work->queue_seq; - smp_mb(); /* mb worker-b1 paired with flush-b0 */ - if (atomic_read(&work->flushing)) - wake_up_all(&work->done); } else if (!freezing(current)) schedule(); @@ -386,7 +382,7 @@ static void insert_kthread_work(struct kthread_worker *worker, lockdep_assert_held(&worker->lock); list_add_tail(&work->node, pos); - work->queue_seq++; + work->worker = worker; if (likely(worker->task)) wake_up_process(worker->task); } @@ -436,25 +432,35 @@ static void kthread_flush_work_fn(struct kthread_work *work) */ void flush_kthread_work(struct kthread_work *work) { - int seq = work->queue_seq; + struct kthread_flush_work fwork = { + KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), + COMPLETION_INITIALIZER_ONSTACK(fwork.done), + }; + struct kthread_worker *worker; + bool noop = false; + +retry: + worker = work->worker; + if (!worker) + return; - atomic_inc(&work->flushing); + spin_lock_irq(&worker->lock); + if (work->worker != worker) { + spin_unlock_irq(&worker->lock); + goto retry; + } - /* - * mb flush-b0 paired with worker-b1, to make sure either - * worker sees the above increment or we see done_seq update. - */ - smp_mb__after_atomic_inc(); + if (!list_empty(&work->node)) + insert_kthread_work(worker, &fwork.work, work->node.next); + else if (worker->current_work == work) + insert_kthread_work(worker, &fwork.work, worker->work_list.next); + else + noop = true; - /* A - B <= 0 tests whether B is in front of A regardless of overflow */ - wait_event(work->done, seq - work->done_seq <= 0); - atomic_dec(&work->flushing); + spin_unlock_irq(&worker->lock); - /* - * rmb flush-b1 paired with worker-b0, to make sure our caller - * sees every change made by work->func(). - */ - smp_mb__after_atomic_dec(); + if (!noop) + wait_for_completion(&fwork.done); } EXPORT_SYMBOL_GPL(flush_kthread_work); -- cgit v1.2.3-55-g7522 From 6fec10a1a5866dda3cd6a825a521fc7c2f226ba5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 22 Jul 2012 10:16:34 -0700 Subject: workqueue: fix spurious CPU locality WARN from process_one_work() 25511a4776 "workqueue: reimplement CPU online rebinding to handle idle workers" added CPU locality sanity check in process_one_work(). It triggers if a worker is executing on a different CPU without UNBOUND or REBIND set. This works for all normal workers but rescuers can trigger this spuriously when they're serving the unbound or a disassociated global_cwq - rescuers don't have either flag set and thus its gcwq->cpu can be a different value including %WORK_CPU_UNBOUND. Fix it by additionally testing %GCWQ_DISASSOCIATED. Signed-off-by: Tejun Heo Reported-by: "Paul E. McKenney" LKML-Refence: <20120721213656.GA7783@linux.vnet.ibm.com> --- kernel/workqueue.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 471996a81633..692d97628a10 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1949,7 +1949,13 @@ __acquires(&gcwq->lock) lockdep_copy_map(&lockdep_map, &work->lockdep_map); #endif + /* + * Ensure we're on the correct CPU. DISASSOCIATED test is + * necessary to avoid spurious warnings from rescuers servicing the + * unbound or a disassociated gcwq. + */ WARN_ON_ONCE(!(worker->flags & (WORKER_UNBOUND | WORKER_REBIND)) && + !(gcwq->flags & GCWQ_DISASSOCIATED) && raw_smp_processor_id() != gcwq->cpu); /* -- cgit v1.2.3-55-g7522