Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * kernel/workqueue.c - generic async execution with shared worker pool
4 : *
5 : * Copyright (C) 2002 Ingo Molnar
6 : *
7 : * Derived from the taskqueue/keventd code by:
8 : * David Woodhouse <dwmw2@infradead.org>
9 : * Andrew Morton
10 : * Kai Petzke <wpp@marie.physik.tu-berlin.de>
11 : * Theodore Ts'o <tytso@mit.edu>
12 : *
13 : * Made to use alloc_percpu by Christoph Lameter.
14 : *
15 : * Copyright (C) 2010 SUSE Linux Products GmbH
16 : * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
17 : *
18 : * This is the generic async execution mechanism. Work items as are
19 : * executed in process context. The worker pool is shared and
20 : * automatically managed. There are two worker pools for each CPU (one for
21 : * normal work items and the other for high priority ones) and some extra
22 : * pools for workqueues which are not bound to any specific CPU - the
23 : * number of these backing pools is dynamic.
24 : *
25 : * Please read Documentation/core-api/workqueue.rst for details.
26 : */
27 :
28 : #include <linux/export.h>
29 : #include <linux/kernel.h>
30 : #include <linux/sched.h>
31 : #include <linux/init.h>
32 : #include <linux/signal.h>
33 : #include <linux/completion.h>
34 : #include <linux/workqueue.h>
35 : #include <linux/slab.h>
36 : #include <linux/cpu.h>
37 : #include <linux/notifier.h>
38 : #include <linux/kthread.h>
39 : #include <linux/hardirq.h>
40 : #include <linux/mempolicy.h>
41 : #include <linux/freezer.h>
42 : #include <linux/debug_locks.h>
43 : #include <linux/lockdep.h>
44 : #include <linux/idr.h>
45 : #include <linux/jhash.h>
46 : #include <linux/hashtable.h>
47 : #include <linux/rculist.h>
48 : #include <linux/nodemask.h>
49 : #include <linux/moduleparam.h>
50 : #include <linux/uaccess.h>
51 : #include <linux/sched/isolation.h>
52 : #include <linux/sched/debug.h>
53 : #include <linux/nmi.h>
54 : #include <linux/kvm_para.h>
55 :
56 : #include "workqueue_internal.h"
57 :
58 : enum {
59 : /*
60 : * worker_pool flags
61 : *
62 : * A bound pool is either associated or disassociated with its CPU.
63 : * While associated (!DISASSOCIATED), all workers are bound to the
64 : * CPU and none has %WORKER_UNBOUND set and concurrency management
65 : * is in effect.
66 : *
67 : * While DISASSOCIATED, the cpu may be offline and all workers have
68 : * %WORKER_UNBOUND set and concurrency management disabled, and may
69 : * be executing on any CPU. The pool behaves as an unbound one.
70 : *
71 : * Note that DISASSOCIATED should be flipped only while holding
72 : * wq_pool_attach_mutex to avoid changing binding state while
73 : * worker_attach_to_pool() is in progress.
74 : */
75 : POOL_MANAGER_ACTIVE = 1 << 0, /* being managed */
76 : POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
77 :
78 : /* worker flags */
79 : WORKER_DIE = 1 << 1, /* die die die */
80 : WORKER_IDLE = 1 << 2, /* is idle */
81 : WORKER_PREP = 1 << 3, /* preparing to run works */
82 : WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
83 : WORKER_UNBOUND = 1 << 7, /* worker is unbound */
84 : WORKER_REBOUND = 1 << 8, /* worker was rebound */
85 :
86 : WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
87 : WORKER_UNBOUND | WORKER_REBOUND,
88 :
89 : NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
90 :
91 : UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
92 : BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
93 :
94 : MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
95 : IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
96 :
97 : MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
98 : /* call for help after 10ms
99 : (min two ticks) */
100 : MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
101 : CREATE_COOLDOWN = HZ, /* time to breath after fail */
102 :
103 : /*
104 : * Rescue workers are used only on emergencies and shared by
105 : * all cpus. Give MIN_NICE.
106 : */
107 : RESCUER_NICE_LEVEL = MIN_NICE,
108 : HIGHPRI_NICE_LEVEL = MIN_NICE,
109 :
110 : WQ_NAME_LEN = 24,
111 : };
112 :
113 : /*
114 : * Structure fields follow one of the following exclusion rules.
115 : *
116 : * I: Modifiable by initialization/destruction paths and read-only for
117 : * everyone else.
118 : *
119 : * P: Preemption protected. Disabling preemption is enough and should
120 : * only be modified and accessed from the local cpu.
121 : *
122 : * L: pool->lock protected. Access with pool->lock held.
123 : *
124 : * X: During normal operation, modification requires pool->lock and should
125 : * be done only from local cpu. Either disabling preemption on local
126 : * cpu or grabbing pool->lock is enough for read access. If
127 : * POOL_DISASSOCIATED is set, it's identical to L.
128 : *
129 : * A: wq_pool_attach_mutex protected.
130 : *
131 : * PL: wq_pool_mutex protected.
132 : *
133 : * PR: wq_pool_mutex protected for writes. RCU protected for reads.
134 : *
135 : * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
136 : *
137 : * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
138 : * RCU for reads.
139 : *
140 : * WQ: wq->mutex protected.
141 : *
142 : * WR: wq->mutex protected for writes. RCU protected for reads.
143 : *
144 : * MD: wq_mayday_lock protected.
145 : *
146 : * WD: Used internally by the watchdog.
147 : */
148 :
149 : /* struct worker is defined in workqueue_internal.h */
150 :
151 : struct worker_pool {
152 : raw_spinlock_t lock; /* the pool lock */
153 : int cpu; /* I: the associated cpu */
154 : int node; /* I: the associated node ID */
155 : int id; /* I: pool ID */
156 : unsigned int flags; /* X: flags */
157 :
158 : unsigned long watchdog_ts; /* L: watchdog timestamp */
159 : bool cpu_stall; /* WD: stalled cpu bound pool */
160 :
161 : /*
162 : * The counter is incremented in a process context on the associated CPU
163 : * w/ preemption disabled, and decremented or reset in the same context
164 : * but w/ pool->lock held. The readers grab pool->lock and are
165 : * guaranteed to see if the counter reached zero.
166 : */
167 : int nr_running;
168 :
169 : struct list_head worklist; /* L: list of pending works */
170 :
171 : int nr_workers; /* L: total number of workers */
172 : int nr_idle; /* L: currently idle workers */
173 :
174 : struct list_head idle_list; /* L: list of idle workers */
175 : struct timer_list idle_timer; /* L: worker idle timeout */
176 : struct work_struct idle_cull_work; /* L: worker idle cleanup */
177 :
178 : struct timer_list mayday_timer; /* L: SOS timer for workers */
179 :
180 : /* a workers is either on busy_hash or idle_list, or the manager */
181 : DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
182 : /* L: hash of busy workers */
183 :
184 : struct worker *manager; /* L: purely informational */
185 : struct list_head workers; /* A: attached workers */
186 : struct list_head dying_workers; /* A: workers about to die */
187 : struct completion *detach_completion; /* all workers detached */
188 :
189 : struct ida worker_ida; /* worker IDs for task name */
190 :
191 : struct workqueue_attrs *attrs; /* I: worker attributes */
192 : struct hlist_node hash_node; /* PL: unbound_pool_hash node */
193 : int refcnt; /* PL: refcnt for unbound pools */
194 :
195 : /*
196 : * Destruction of pool is RCU protected to allow dereferences
197 : * from get_work_pool().
198 : */
199 : struct rcu_head rcu;
200 : };
201 :
202 : /*
203 : * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
204 : * of work_struct->data are used for flags and the remaining high bits
205 : * point to the pwq; thus, pwqs need to be aligned at two's power of the
206 : * number of flag bits.
207 : */
208 : struct pool_workqueue {
209 : struct worker_pool *pool; /* I: the associated pool */
210 : struct workqueue_struct *wq; /* I: the owning workqueue */
211 : int work_color; /* L: current color */
212 : int flush_color; /* L: flushing color */
213 : int refcnt; /* L: reference count */
214 : int nr_in_flight[WORK_NR_COLORS];
215 : /* L: nr of in_flight works */
216 :
217 : /*
218 : * nr_active management and WORK_STRUCT_INACTIVE:
219 : *
220 : * When pwq->nr_active >= max_active, new work item is queued to
221 : * pwq->inactive_works instead of pool->worklist and marked with
222 : * WORK_STRUCT_INACTIVE.
223 : *
224 : * All work items marked with WORK_STRUCT_INACTIVE do not participate
225 : * in pwq->nr_active and all work items in pwq->inactive_works are
226 : * marked with WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE
227 : * work items are in pwq->inactive_works. Some of them are ready to
228 : * run in pool->worklist or worker->scheduled. Those work itmes are
229 : * only struct wq_barrier which is used for flush_work() and should
230 : * not participate in pwq->nr_active. For non-barrier work item, it
231 : * is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
232 : */
233 : int nr_active; /* L: nr of active works */
234 : int max_active; /* L: max active works */
235 : struct list_head inactive_works; /* L: inactive works */
236 : struct list_head pwqs_node; /* WR: node on wq->pwqs */
237 : struct list_head mayday_node; /* MD: node on wq->maydays */
238 :
239 : /*
240 : * Release of unbound pwq is punted to system_wq. See put_pwq()
241 : * and pwq_unbound_release_workfn() for details. pool_workqueue
242 : * itself is also RCU protected so that the first pwq can be
243 : * determined without grabbing wq->mutex.
244 : */
245 : struct work_struct unbound_release_work;
246 : struct rcu_head rcu;
247 : } __aligned(1 << WORK_STRUCT_FLAG_BITS);
248 :
249 : /*
250 : * Structure used to wait for workqueue flush.
251 : */
252 : struct wq_flusher {
253 : struct list_head list; /* WQ: list of flushers */
254 : int flush_color; /* WQ: flush color waiting for */
255 : struct completion done; /* flush completion */
256 : };
257 :
258 : struct wq_device;
259 :
260 : /*
261 : * The externally visible workqueue. It relays the issued work items to
262 : * the appropriate worker_pool through its pool_workqueues.
263 : */
264 : struct workqueue_struct {
265 : struct list_head pwqs; /* WR: all pwqs of this wq */
266 : struct list_head list; /* PR: list of all workqueues */
267 :
268 : struct mutex mutex; /* protects this wq */
269 : int work_color; /* WQ: current work color */
270 : int flush_color; /* WQ: current flush color */
271 : atomic_t nr_pwqs_to_flush; /* flush in progress */
272 : struct wq_flusher *first_flusher; /* WQ: first flusher */
273 : struct list_head flusher_queue; /* WQ: flush waiters */
274 : struct list_head flusher_overflow; /* WQ: flush overflow list */
275 :
276 : struct list_head maydays; /* MD: pwqs requesting rescue */
277 : struct worker *rescuer; /* MD: rescue worker */
278 :
279 : int nr_drainers; /* WQ: drain in progress */
280 : int saved_max_active; /* WQ: saved pwq max_active */
281 :
282 : struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
283 : struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
284 :
285 : #ifdef CONFIG_SYSFS
286 : struct wq_device *wq_dev; /* I: for sysfs interface */
287 : #endif
288 : #ifdef CONFIG_LOCKDEP
289 : char *lock_name;
290 : struct lock_class_key key;
291 : struct lockdep_map lockdep_map;
292 : #endif
293 : char name[WQ_NAME_LEN]; /* I: workqueue name */
294 :
295 : /*
296 : * Destruction of workqueue_struct is RCU protected to allow walking
297 : * the workqueues list without grabbing wq_pool_mutex.
298 : * This is used to dump all workqueues from sysrq.
299 : */
300 : struct rcu_head rcu;
301 :
302 : /* hot fields used during command issue, aligned to cacheline */
303 : unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
304 : struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
305 : struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
306 : };
307 :
308 : static struct kmem_cache *pwq_cache;
309 :
310 : static cpumask_var_t *wq_numa_possible_cpumask;
311 : /* possible CPUs of each node */
312 :
313 : static bool wq_disable_numa;
314 : module_param_named(disable_numa, wq_disable_numa, bool, 0444);
315 :
316 : /* see the comment above the definition of WQ_POWER_EFFICIENT */
317 : static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
318 : module_param_named(power_efficient, wq_power_efficient, bool, 0444);
319 :
320 : static bool wq_online; /* can kworkers be created yet? */
321 :
322 : static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
323 :
324 : /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
325 : static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
326 :
327 : static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
328 : static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
329 : static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
330 : /* wait for manager to go away */
331 : static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
332 :
333 : static LIST_HEAD(workqueues); /* PR: list of all workqueues */
334 : static bool workqueue_freezing; /* PL: have wqs started freezing? */
335 :
336 : /* PL&A: allowable cpus for unbound wqs and work items */
337 : static cpumask_var_t wq_unbound_cpumask;
338 :
339 : /* CPU where unbound work was last round robin scheduled from this CPU */
340 : static DEFINE_PER_CPU(int, wq_rr_cpu_last);
341 :
342 : /*
343 : * Local execution of unbound work items is no longer guaranteed. The
344 : * following always forces round-robin CPU selection on unbound work items
345 : * to uncover usages which depend on it.
346 : */
347 : #ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
348 : static bool wq_debug_force_rr_cpu = true;
349 : #else
350 : static bool wq_debug_force_rr_cpu = false;
351 : #endif
352 : module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
353 :
354 : /* the per-cpu worker pools */
355 : static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
356 :
357 : static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
358 :
359 : /* PL: hash of all unbound pools keyed by pool->attrs */
360 : static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
361 :
362 : /* I: attributes used when instantiating standard unbound pools on demand */
363 : static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
364 :
365 : /* I: attributes used when instantiating ordered pools on demand */
366 : static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
367 :
368 : struct workqueue_struct *system_wq __read_mostly;
369 : EXPORT_SYMBOL(system_wq);
370 : struct workqueue_struct *system_highpri_wq __read_mostly;
371 : EXPORT_SYMBOL_GPL(system_highpri_wq);
372 : struct workqueue_struct *system_long_wq __read_mostly;
373 : EXPORT_SYMBOL_GPL(system_long_wq);
374 : struct workqueue_struct *system_unbound_wq __read_mostly;
375 : EXPORT_SYMBOL_GPL(system_unbound_wq);
376 : struct workqueue_struct *system_freezable_wq __read_mostly;
377 : EXPORT_SYMBOL_GPL(system_freezable_wq);
378 : struct workqueue_struct *system_power_efficient_wq __read_mostly;
379 : EXPORT_SYMBOL_GPL(system_power_efficient_wq);
380 : struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
381 : EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
382 :
383 : static int worker_thread(void *__worker);
384 : static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
385 : static void show_pwq(struct pool_workqueue *pwq);
386 : static void show_one_worker_pool(struct worker_pool *pool);
387 :
388 : #define CREATE_TRACE_POINTS
389 : #include <trace/events/workqueue.h>
390 :
391 : #define assert_rcu_or_pool_mutex() \
392 : RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
393 : !lockdep_is_held(&wq_pool_mutex), \
394 : "RCU or wq_pool_mutex should be held")
395 :
396 : #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
397 : RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
398 : !lockdep_is_held(&wq->mutex) && \
399 : !lockdep_is_held(&wq_pool_mutex), \
400 : "RCU, wq->mutex or wq_pool_mutex should be held")
401 :
402 : #define for_each_cpu_worker_pool(pool, cpu) \
403 : for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
404 : (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
405 : (pool)++)
406 :
407 : /**
408 : * for_each_pool - iterate through all worker_pools in the system
409 : * @pool: iteration cursor
410 : * @pi: integer used for iteration
411 : *
412 : * This must be called either with wq_pool_mutex held or RCU read
413 : * locked. If the pool needs to be used beyond the locking in effect, the
414 : * caller is responsible for guaranteeing that the pool stays online.
415 : *
416 : * The if/else clause exists only for the lockdep assertion and can be
417 : * ignored.
418 : */
419 : #define for_each_pool(pool, pi) \
420 : idr_for_each_entry(&worker_pool_idr, pool, pi) \
421 : if (({ assert_rcu_or_pool_mutex(); false; })) { } \
422 : else
423 :
424 : /**
425 : * for_each_pool_worker - iterate through all workers of a worker_pool
426 : * @worker: iteration cursor
427 : * @pool: worker_pool to iterate workers of
428 : *
429 : * This must be called with wq_pool_attach_mutex.
430 : *
431 : * The if/else clause exists only for the lockdep assertion and can be
432 : * ignored.
433 : */
434 : #define for_each_pool_worker(worker, pool) \
435 : list_for_each_entry((worker), &(pool)->workers, node) \
436 : if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
437 : else
438 :
439 : /**
440 : * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
441 : * @pwq: iteration cursor
442 : * @wq: the target workqueue
443 : *
444 : * This must be called either with wq->mutex held or RCU read locked.
445 : * If the pwq needs to be used beyond the locking in effect, the caller is
446 : * responsible for guaranteeing that the pwq stays online.
447 : *
448 : * The if/else clause exists only for the lockdep assertion and can be
449 : * ignored.
450 : */
451 : #define for_each_pwq(pwq, wq) \
452 : list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
453 : lockdep_is_held(&(wq->mutex)))
454 :
455 : #ifdef CONFIG_DEBUG_OBJECTS_WORK
456 :
457 : static const struct debug_obj_descr work_debug_descr;
458 :
459 : static void *work_debug_hint(void *addr)
460 : {
461 : return ((struct work_struct *) addr)->func;
462 : }
463 :
464 : static bool work_is_static_object(void *addr)
465 : {
466 : struct work_struct *work = addr;
467 :
468 : return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
469 : }
470 :
471 : /*
472 : * fixup_init is called when:
473 : * - an active object is initialized
474 : */
475 : static bool work_fixup_init(void *addr, enum debug_obj_state state)
476 : {
477 : struct work_struct *work = addr;
478 :
479 : switch (state) {
480 : case ODEBUG_STATE_ACTIVE:
481 : cancel_work_sync(work);
482 : debug_object_init(work, &work_debug_descr);
483 : return true;
484 : default:
485 : return false;
486 : }
487 : }
488 :
489 : /*
490 : * fixup_free is called when:
491 : * - an active object is freed
492 : */
493 : static bool work_fixup_free(void *addr, enum debug_obj_state state)
494 : {
495 : struct work_struct *work = addr;
496 :
497 : switch (state) {
498 : case ODEBUG_STATE_ACTIVE:
499 : cancel_work_sync(work);
500 : debug_object_free(work, &work_debug_descr);
501 : return true;
502 : default:
503 : return false;
504 : }
505 : }
506 :
507 : static const struct debug_obj_descr work_debug_descr = {
508 : .name = "work_struct",
509 : .debug_hint = work_debug_hint,
510 : .is_static_object = work_is_static_object,
511 : .fixup_init = work_fixup_init,
512 : .fixup_free = work_fixup_free,
513 : };
514 :
515 : static inline void debug_work_activate(struct work_struct *work)
516 : {
517 : debug_object_activate(work, &work_debug_descr);
518 : }
519 :
520 : static inline void debug_work_deactivate(struct work_struct *work)
521 : {
522 : debug_object_deactivate(work, &work_debug_descr);
523 : }
524 :
525 : void __init_work(struct work_struct *work, int onstack)
526 : {
527 : if (onstack)
528 : debug_object_init_on_stack(work, &work_debug_descr);
529 : else
530 : debug_object_init(work, &work_debug_descr);
531 : }
532 : EXPORT_SYMBOL_GPL(__init_work);
533 :
534 : void destroy_work_on_stack(struct work_struct *work)
535 : {
536 : debug_object_free(work, &work_debug_descr);
537 : }
538 : EXPORT_SYMBOL_GPL(destroy_work_on_stack);
539 :
540 : void destroy_delayed_work_on_stack(struct delayed_work *work)
541 : {
542 : destroy_timer_on_stack(&work->timer);
543 : debug_object_free(&work->work, &work_debug_descr);
544 : }
545 : EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
546 :
547 : #else
548 : static inline void debug_work_activate(struct work_struct *work) { }
549 : static inline void debug_work_deactivate(struct work_struct *work) { }
550 : #endif
551 :
552 : /**
553 : * worker_pool_assign_id - allocate ID and assign it to @pool
554 : * @pool: the pool pointer of interest
555 : *
556 : * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
557 : * successfully, -errno on failure.
558 : */
559 : static int worker_pool_assign_id(struct worker_pool *pool)
560 : {
561 : int ret;
562 :
563 : lockdep_assert_held(&wq_pool_mutex);
564 :
565 3 : ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
566 : GFP_KERNEL);
567 3 : if (ret >= 0) {
568 3 : pool->id = ret;
569 : return 0;
570 : }
571 : return ret;
572 : }
573 :
574 : /**
575 : * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
576 : * @wq: the target workqueue
577 : * @node: the node ID
578 : *
579 : * This must be called with any of wq_pool_mutex, wq->mutex or RCU
580 : * read locked.
581 : * If the pwq needs to be used beyond the locking in effect, the caller is
582 : * responsible for guaranteeing that the pwq stays online.
583 : *
584 : * Return: The unbound pool_workqueue for @node.
585 : */
586 : static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
587 : int node)
588 : {
589 : assert_rcu_or_wq_mutex_or_pool_mutex(wq);
590 :
591 : /*
592 : * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
593 : * delayed item is pending. The plan is to keep CPU -> NODE
594 : * mapping valid and stable across CPU on/offlines. Once that
595 : * happens, this workaround can be removed.
596 : */
597 : if (unlikely(node == NUMA_NO_NODE))
598 : return wq->dfl_pwq;
599 :
600 33 : return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
601 : }
602 :
603 : static unsigned int work_color_to_flags(int color)
604 : {
605 86 : return color << WORK_STRUCT_COLOR_SHIFT;
606 : }
607 :
608 : static int get_work_color(unsigned long work_data)
609 : {
610 174 : return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
611 : ((1 << WORK_STRUCT_COLOR_BITS) - 1);
612 : }
613 :
614 : static int work_next_color(int color)
615 : {
616 0 : return (color + 1) % WORK_NR_COLORS;
617 : }
618 :
619 : /*
620 : * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
621 : * contain the pointer to the queued pwq. Once execution starts, the flag
622 : * is cleared and the high bits contain OFFQ flags and pool ID.
623 : *
624 : * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
625 : * and clear_work_data() can be used to set the pwq, pool or clear
626 : * work->data. These functions should only be called while the work is
627 : * owned - ie. while the PENDING bit is set.
628 : *
629 : * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
630 : * corresponding to a work. Pool is available once the work has been
631 : * queued anywhere after initialization until it is sync canceled. pwq is
632 : * available only while the work item is queued.
633 : *
634 : * %WORK_OFFQ_CANCELING is used to mark a work item which is being
635 : * canceled. While being canceled, a work item may have its PENDING set
636 : * but stay off timer and worklist for arbitrarily long and nobody should
637 : * try to steal the PENDING bit.
638 : */
639 210 : static inline void set_work_data(struct work_struct *work, unsigned long data,
640 : unsigned long flags)
641 : {
642 420 : WARN_ON_ONCE(!work_pending(work));
643 420 : atomic_long_set(&work->data, data | flags | work_static(work));
644 210 : }
645 :
646 : static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
647 : unsigned long extra_flags)
648 : {
649 86 : set_work_data(work, (unsigned long)pwq,
650 : WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
651 : }
652 :
653 : static void set_work_pool_and_keep_pending(struct work_struct *work,
654 : int pool_id)
655 : {
656 0 : set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
657 : WORK_STRUCT_PENDING);
658 : }
659 :
660 : static void set_work_pool_and_clear_pending(struct work_struct *work,
661 : int pool_id)
662 : {
663 : /*
664 : * The following wmb is paired with the implied mb in
665 : * test_and_set_bit(PENDING) and ensures all updates to @work made
666 : * here are visible to and precede any updates by the next PENDING
667 : * owner.
668 : */
669 124 : smp_wmb();
670 124 : set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
671 : /*
672 : * The following mb guarantees that previous clear of a PENDING bit
673 : * will not be reordered with any speculative LOADS or STORES from
674 : * work->current_func, which is executed afterwards. This possible
675 : * reordering can lead to a missed execution on attempt to queue
676 : * the same @work. E.g. consider this case:
677 : *
678 : * CPU#0 CPU#1
679 : * ---------------------------- --------------------------------
680 : *
681 : * 1 STORE event_indicated
682 : * 2 queue_work_on() {
683 : * 3 test_and_set_bit(PENDING)
684 : * 4 } set_..._and_clear_pending() {
685 : * 5 set_work_data() # clear bit
686 : * 6 smp_mb()
687 : * 7 work->current_func() {
688 : * 8 LOAD event_indicated
689 : * }
690 : *
691 : * Without an explicit full barrier speculative LOAD on line 8 can
692 : * be executed before CPU#0 does STORE on line 1. If that happens,
693 : * CPU#0 observes the PENDING bit is still set and new execution of
694 : * a @work is not queued in a hope, that CPU#1 will eventually
695 : * finish the queued @work. Meanwhile CPU#1 does not see
696 : * event_indicated is set, because speculative LOAD was executed
697 : * before actual STORE.
698 : */
699 124 : smp_mb();
700 : }
701 :
702 : static void clear_work_data(struct work_struct *work)
703 : {
704 0 : smp_wmb(); /* see set_work_pool_and_clear_pending() */
705 0 : set_work_data(work, WORK_STRUCT_NO_POOL, 0);
706 : }
707 :
708 : static struct pool_workqueue *get_work_pwq(struct work_struct *work)
709 : {
710 180 : unsigned long data = atomic_long_read(&work->data);
711 :
712 90 : if (data & WORK_STRUCT_PWQ)
713 88 : return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
714 : else
715 : return NULL;
716 : }
717 :
718 : /**
719 : * get_work_pool - return the worker_pool a given work was associated with
720 : * @work: the work item of interest
721 : *
722 : * Pools are created and destroyed under wq_pool_mutex, and allows read
723 : * access under RCU read lock. As such, this function should be
724 : * called under wq_pool_mutex or inside of a rcu_read_lock() region.
725 : *
726 : * All fields of the returned pool are accessible as long as the above
727 : * mentioned locking is in effect. If the returned pool needs to be used
728 : * beyond the critical section, the caller is responsible for ensuring the
729 : * returned pool is and stays online.
730 : *
731 : * Return: The worker_pool @work was last associated with. %NULL if none.
732 : */
733 110 : static struct worker_pool *get_work_pool(struct work_struct *work)
734 : {
735 220 : unsigned long data = atomic_long_read(&work->data);
736 : int pool_id;
737 :
738 : assert_rcu_or_pool_mutex();
739 :
740 110 : if (data & WORK_STRUCT_PWQ)
741 2 : return ((struct pool_workqueue *)
742 4 : (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
743 :
744 108 : pool_id = data >> WORK_OFFQ_POOL_SHIFT;
745 108 : if (pool_id == WORK_OFFQ_POOL_NONE)
746 : return NULL;
747 :
748 58 : return idr_find(&worker_pool_idr, pool_id);
749 : }
750 :
751 : /**
752 : * get_work_pool_id - return the worker pool ID a given work is associated with
753 : * @work: the work item of interest
754 : *
755 : * Return: The worker_pool ID @work was last associated with.
756 : * %WORK_OFFQ_POOL_NONE if none.
757 : */
758 : static int get_work_pool_id(struct work_struct *work)
759 : {
760 76 : unsigned long data = atomic_long_read(&work->data);
761 :
762 38 : if (data & WORK_STRUCT_PWQ)
763 0 : return ((struct pool_workqueue *)
764 0 : (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
765 :
766 38 : return data >> WORK_OFFQ_POOL_SHIFT;
767 : }
768 :
769 0 : static void mark_work_canceling(struct work_struct *work)
770 : {
771 0 : unsigned long pool_id = get_work_pool_id(work);
772 :
773 0 : pool_id <<= WORK_OFFQ_POOL_SHIFT;
774 0 : set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
775 0 : }
776 :
777 : static bool work_is_canceling(struct work_struct *work)
778 : {
779 0 : unsigned long data = atomic_long_read(&work->data);
780 :
781 0 : return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
782 : }
783 :
784 : /*
785 : * Policy functions. These define the policies on how the global worker
786 : * pools are managed. Unless noted otherwise, these functions assume that
787 : * they're being called with pool->lock held.
788 : */
789 :
790 : static bool __need_more_worker(struct worker_pool *pool)
791 : {
792 : return !pool->nr_running;
793 : }
794 :
795 : /*
796 : * Need to wake up a worker? Called from anything but currently
797 : * running workers.
798 : *
799 : * Note that, because unbound workers never contribute to nr_running, this
800 : * function will always return %true for unbound pools as long as the
801 : * worklist isn't empty.
802 : */
803 : static bool need_more_worker(struct worker_pool *pool)
804 : {
805 356 : return !list_empty(&pool->worklist) && __need_more_worker(pool);
806 : }
807 :
808 : /* Can I start working? Called from busy but !running workers. */
809 : static bool may_start_working(struct worker_pool *pool)
810 : {
811 : return pool->nr_idle;
812 : }
813 :
814 : /* Do I need to keep working? Called from currently running workers. */
815 : static bool keep_working(struct worker_pool *pool)
816 : {
817 168 : return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
818 : }
819 :
820 : /* Do we need a new worker? Called from manager. */
821 : static bool need_to_create_worker(struct worker_pool *pool)
822 : {
823 2 : return need_more_worker(pool) && !may_start_working(pool);
824 : }
825 :
826 : /* Do we have too many workers and should some go away? */
827 : static bool too_many_workers(struct worker_pool *pool)
828 : {
829 93 : bool managing = pool->flags & POOL_MANAGER_ACTIVE;
830 93 : int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
831 93 : int nr_busy = pool->nr_workers - nr_idle;
832 :
833 93 : return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
834 : }
835 :
836 : /*
837 : * Wake up functions.
838 : */
839 :
840 : /* Return the first idle worker. Called with pool->lock held. */
841 : static struct worker *first_idle_worker(struct worker_pool *pool)
842 : {
843 172 : if (unlikely(list_empty(&pool->idle_list)))
844 : return NULL;
845 :
846 86 : return list_first_entry(&pool->idle_list, struct worker, entry);
847 : }
848 :
849 : /**
850 : * wake_up_worker - wake up an idle worker
851 : * @pool: worker pool to wake worker from
852 : *
853 : * Wake up the first idle worker of @pool.
854 : *
855 : * CONTEXT:
856 : * raw_spin_lock_irq(pool->lock).
857 : */
858 : static void wake_up_worker(struct worker_pool *pool)
859 : {
860 86 : struct worker *worker = first_idle_worker(pool);
861 :
862 86 : if (likely(worker))
863 86 : wake_up_process(worker->task);
864 : }
865 :
866 : /**
867 : * wq_worker_running - a worker is running again
868 : * @task: task waking up
869 : *
870 : * This function is called when a worker returns from schedule()
871 : */
872 85 : void wq_worker_running(struct task_struct *task)
873 : {
874 85 : struct worker *worker = kthread_data(task);
875 :
876 85 : if (!worker->sleeping)
877 : return;
878 :
879 : /*
880 : * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
881 : * and the nr_running increment below, we may ruin the nr_running reset
882 : * and leave with an unexpected pool->nr_running == 1 on the newly unbound
883 : * pool. Protect against such race.
884 : */
885 0 : preempt_disable();
886 0 : if (!(worker->flags & WORKER_NOT_RUNNING))
887 0 : worker->pool->nr_running++;
888 0 : preempt_enable();
889 0 : worker->sleeping = 0;
890 : }
891 :
892 : /**
893 : * wq_worker_sleeping - a worker is going to sleep
894 : * @task: task going to sleep
895 : *
896 : * This function is called from schedule() when a busy worker is
897 : * going to sleep.
898 : */
899 94 : void wq_worker_sleeping(struct task_struct *task)
900 : {
901 94 : struct worker *worker = kthread_data(task);
902 : struct worker_pool *pool;
903 :
904 : /*
905 : * Rescuers, which may not have all the fields set up like normal
906 : * workers, also reach here, let's not access anything before
907 : * checking NOT_RUNNING.
908 : */
909 94 : if (worker->flags & WORKER_NOT_RUNNING)
910 : return;
911 :
912 0 : pool = worker->pool;
913 :
914 : /* Return if preempted before wq_worker_running() was reached */
915 0 : if (worker->sleeping)
916 : return;
917 :
918 0 : worker->sleeping = 1;
919 0 : raw_spin_lock_irq(&pool->lock);
920 :
921 : /*
922 : * Recheck in case unbind_workers() preempted us. We don't
923 : * want to decrement nr_running after the worker is unbound
924 : * and nr_running has been reset.
925 : */
926 0 : if (worker->flags & WORKER_NOT_RUNNING) {
927 0 : raw_spin_unlock_irq(&pool->lock);
928 0 : return;
929 : }
930 :
931 0 : pool->nr_running--;
932 0 : if (need_more_worker(pool))
933 : wake_up_worker(pool);
934 0 : raw_spin_unlock_irq(&pool->lock);
935 : }
936 :
937 : /**
938 : * wq_worker_last_func - retrieve worker's last work function
939 : * @task: Task to retrieve last work function of.
940 : *
941 : * Determine the last function a worker executed. This is called from
942 : * the scheduler to get a worker's last known identity.
943 : *
944 : * CONTEXT:
945 : * raw_spin_lock_irq(rq->lock)
946 : *
947 : * This function is called during schedule() when a kworker is going
948 : * to sleep. It's used by psi to identify aggregation workers during
949 : * dequeuing, to allow periodic aggregation to shut-off when that
950 : * worker is the last task in the system or cgroup to go to sleep.
951 : *
952 : * As this function doesn't involve any workqueue-related locking, it
953 : * only returns stable values when called from inside the scheduler's
954 : * queuing and dequeuing paths, when @task, which must be a kworker,
955 : * is guaranteed to not be processing any works.
956 : *
957 : * Return:
958 : * The last work function %current executed as a worker, NULL if it
959 : * hasn't executed any work yet.
960 : */
961 0 : work_func_t wq_worker_last_func(struct task_struct *task)
962 : {
963 0 : struct worker *worker = kthread_data(task);
964 :
965 0 : return worker->last_func;
966 : }
967 :
968 : /**
969 : * worker_set_flags - set worker flags and adjust nr_running accordingly
970 : * @worker: self
971 : * @flags: flags to set
972 : *
973 : * Set @flags in @worker->flags and adjust nr_running accordingly.
974 : *
975 : * CONTEXT:
976 : * raw_spin_lock_irq(pool->lock)
977 : */
978 83 : static inline void worker_set_flags(struct worker *worker, unsigned int flags)
979 : {
980 83 : struct worker_pool *pool = worker->pool;
981 :
982 166 : WARN_ON_ONCE(worker->task != current);
983 :
984 : /* If transitioning into NOT_RUNNING, adjust nr_running. */
985 166 : if ((flags & WORKER_NOT_RUNNING) &&
986 83 : !(worker->flags & WORKER_NOT_RUNNING)) {
987 50 : pool->nr_running--;
988 : }
989 :
990 83 : worker->flags |= flags;
991 83 : }
992 :
993 : /**
994 : * worker_clr_flags - clear worker flags and adjust nr_running accordingly
995 : * @worker: self
996 : * @flags: flags to clear
997 : *
998 : * Clear @flags in @worker->flags and adjust nr_running accordingly.
999 : *
1000 : * CONTEXT:
1001 : * raw_spin_lock_irq(pool->lock)
1002 : */
1003 171 : static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
1004 : {
1005 171 : struct worker_pool *pool = worker->pool;
1006 171 : unsigned int oflags = worker->flags;
1007 :
1008 342 : WARN_ON_ONCE(worker->task != current);
1009 :
1010 171 : worker->flags &= ~flags;
1011 :
1012 : /*
1013 : * If transitioning out of NOT_RUNNING, increment nr_running. Note
1014 : * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1015 : * of multiple flags, not a single flag.
1016 : */
1017 171 : if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1018 83 : if (!(worker->flags & WORKER_NOT_RUNNING))
1019 50 : pool->nr_running++;
1020 171 : }
1021 :
1022 : /**
1023 : * find_worker_executing_work - find worker which is executing a work
1024 : * @pool: pool of interest
1025 : * @work: work to find worker for
1026 : *
1027 : * Find a worker which is executing @work on @pool by searching
1028 : * @pool->busy_hash which is keyed by the address of @work. For a worker
1029 : * to match, its current execution should match the address of @work and
1030 : * its work function. This is to avoid unwanted dependency between
1031 : * unrelated work executions through a work item being recycled while still
1032 : * being executed.
1033 : *
1034 : * This is a bit tricky. A work item may be freed once its execution
1035 : * starts and nothing prevents the freed area from being recycled for
1036 : * another work item. If the same work item address ends up being reused
1037 : * before the original execution finishes, workqueue will identify the
1038 : * recycled work item as currently executing and make it wait until the
1039 : * current execution finishes, introducing an unwanted dependency.
1040 : *
1041 : * This function checks the work item address and work function to avoid
1042 : * false positives. Note that this isn't complete as one may construct a
1043 : * work function which can introduce dependency onto itself through a
1044 : * recycled work item. Well, if somebody wants to shoot oneself in the
1045 : * foot that badly, there's only so much we can do, and if such deadlock
1046 : * actually occurs, it should be easy to locate the culprit work function.
1047 : *
1048 : * CONTEXT:
1049 : * raw_spin_lock_irq(pool->lock).
1050 : *
1051 : * Return:
1052 : * Pointer to worker which is executing @work if found, %NULL
1053 : * otherwise.
1054 : */
1055 : static struct worker *find_worker_executing_work(struct worker_pool *pool,
1056 : struct work_struct *work)
1057 : {
1058 : struct worker *worker;
1059 :
1060 176 : hash_for_each_possible(pool->busy_hash, worker, hentry,
1061 : (unsigned long)work)
1062 0 : if (worker->current_work == work &&
1063 0 : worker->current_func == work->func)
1064 : return worker;
1065 :
1066 : return NULL;
1067 : }
1068 :
1069 : /**
1070 : * move_linked_works - move linked works to a list
1071 : * @work: start of series of works to be scheduled
1072 : * @head: target list to append @work to
1073 : * @nextp: out parameter for nested worklist walking
1074 : *
1075 : * Schedule linked works starting from @work to @head. Work series to
1076 : * be scheduled starts at @work and includes any consecutive work with
1077 : * WORK_STRUCT_LINKED set in its predecessor.
1078 : *
1079 : * If @nextp is not NULL, it's updated to point to the next work of
1080 : * the last scheduled work. This allows move_linked_works() to be
1081 : * nested inside outer list_for_each_entry_safe().
1082 : *
1083 : * CONTEXT:
1084 : * raw_spin_lock_irq(pool->lock).
1085 : */
1086 : static void move_linked_works(struct work_struct *work, struct list_head *head,
1087 : struct work_struct **nextp)
1088 : {
1089 : struct work_struct *n;
1090 :
1091 : /*
1092 : * Linked worklist will always end before the end of the list,
1093 : * use NULL for list head.
1094 : */
1095 4 : list_for_each_entry_safe_from(work, n, NULL, entry) {
1096 8 : list_move_tail(&work->entry, head);
1097 4 : if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1098 : break;
1099 : }
1100 :
1101 : /*
1102 : * If we're already inside safe list traversal and have moved
1103 : * multiple works to the scheduled queue, the next position
1104 : * needs to be updated.
1105 : */
1106 : if (nextp)
1107 : *nextp = n;
1108 : }
1109 :
1110 : /**
1111 : * get_pwq - get an extra reference on the specified pool_workqueue
1112 : * @pwq: pool_workqueue to get
1113 : *
1114 : * Obtain an extra reference on @pwq. The caller should guarantee that
1115 : * @pwq has positive refcnt and be holding the matching pool->lock.
1116 : */
1117 86 : static void get_pwq(struct pool_workqueue *pwq)
1118 : {
1119 : lockdep_assert_held(&pwq->pool->lock);
1120 86 : WARN_ON_ONCE(pwq->refcnt <= 0);
1121 86 : pwq->refcnt++;
1122 86 : }
1123 :
1124 : /**
1125 : * put_pwq - put a pool_workqueue reference
1126 : * @pwq: pool_workqueue to put
1127 : *
1128 : * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1129 : * destruction. The caller should be holding the matching pool->lock.
1130 : */
1131 86 : static void put_pwq(struct pool_workqueue *pwq)
1132 : {
1133 : lockdep_assert_held(&pwq->pool->lock);
1134 86 : if (likely(--pwq->refcnt))
1135 : return;
1136 0 : if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1137 : return;
1138 : /*
1139 : * @pwq can't be released under pool->lock, bounce to
1140 : * pwq_unbound_release_workfn(). This never recurses on the same
1141 : * pool->lock as this path is taken only for unbound workqueues and
1142 : * the release work item is scheduled on a per-cpu workqueue. To
1143 : * avoid lockdep warning, unbound pool->locks are given lockdep
1144 : * subclass of 1 in get_unbound_pool().
1145 : */
1146 0 : schedule_work(&pwq->unbound_release_work);
1147 : }
1148 :
1149 : /**
1150 : * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1151 : * @pwq: pool_workqueue to put (can be %NULL)
1152 : *
1153 : * put_pwq() with locking. This function also allows %NULL @pwq.
1154 : */
1155 6 : static void put_pwq_unlocked(struct pool_workqueue *pwq)
1156 : {
1157 6 : if (pwq) {
1158 : /*
1159 : * As both pwqs and pools are RCU protected, the
1160 : * following lock operations are safe.
1161 : */
1162 0 : raw_spin_lock_irq(&pwq->pool->lock);
1163 0 : put_pwq(pwq);
1164 0 : raw_spin_unlock_irq(&pwq->pool->lock);
1165 : }
1166 6 : }
1167 :
1168 0 : static void pwq_activate_inactive_work(struct work_struct *work)
1169 : {
1170 0 : struct pool_workqueue *pwq = get_work_pwq(work);
1171 :
1172 0 : trace_workqueue_activate_work(work);
1173 0 : if (list_empty(&pwq->pool->worklist))
1174 0 : pwq->pool->watchdog_ts = jiffies;
1175 0 : move_linked_works(work, &pwq->pool->worklist, NULL);
1176 0 : __clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
1177 0 : pwq->nr_active++;
1178 0 : }
1179 :
1180 : static void pwq_activate_first_inactive(struct pool_workqueue *pwq)
1181 : {
1182 0 : struct work_struct *work = list_first_entry(&pwq->inactive_works,
1183 : struct work_struct, entry);
1184 :
1185 0 : pwq_activate_inactive_work(work);
1186 : }
1187 :
1188 : /**
1189 : * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1190 : * @pwq: pwq of interest
1191 : * @work_data: work_data of work which left the queue
1192 : *
1193 : * A work either has completed or is removed from pending queue,
1194 : * decrement nr_in_flight of its pwq and handle workqueue flushing.
1195 : *
1196 : * CONTEXT:
1197 : * raw_spin_lock_irq(pool->lock).
1198 : */
1199 86 : static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
1200 : {
1201 86 : int color = get_work_color(work_data);
1202 :
1203 86 : if (!(work_data & WORK_STRUCT_INACTIVE)) {
1204 84 : pwq->nr_active--;
1205 168 : if (!list_empty(&pwq->inactive_works)) {
1206 : /* one down, submit an inactive one */
1207 0 : if (pwq->nr_active < pwq->max_active)
1208 0 : pwq_activate_first_inactive(pwq);
1209 : }
1210 : }
1211 :
1212 86 : pwq->nr_in_flight[color]--;
1213 :
1214 : /* is flush in progress and are we at the flushing tip? */
1215 86 : if (likely(pwq->flush_color != color))
1216 : goto out_put;
1217 :
1218 : /* are there still in-flight works? */
1219 0 : if (pwq->nr_in_flight[color])
1220 : goto out_put;
1221 :
1222 : /* this pwq is done, clear flush_color */
1223 0 : pwq->flush_color = -1;
1224 :
1225 : /*
1226 : * If this was the last pwq, wake up the first flusher. It
1227 : * will handle the rest.
1228 : */
1229 0 : if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1230 0 : complete(&pwq->wq->first_flusher->done);
1231 : out_put:
1232 86 : put_pwq(pwq);
1233 86 : }
1234 :
1235 : /**
1236 : * try_to_grab_pending - steal work item from worklist and disable irq
1237 : * @work: work item to steal
1238 : * @is_dwork: @work is a delayed_work
1239 : * @flags: place to store irq state
1240 : *
1241 : * Try to grab PENDING bit of @work. This function can handle @work in any
1242 : * stable state - idle, on timer or on worklist.
1243 : *
1244 : * Return:
1245 : *
1246 : * ======== ================================================================
1247 : * 1 if @work was pending and we successfully stole PENDING
1248 : * 0 if @work was idle and we claimed PENDING
1249 : * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1250 : * -ENOENT if someone else is canceling @work, this state may persist
1251 : * for arbitrarily long
1252 : * ======== ================================================================
1253 : *
1254 : * Note:
1255 : * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
1256 : * interrupted while holding PENDING and @work off queue, irq must be
1257 : * disabled on entry. This, combined with delayed_work->timer being
1258 : * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1259 : *
1260 : * On successful return, >= 0, irq is disabled and the caller is
1261 : * responsible for releasing it using local_irq_restore(*@flags).
1262 : *
1263 : * This function is safe to call from any context including IRQ handler.
1264 : */
1265 38 : static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1266 : unsigned long *flags)
1267 : {
1268 : struct worker_pool *pool;
1269 : struct pool_workqueue *pwq;
1270 :
1271 38 : local_irq_save(*flags);
1272 :
1273 : /* try to steal the timer if it exists */
1274 38 : if (is_dwork) {
1275 38 : struct delayed_work *dwork = to_delayed_work(work);
1276 :
1277 : /*
1278 : * dwork->timer is irqsafe. If del_timer() fails, it's
1279 : * guaranteed that the timer is not queued anywhere and not
1280 : * running on the local CPU.
1281 : */
1282 76 : if (likely(del_timer(&dwork->timer)))
1283 : return 1;
1284 : }
1285 :
1286 : /* try to claim PENDING the normal way */
1287 76 : if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1288 : return 0;
1289 :
1290 : rcu_read_lock();
1291 : /*
1292 : * The queueing is in progress, or it is already queued. Try to
1293 : * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1294 : */
1295 0 : pool = get_work_pool(work);
1296 0 : if (!pool)
1297 : goto fail;
1298 :
1299 0 : raw_spin_lock(&pool->lock);
1300 : /*
1301 : * work->data is guaranteed to point to pwq only while the work
1302 : * item is queued on pwq->wq, and both updating work->data to point
1303 : * to pwq on queueing and to pool on dequeueing are done under
1304 : * pwq->pool->lock. This in turn guarantees that, if work->data
1305 : * points to pwq which is associated with a locked pool, the work
1306 : * item is currently queued on that pool.
1307 : */
1308 0 : pwq = get_work_pwq(work);
1309 0 : if (pwq && pwq->pool == pool) {
1310 0 : debug_work_deactivate(work);
1311 :
1312 : /*
1313 : * A cancelable inactive work item must be in the
1314 : * pwq->inactive_works since a queued barrier can't be
1315 : * canceled (see the comments in insert_wq_barrier()).
1316 : *
1317 : * An inactive work item cannot be grabbed directly because
1318 : * it might have linked barrier work items which, if left
1319 : * on the inactive_works list, will confuse pwq->nr_active
1320 : * management later on and cause stall. Make sure the work
1321 : * item is activated before grabbing.
1322 : */
1323 0 : if (*work_data_bits(work) & WORK_STRUCT_INACTIVE)
1324 0 : pwq_activate_inactive_work(work);
1325 :
1326 0 : list_del_init(&work->entry);
1327 0 : pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
1328 :
1329 : /* work->data points to pwq iff queued, point to pool */
1330 0 : set_work_pool_and_keep_pending(work, pool->id);
1331 :
1332 0 : raw_spin_unlock(&pool->lock);
1333 : rcu_read_unlock();
1334 0 : return 1;
1335 : }
1336 0 : raw_spin_unlock(&pool->lock);
1337 : fail:
1338 : rcu_read_unlock();
1339 0 : local_irq_restore(*flags);
1340 0 : if (work_is_canceling(work))
1341 : return -ENOENT;
1342 : cpu_relax();
1343 0 : return -EAGAIN;
1344 : }
1345 :
1346 : /**
1347 : * insert_work - insert a work into a pool
1348 : * @pwq: pwq @work belongs to
1349 : * @work: work to insert
1350 : * @head: insertion point
1351 : * @extra_flags: extra WORK_STRUCT_* flags to set
1352 : *
1353 : * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
1354 : * work_struct flags.
1355 : *
1356 : * CONTEXT:
1357 : * raw_spin_lock_irq(pool->lock).
1358 : */
1359 86 : static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1360 : struct list_head *head, unsigned int extra_flags)
1361 : {
1362 86 : struct worker_pool *pool = pwq->pool;
1363 :
1364 : /* record the work call stack in order to print it in KASAN reports */
1365 86 : kasan_record_aux_stack_noalloc(work);
1366 :
1367 : /* we own @work, set data and link */
1368 172 : set_work_pwq(work, pwq, extra_flags);
1369 172 : list_add_tail(&work->entry, head);
1370 86 : get_pwq(pwq);
1371 :
1372 86 : if (__need_more_worker(pool))
1373 : wake_up_worker(pool);
1374 86 : }
1375 :
1376 : /*
1377 : * Test whether @work is being queued from another work executing on the
1378 : * same workqueue.
1379 : */
1380 : static bool is_chained_work(struct workqueue_struct *wq)
1381 : {
1382 : struct worker *worker;
1383 :
1384 0 : worker = current_wq_worker();
1385 : /*
1386 : * Return %true iff I'm a worker executing a work item on @wq. If
1387 : * I'm @worker, it's safe to dereference it without locking.
1388 : */
1389 0 : return worker && worker->current_pwq->wq == wq;
1390 : }
1391 :
1392 : /*
1393 : * When queueing an unbound work item to a wq, prefer local CPU if allowed
1394 : * by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
1395 : * avoid perturbing sensitive tasks.
1396 : */
1397 33 : static int wq_select_unbound_cpu(int cpu)
1398 : {
1399 : int new_cpu;
1400 :
1401 33 : if (likely(!wq_debug_force_rr_cpu)) {
1402 33 : if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
1403 : return cpu;
1404 : } else {
1405 0 : pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
1406 : }
1407 :
1408 0 : if (cpumask_empty(wq_unbound_cpumask))
1409 : return cpu;
1410 :
1411 0 : new_cpu = __this_cpu_read(wq_rr_cpu_last);
1412 0 : new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
1413 0 : if (unlikely(new_cpu >= nr_cpu_ids)) {
1414 0 : new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
1415 0 : if (unlikely(new_cpu >= nr_cpu_ids))
1416 : return cpu;
1417 : }
1418 0 : __this_cpu_write(wq_rr_cpu_last, new_cpu);
1419 :
1420 0 : return new_cpu;
1421 : }
1422 :
1423 84 : static void __queue_work(int cpu, struct workqueue_struct *wq,
1424 : struct work_struct *work)
1425 : {
1426 : struct pool_workqueue *pwq;
1427 : struct worker_pool *last_pool;
1428 : struct list_head *worklist;
1429 : unsigned int work_flags;
1430 84 : unsigned int req_cpu = cpu;
1431 :
1432 : /*
1433 : * While a work item is PENDING && off queue, a task trying to
1434 : * steal the PENDING will busy-loop waiting for it to either get
1435 : * queued or lose PENDING. Grabbing PENDING and queueing should
1436 : * happen with IRQ disabled.
1437 : */
1438 : lockdep_assert_irqs_disabled();
1439 :
1440 :
1441 : /*
1442 : * For a draining wq, only works from the same workqueue are
1443 : * allowed. The __WQ_DESTROYING helps to spot the issue that
1444 : * queues a new work item to a wq after destroy_workqueue(wq).
1445 : */
1446 84 : if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
1447 : WARN_ON_ONCE(!is_chained_work(wq))))
1448 : return;
1449 : rcu_read_lock();
1450 : retry:
1451 : /* pwq which will be used unless @work is executing elsewhere */
1452 84 : if (wq->flags & WQ_UNBOUND) {
1453 33 : if (req_cpu == WORK_CPU_UNBOUND)
1454 33 : cpu = wq_select_unbound_cpu(raw_smp_processor_id());
1455 33 : pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1456 : } else {
1457 51 : if (req_cpu == WORK_CPU_UNBOUND)
1458 51 : cpu = raw_smp_processor_id();
1459 51 : pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1460 : }
1461 :
1462 : /*
1463 : * If @work was previously on a different pool, it might still be
1464 : * running there, in which case the work needs to be queued on that
1465 : * pool to guarantee non-reentrancy.
1466 : */
1467 84 : last_pool = get_work_pool(work);
1468 84 : if (last_pool && last_pool != pwq->pool) {
1469 : struct worker *worker;
1470 :
1471 0 : raw_spin_lock(&last_pool->lock);
1472 :
1473 0 : worker = find_worker_executing_work(last_pool, work);
1474 :
1475 0 : if (worker && worker->current_pwq->wq == wq) {
1476 : pwq = worker->current_pwq;
1477 : } else {
1478 : /* meh... not running there, queue here */
1479 0 : raw_spin_unlock(&last_pool->lock);
1480 0 : raw_spin_lock(&pwq->pool->lock);
1481 : }
1482 : } else {
1483 84 : raw_spin_lock(&pwq->pool->lock);
1484 : }
1485 :
1486 : /*
1487 : * pwq is determined and locked. For unbound pools, we could have
1488 : * raced with pwq release and it could already be dead. If its
1489 : * refcnt is zero, repeat pwq selection. Note that pwqs never die
1490 : * without another pwq replacing it in the numa_pwq_tbl or while
1491 : * work items are executing on it, so the retrying is guaranteed to
1492 : * make forward-progress.
1493 : */
1494 84 : if (unlikely(!pwq->refcnt)) {
1495 0 : if (wq->flags & WQ_UNBOUND) {
1496 0 : raw_spin_unlock(&pwq->pool->lock);
1497 : cpu_relax();
1498 : goto retry;
1499 : }
1500 : /* oops */
1501 0 : WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1502 : wq->name, cpu);
1503 : }
1504 :
1505 : /* pwq determined, queue */
1506 84 : trace_workqueue_queue_work(req_cpu, pwq, work);
1507 :
1508 168 : if (WARN_ON(!list_empty(&work->entry)))
1509 : goto out;
1510 :
1511 84 : pwq->nr_in_flight[pwq->work_color]++;
1512 168 : work_flags = work_color_to_flags(pwq->work_color);
1513 :
1514 84 : if (likely(pwq->nr_active < pwq->max_active)) {
1515 84 : trace_workqueue_activate_work(work);
1516 84 : pwq->nr_active++;
1517 84 : worklist = &pwq->pool->worklist;
1518 84 : if (list_empty(worklist))
1519 83 : pwq->pool->watchdog_ts = jiffies;
1520 : } else {
1521 0 : work_flags |= WORK_STRUCT_INACTIVE;
1522 0 : worklist = &pwq->inactive_works;
1523 : }
1524 :
1525 84 : debug_work_activate(work);
1526 84 : insert_work(pwq, work, worklist, work_flags);
1527 :
1528 : out:
1529 84 : raw_spin_unlock(&pwq->pool->lock);
1530 : rcu_read_unlock();
1531 : }
1532 :
1533 : /**
1534 : * queue_work_on - queue work on specific cpu
1535 : * @cpu: CPU number to execute work on
1536 : * @wq: workqueue to use
1537 : * @work: work to queue
1538 : *
1539 : * We queue the work to a specific CPU, the caller must ensure it
1540 : * can't go away. Callers that fail to ensure that the specified
1541 : * CPU cannot go away will execute on a randomly chosen CPU.
1542 : *
1543 : * Return: %false if @work was already on a queue, %true otherwise.
1544 : */
1545 76 : bool queue_work_on(int cpu, struct workqueue_struct *wq,
1546 : struct work_struct *work)
1547 : {
1548 76 : bool ret = false;
1549 : unsigned long flags;
1550 :
1551 76 : local_irq_save(flags);
1552 :
1553 152 : if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1554 75 : __queue_work(cpu, wq, work);
1555 75 : ret = true;
1556 : }
1557 :
1558 152 : local_irq_restore(flags);
1559 76 : return ret;
1560 : }
1561 : EXPORT_SYMBOL(queue_work_on);
1562 :
1563 : /**
1564 : * workqueue_select_cpu_near - Select a CPU based on NUMA node
1565 : * @node: NUMA node ID that we want to select a CPU from
1566 : *
1567 : * This function will attempt to find a "random" cpu available on a given
1568 : * node. If there are no CPUs available on the given node it will return
1569 : * WORK_CPU_UNBOUND indicating that we should just schedule to any
1570 : * available CPU if we need to schedule this work.
1571 : */
1572 : static int workqueue_select_cpu_near(int node)
1573 : {
1574 : int cpu;
1575 :
1576 : /* No point in doing this if NUMA isn't enabled for workqueues */
1577 0 : if (!wq_numa_enabled)
1578 : return WORK_CPU_UNBOUND;
1579 :
1580 : /* Delay binding to CPU if node is not valid or online */
1581 0 : if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
1582 : return WORK_CPU_UNBOUND;
1583 :
1584 : /* Use local node/cpu if we are already there */
1585 : cpu = raw_smp_processor_id();
1586 : if (node == cpu_to_node(cpu))
1587 : return cpu;
1588 :
1589 : /* Use "random" otherwise know as "first" online CPU of node */
1590 : cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
1591 :
1592 : /* If CPU is valid return that, otherwise just defer */
1593 : return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
1594 : }
1595 :
1596 : /**
1597 : * queue_work_node - queue work on a "random" cpu for a given NUMA node
1598 : * @node: NUMA node that we are targeting the work for
1599 : * @wq: workqueue to use
1600 : * @work: work to queue
1601 : *
1602 : * We queue the work to a "random" CPU within a given NUMA node. The basic
1603 : * idea here is to provide a way to somehow associate work with a given
1604 : * NUMA node.
1605 : *
1606 : * This function will only make a best effort attempt at getting this onto
1607 : * the right NUMA node. If no node is requested or the requested node is
1608 : * offline then we just fall back to standard queue_work behavior.
1609 : *
1610 : * Currently the "random" CPU ends up being the first available CPU in the
1611 : * intersection of cpu_online_mask and the cpumask of the node, unless we
1612 : * are running on the node. In that case we just use the current CPU.
1613 : *
1614 : * Return: %false if @work was already on a queue, %true otherwise.
1615 : */
1616 0 : bool queue_work_node(int node, struct workqueue_struct *wq,
1617 : struct work_struct *work)
1618 : {
1619 : unsigned long flags;
1620 0 : bool ret = false;
1621 :
1622 : /*
1623 : * This current implementation is specific to unbound workqueues.
1624 : * Specifically we only return the first available CPU for a given
1625 : * node instead of cycling through individual CPUs within the node.
1626 : *
1627 : * If this is used with a per-cpu workqueue then the logic in
1628 : * workqueue_select_cpu_near would need to be updated to allow for
1629 : * some round robin type logic.
1630 : */
1631 0 : WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
1632 :
1633 0 : local_irq_save(flags);
1634 :
1635 0 : if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1636 0 : int cpu = workqueue_select_cpu_near(node);
1637 :
1638 0 : __queue_work(cpu, wq, work);
1639 0 : ret = true;
1640 : }
1641 :
1642 0 : local_irq_restore(flags);
1643 0 : return ret;
1644 : }
1645 : EXPORT_SYMBOL_GPL(queue_work_node);
1646 :
1647 9 : void delayed_work_timer_fn(struct timer_list *t)
1648 : {
1649 9 : struct delayed_work *dwork = from_timer(dwork, t, timer);
1650 :
1651 : /* should have been called from irqsafe timer with irq already off */
1652 9 : __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1653 9 : }
1654 : EXPORT_SYMBOL(delayed_work_timer_fn);
1655 :
1656 11 : static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1657 : struct delayed_work *dwork, unsigned long delay)
1658 : {
1659 11 : struct timer_list *timer = &dwork->timer;
1660 11 : struct work_struct *work = &dwork->work;
1661 :
1662 11 : WARN_ON_ONCE(!wq);
1663 11 : WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
1664 11 : WARN_ON_ONCE(timer_pending(timer));
1665 22 : WARN_ON_ONCE(!list_empty(&work->entry));
1666 :
1667 : /*
1668 : * If @delay is 0, queue @dwork->work immediately. This is for
1669 : * both optimization and correctness. The earliest @timer can
1670 : * expire is on the closest next tick and delayed_work users depend
1671 : * on that there's no such delay when @delay is 0.
1672 : */
1673 11 : if (!delay) {
1674 0 : __queue_work(cpu, wq, &dwork->work);
1675 0 : return;
1676 : }
1677 :
1678 11 : dwork->wq = wq;
1679 11 : dwork->cpu = cpu;
1680 11 : timer->expires = jiffies + delay;
1681 :
1682 11 : if (unlikely(cpu != WORK_CPU_UNBOUND))
1683 0 : add_timer_on(timer, cpu);
1684 : else
1685 11 : add_timer(timer);
1686 : }
1687 :
1688 : /**
1689 : * queue_delayed_work_on - queue work on specific CPU after delay
1690 : * @cpu: CPU number to execute work on
1691 : * @wq: workqueue to use
1692 : * @dwork: work to queue
1693 : * @delay: number of jiffies to wait before queueing
1694 : *
1695 : * Return: %false if @work was already on a queue, %true otherwise. If
1696 : * @delay is zero and @dwork is idle, it will be scheduled for immediate
1697 : * execution.
1698 : */
1699 11 : bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1700 : struct delayed_work *dwork, unsigned long delay)
1701 : {
1702 11 : struct work_struct *work = &dwork->work;
1703 11 : bool ret = false;
1704 : unsigned long flags;
1705 :
1706 : /* read the comment in __queue_work() */
1707 11 : local_irq_save(flags);
1708 :
1709 22 : if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1710 11 : __queue_delayed_work(cpu, wq, dwork, delay);
1711 11 : ret = true;
1712 : }
1713 :
1714 22 : local_irq_restore(flags);
1715 11 : return ret;
1716 : }
1717 : EXPORT_SYMBOL(queue_delayed_work_on);
1718 :
1719 : /**
1720 : * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1721 : * @cpu: CPU number to execute work on
1722 : * @wq: workqueue to use
1723 : * @dwork: work to queue
1724 : * @delay: number of jiffies to wait before queueing
1725 : *
1726 : * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1727 : * modify @dwork's timer so that it expires after @delay. If @delay is
1728 : * zero, @work is guaranteed to be scheduled immediately regardless of its
1729 : * current state.
1730 : *
1731 : * Return: %false if @dwork was idle and queued, %true if @dwork was
1732 : * pending and its timer was modified.
1733 : *
1734 : * This function is safe to call from any context including IRQ handler.
1735 : * See try_to_grab_pending() for details.
1736 : */
1737 0 : bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1738 : struct delayed_work *dwork, unsigned long delay)
1739 : {
1740 : unsigned long flags;
1741 : int ret;
1742 :
1743 : do {
1744 0 : ret = try_to_grab_pending(&dwork->work, true, &flags);
1745 0 : } while (unlikely(ret == -EAGAIN));
1746 :
1747 0 : if (likely(ret >= 0)) {
1748 0 : __queue_delayed_work(cpu, wq, dwork, delay);
1749 0 : local_irq_restore(flags);
1750 : }
1751 :
1752 : /* -ENOENT from try_to_grab_pending() becomes %true */
1753 0 : return ret;
1754 : }
1755 : EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1756 :
1757 0 : static void rcu_work_rcufn(struct rcu_head *rcu)
1758 : {
1759 0 : struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
1760 :
1761 : /* read the comment in __queue_work() */
1762 : local_irq_disable();
1763 0 : __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
1764 : local_irq_enable();
1765 0 : }
1766 :
1767 : /**
1768 : * queue_rcu_work - queue work after a RCU grace period
1769 : * @wq: workqueue to use
1770 : * @rwork: work to queue
1771 : *
1772 : * Return: %false if @rwork was already pending, %true otherwise. Note
1773 : * that a full RCU grace period is guaranteed only after a %true return.
1774 : * While @rwork is guaranteed to be executed after a %false return, the
1775 : * execution may happen before a full RCU grace period has passed.
1776 : */
1777 0 : bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
1778 : {
1779 0 : struct work_struct *work = &rwork->work;
1780 :
1781 0 : if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1782 0 : rwork->wq = wq;
1783 0 : call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
1784 0 : return true;
1785 : }
1786 :
1787 : return false;
1788 : }
1789 : EXPORT_SYMBOL(queue_rcu_work);
1790 :
1791 : /**
1792 : * worker_enter_idle - enter idle state
1793 : * @worker: worker which is entering idle state
1794 : *
1795 : * @worker is entering idle state. Update stats and idle timer if
1796 : * necessary.
1797 : *
1798 : * LOCKING:
1799 : * raw_spin_lock_irq(pool->lock).
1800 : */
1801 93 : static void worker_enter_idle(struct worker *worker)
1802 : {
1803 93 : struct worker_pool *pool = worker->pool;
1804 :
1805 186 : if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1806 186 : WARN_ON_ONCE(!list_empty(&worker->entry) &&
1807 : (worker->hentry.next || worker->hentry.pprev)))
1808 : return;
1809 :
1810 : /* can't use worker_set_flags(), also called from create_worker() */
1811 93 : worker->flags |= WORKER_IDLE;
1812 93 : pool->nr_idle++;
1813 93 : worker->last_active = jiffies;
1814 :
1815 : /* idle_list is LIFO */
1816 186 : list_add(&worker->entry, &pool->idle_list);
1817 :
1818 186 : if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1819 0 : mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1820 :
1821 : /* Sanity check nr_running. */
1822 93 : WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
1823 : }
1824 :
1825 : /**
1826 : * worker_leave_idle - leave idle state
1827 : * @worker: worker which is leaving idle state
1828 : *
1829 : * @worker is leaving idle state. Update stats.
1830 : *
1831 : * LOCKING:
1832 : * raw_spin_lock_irq(pool->lock).
1833 : */
1834 88 : static void worker_leave_idle(struct worker *worker)
1835 : {
1836 88 : struct worker_pool *pool = worker->pool;
1837 :
1838 88 : if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1839 : return;
1840 88 : worker_clr_flags(worker, WORKER_IDLE);
1841 88 : pool->nr_idle--;
1842 88 : list_del_init(&worker->entry);
1843 : }
1844 :
1845 9 : static struct worker *alloc_worker(int node)
1846 : {
1847 : struct worker *worker;
1848 :
1849 9 : worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
1850 9 : if (worker) {
1851 18 : INIT_LIST_HEAD(&worker->entry);
1852 18 : INIT_LIST_HEAD(&worker->scheduled);
1853 18 : INIT_LIST_HEAD(&worker->node);
1854 : /* on creation a worker is in !idle && prep state */
1855 9 : worker->flags = WORKER_PREP;
1856 : }
1857 9 : return worker;
1858 : }
1859 :
1860 : /**
1861 : * worker_attach_to_pool() - attach a worker to a pool
1862 : * @worker: worker to be attached
1863 : * @pool: the target pool
1864 : *
1865 : * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1866 : * cpu-binding of @worker are kept coordinated with the pool across
1867 : * cpu-[un]hotplugs.
1868 : */
1869 5 : static void worker_attach_to_pool(struct worker *worker,
1870 : struct worker_pool *pool)
1871 : {
1872 5 : mutex_lock(&wq_pool_attach_mutex);
1873 :
1874 : /*
1875 : * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains
1876 : * stable across this function. See the comments above the flag
1877 : * definition for details.
1878 : */
1879 5 : if (pool->flags & POOL_DISASSOCIATED)
1880 2 : worker->flags |= WORKER_UNBOUND;
1881 : else
1882 3 : kthread_set_per_cpu(worker->task, pool->cpu);
1883 :
1884 5 : if (worker->rescue_wq)
1885 0 : set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1886 :
1887 10 : list_add_tail(&worker->node, &pool->workers);
1888 5 : worker->pool = pool;
1889 :
1890 5 : mutex_unlock(&wq_pool_attach_mutex);
1891 5 : }
1892 :
1893 : /**
1894 : * worker_detach_from_pool() - detach a worker from its pool
1895 : * @worker: worker which is attached to its pool
1896 : *
1897 : * Undo the attaching which had been done in worker_attach_to_pool(). The
1898 : * caller worker shouldn't access to the pool after detached except it has
1899 : * other reference to the pool.
1900 : */
1901 0 : static void worker_detach_from_pool(struct worker *worker)
1902 : {
1903 0 : struct worker_pool *pool = worker->pool;
1904 0 : struct completion *detach_completion = NULL;
1905 :
1906 0 : mutex_lock(&wq_pool_attach_mutex);
1907 :
1908 0 : kthread_set_per_cpu(worker->task, -1);
1909 0 : list_del(&worker->node);
1910 0 : worker->pool = NULL;
1911 :
1912 0 : if (list_empty(&pool->workers) && list_empty(&pool->dying_workers))
1913 0 : detach_completion = pool->detach_completion;
1914 0 : mutex_unlock(&wq_pool_attach_mutex);
1915 :
1916 : /* clear leftover flags without pool->lock after it is detached */
1917 0 : worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
1918 :
1919 0 : if (detach_completion)
1920 0 : complete(detach_completion);
1921 0 : }
1922 :
1923 : /**
1924 : * create_worker - create a new workqueue worker
1925 : * @pool: pool the new worker will belong to
1926 : *
1927 : * Create and start a new worker which is attached to @pool.
1928 : *
1929 : * CONTEXT:
1930 : * Might sleep. Does GFP_KERNEL allocations.
1931 : *
1932 : * Return:
1933 : * Pointer to the newly created worker.
1934 : */
1935 5 : static struct worker *create_worker(struct worker_pool *pool)
1936 : {
1937 : struct worker *worker;
1938 : int id;
1939 : char id_buf[16];
1940 :
1941 : /* ID is needed to determine kthread name */
1942 10 : id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
1943 5 : if (id < 0) {
1944 0 : pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
1945 : ERR_PTR(id));
1946 : return NULL;
1947 : }
1948 :
1949 5 : worker = alloc_worker(pool->node);
1950 5 : if (!worker) {
1951 0 : pr_err_once("workqueue: Failed to allocate a worker\n");
1952 : goto fail;
1953 : }
1954 :
1955 5 : worker->id = id;
1956 :
1957 5 : if (pool->cpu >= 0)
1958 3 : snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1959 3 : pool->attrs->nice < 0 ? "H" : "");
1960 : else
1961 2 : snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1962 :
1963 5 : worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1964 : "kworker/%s", id_buf);
1965 10 : if (IS_ERR(worker->task)) {
1966 0 : if (PTR_ERR(worker->task) == -EINTR) {
1967 0 : pr_err("workqueue: Interrupted when creating a worker thread \"kworker/%s\"\n",
1968 : id_buf);
1969 : } else {
1970 0 : pr_err_once("workqueue: Failed to create a worker thread: %pe",
1971 : worker->task);
1972 : }
1973 : goto fail;
1974 : }
1975 :
1976 5 : set_user_nice(worker->task, pool->attrs->nice);
1977 5 : kthread_bind_mask(worker->task, pool->attrs->cpumask);
1978 :
1979 : /* successful, attach the worker to the pool */
1980 5 : worker_attach_to_pool(worker, pool);
1981 :
1982 : /* start the newly created worker */
1983 5 : raw_spin_lock_irq(&pool->lock);
1984 5 : worker->pool->nr_workers++;
1985 5 : worker_enter_idle(worker);
1986 5 : wake_up_process(worker->task);
1987 5 : raw_spin_unlock_irq(&pool->lock);
1988 :
1989 5 : return worker;
1990 :
1991 : fail:
1992 0 : ida_free(&pool->worker_ida, id);
1993 0 : kfree(worker);
1994 0 : return NULL;
1995 : }
1996 :
1997 0 : static void unbind_worker(struct worker *worker)
1998 : {
1999 : lockdep_assert_held(&wq_pool_attach_mutex);
2000 :
2001 0 : kthread_set_per_cpu(worker->task, -1);
2002 0 : if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
2003 0 : WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
2004 : else
2005 0 : WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
2006 0 : }
2007 :
2008 0 : static void wake_dying_workers(struct list_head *cull_list)
2009 : {
2010 : struct worker *worker, *tmp;
2011 :
2012 0 : list_for_each_entry_safe(worker, tmp, cull_list, entry) {
2013 0 : list_del_init(&worker->entry);
2014 0 : unbind_worker(worker);
2015 : /*
2016 : * If the worker was somehow already running, then it had to be
2017 : * in pool->idle_list when set_worker_dying() happened or we
2018 : * wouldn't have gotten here.
2019 : *
2020 : * Thus, the worker must either have observed the WORKER_DIE
2021 : * flag, or have set its state to TASK_IDLE. Either way, the
2022 : * below will be observed by the worker and is safe to do
2023 : * outside of pool->lock.
2024 : */
2025 0 : wake_up_process(worker->task);
2026 : }
2027 0 : }
2028 :
2029 : /**
2030 : * set_worker_dying - Tag a worker for destruction
2031 : * @worker: worker to be destroyed
2032 : * @list: transfer worker away from its pool->idle_list and into list
2033 : *
2034 : * Tag @worker for destruction and adjust @pool stats accordingly. The worker
2035 : * should be idle.
2036 : *
2037 : * CONTEXT:
2038 : * raw_spin_lock_irq(pool->lock).
2039 : */
2040 0 : static void set_worker_dying(struct worker *worker, struct list_head *list)
2041 : {
2042 0 : struct worker_pool *pool = worker->pool;
2043 :
2044 : lockdep_assert_held(&pool->lock);
2045 : lockdep_assert_held(&wq_pool_attach_mutex);
2046 :
2047 : /* sanity check frenzy */
2048 0 : if (WARN_ON(worker->current_work) ||
2049 0 : WARN_ON(!list_empty(&worker->scheduled)) ||
2050 0 : WARN_ON(!(worker->flags & WORKER_IDLE)))
2051 : return;
2052 :
2053 0 : pool->nr_workers--;
2054 0 : pool->nr_idle--;
2055 :
2056 0 : worker->flags |= WORKER_DIE;
2057 :
2058 0 : list_move(&worker->entry, list);
2059 0 : list_move(&worker->node, &pool->dying_workers);
2060 : }
2061 :
2062 : /**
2063 : * idle_worker_timeout - check if some idle workers can now be deleted.
2064 : * @t: The pool's idle_timer that just expired
2065 : *
2066 : * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
2067 : * worker_leave_idle(), as a worker flicking between idle and active while its
2068 : * pool is at the too_many_workers() tipping point would cause too much timer
2069 : * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
2070 : * it expire and re-evaluate things from there.
2071 : */
2072 0 : static void idle_worker_timeout(struct timer_list *t)
2073 : {
2074 0 : struct worker_pool *pool = from_timer(pool, t, idle_timer);
2075 0 : bool do_cull = false;
2076 :
2077 0 : if (work_pending(&pool->idle_cull_work))
2078 : return;
2079 :
2080 0 : raw_spin_lock_irq(&pool->lock);
2081 :
2082 0 : if (too_many_workers(pool)) {
2083 : struct worker *worker;
2084 : unsigned long expires;
2085 :
2086 : /* idle_list is kept in LIFO order, check the last one */
2087 0 : worker = list_entry(pool->idle_list.prev, struct worker, entry);
2088 0 : expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2089 0 : do_cull = !time_before(jiffies, expires);
2090 :
2091 0 : if (!do_cull)
2092 0 : mod_timer(&pool->idle_timer, expires);
2093 : }
2094 0 : raw_spin_unlock_irq(&pool->lock);
2095 :
2096 0 : if (do_cull)
2097 0 : queue_work(system_unbound_wq, &pool->idle_cull_work);
2098 : }
2099 :
2100 : /**
2101 : * idle_cull_fn - cull workers that have been idle for too long.
2102 : * @work: the pool's work for handling these idle workers
2103 : *
2104 : * This goes through a pool's idle workers and gets rid of those that have been
2105 : * idle for at least IDLE_WORKER_TIMEOUT seconds.
2106 : *
2107 : * We don't want to disturb isolated CPUs because of a pcpu kworker being
2108 : * culled, so this also resets worker affinity. This requires a sleepable
2109 : * context, hence the split between timer callback and work item.
2110 : */
2111 0 : static void idle_cull_fn(struct work_struct *work)
2112 : {
2113 0 : struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
2114 : struct list_head cull_list;
2115 :
2116 0 : INIT_LIST_HEAD(&cull_list);
2117 : /*
2118 : * Grabbing wq_pool_attach_mutex here ensures an already-running worker
2119 : * cannot proceed beyong worker_detach_from_pool() in its self-destruct
2120 : * path. This is required as a previously-preempted worker could run after
2121 : * set_worker_dying() has happened but before wake_dying_workers() did.
2122 : */
2123 0 : mutex_lock(&wq_pool_attach_mutex);
2124 0 : raw_spin_lock_irq(&pool->lock);
2125 :
2126 0 : while (too_many_workers(pool)) {
2127 : struct worker *worker;
2128 : unsigned long expires;
2129 :
2130 0 : worker = list_entry(pool->idle_list.prev, struct worker, entry);
2131 0 : expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2132 :
2133 0 : if (time_before(jiffies, expires)) {
2134 0 : mod_timer(&pool->idle_timer, expires);
2135 0 : break;
2136 : }
2137 :
2138 0 : set_worker_dying(worker, &cull_list);
2139 : }
2140 :
2141 0 : raw_spin_unlock_irq(&pool->lock);
2142 0 : wake_dying_workers(&cull_list);
2143 0 : mutex_unlock(&wq_pool_attach_mutex);
2144 0 : }
2145 :
2146 0 : static void send_mayday(struct work_struct *work)
2147 : {
2148 0 : struct pool_workqueue *pwq = get_work_pwq(work);
2149 0 : struct workqueue_struct *wq = pwq->wq;
2150 :
2151 : lockdep_assert_held(&wq_mayday_lock);
2152 :
2153 0 : if (!wq->rescuer)
2154 : return;
2155 :
2156 : /* mayday mayday mayday */
2157 0 : if (list_empty(&pwq->mayday_node)) {
2158 : /*
2159 : * If @pwq is for an unbound wq, its base ref may be put at
2160 : * any time due to an attribute change. Pin @pwq until the
2161 : * rescuer is done with it.
2162 : */
2163 0 : get_pwq(pwq);
2164 0 : list_add_tail(&pwq->mayday_node, &wq->maydays);
2165 0 : wake_up_process(wq->rescuer->task);
2166 : }
2167 : }
2168 :
2169 0 : static void pool_mayday_timeout(struct timer_list *t)
2170 : {
2171 0 : struct worker_pool *pool = from_timer(pool, t, mayday_timer);
2172 : struct work_struct *work;
2173 :
2174 0 : raw_spin_lock_irq(&pool->lock);
2175 0 : raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
2176 :
2177 0 : if (need_to_create_worker(pool)) {
2178 : /*
2179 : * We've been trying to create a new worker but
2180 : * haven't been successful. We might be hitting an
2181 : * allocation deadlock. Send distress signals to
2182 : * rescuers.
2183 : */
2184 0 : list_for_each_entry(work, &pool->worklist, entry)
2185 0 : send_mayday(work);
2186 : }
2187 :
2188 0 : raw_spin_unlock(&wq_mayday_lock);
2189 0 : raw_spin_unlock_irq(&pool->lock);
2190 :
2191 0 : mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
2192 0 : }
2193 :
2194 : /**
2195 : * maybe_create_worker - create a new worker if necessary
2196 : * @pool: pool to create a new worker for
2197 : *
2198 : * Create a new worker for @pool if necessary. @pool is guaranteed to
2199 : * have at least one idle worker on return from this function. If
2200 : * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
2201 : * sent to all rescuers with works scheduled on @pool to resolve
2202 : * possible allocation deadlock.
2203 : *
2204 : * On return, need_to_create_worker() is guaranteed to be %false and
2205 : * may_start_working() %true.
2206 : *
2207 : * LOCKING:
2208 : * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2209 : * multiple times. Does GFP_KERNEL allocations. Called only from
2210 : * manager.
2211 : */
2212 2 : static void maybe_create_worker(struct worker_pool *pool)
2213 : __releases(&pool->lock)
2214 : __acquires(&pool->lock)
2215 : {
2216 : restart:
2217 2 : raw_spin_unlock_irq(&pool->lock);
2218 :
2219 : /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
2220 2 : mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
2221 :
2222 : while (true) {
2223 2 : if (create_worker(pool) || !need_to_create_worker(pool))
2224 : break;
2225 :
2226 0 : schedule_timeout_interruptible(CREATE_COOLDOWN);
2227 :
2228 0 : if (!need_to_create_worker(pool))
2229 : break;
2230 : }
2231 :
2232 4 : del_timer_sync(&pool->mayday_timer);
2233 2 : raw_spin_lock_irq(&pool->lock);
2234 : /*
2235 : * This is necessary even after a new worker was just successfully
2236 : * created as @pool->lock was dropped and the new worker might have
2237 : * already become busy.
2238 : */
2239 2 : if (need_to_create_worker(pool))
2240 : goto restart;
2241 2 : }
2242 :
2243 : /**
2244 : * manage_workers - manage worker pool
2245 : * @worker: self
2246 : *
2247 : * Assume the manager role and manage the worker pool @worker belongs
2248 : * to. At any given time, there can be only zero or one manager per
2249 : * pool. The exclusion is handled automatically by this function.
2250 : *
2251 : * The caller can safely start processing works on false return. On
2252 : * true return, it's guaranteed that need_to_create_worker() is false
2253 : * and may_start_working() is true.
2254 : *
2255 : * CONTEXT:
2256 : * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2257 : * multiple times. Does GFP_KERNEL allocations.
2258 : *
2259 : * Return:
2260 : * %false if the pool doesn't need management and the caller can safely
2261 : * start processing works, %true if management function was performed and
2262 : * the conditions that the caller verified before calling the function may
2263 : * no longer be true.
2264 : */
2265 2 : static bool manage_workers(struct worker *worker)
2266 : {
2267 2 : struct worker_pool *pool = worker->pool;
2268 :
2269 2 : if (pool->flags & POOL_MANAGER_ACTIVE)
2270 : return false;
2271 :
2272 2 : pool->flags |= POOL_MANAGER_ACTIVE;
2273 2 : pool->manager = worker;
2274 :
2275 2 : maybe_create_worker(pool);
2276 :
2277 2 : pool->manager = NULL;
2278 2 : pool->flags &= ~POOL_MANAGER_ACTIVE;
2279 2 : rcuwait_wake_up(&manager_wait);
2280 2 : return true;
2281 : }
2282 :
2283 : /**
2284 : * process_one_work - process single work
2285 : * @worker: self
2286 : * @work: work to process
2287 : *
2288 : * Process @work. This function contains all the logics necessary to
2289 : * process a single work including synchronization against and
2290 : * interaction with other workers on the same cpu, queueing and
2291 : * flushing. As long as context requirement is met, any worker can
2292 : * call this function to process a work.
2293 : *
2294 : * CONTEXT:
2295 : * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2296 : */
2297 86 : static void process_one_work(struct worker *worker, struct work_struct *work)
2298 : __releases(&pool->lock)
2299 : __acquires(&pool->lock)
2300 : {
2301 86 : struct pool_workqueue *pwq = get_work_pwq(work);
2302 86 : struct worker_pool *pool = worker->pool;
2303 86 : bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2304 : unsigned long work_data;
2305 : struct worker *collision;
2306 : #ifdef CONFIG_LOCKDEP
2307 : /*
2308 : * It is permissible to free the struct work_struct from
2309 : * inside the function that is called from it, this we need to
2310 : * take into account for lockdep too. To avoid bogus "held
2311 : * lock freed" warnings as well as problems when looking into
2312 : * work->lockdep_map, make a copy and use that here.
2313 : */
2314 : struct lockdep_map lockdep_map;
2315 :
2316 : lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2317 : #endif
2318 : /* ensure we're on the correct CPU */
2319 86 : WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
2320 : raw_smp_processor_id() != pool->cpu);
2321 :
2322 : /*
2323 : * A single work shouldn't be executed concurrently by
2324 : * multiple workers on a single cpu. Check whether anyone is
2325 : * already processing the work. If so, defer the work to the
2326 : * currently executing one.
2327 : */
2328 86 : collision = find_worker_executing_work(pool, work);
2329 86 : if (unlikely(collision)) {
2330 0 : move_linked_works(work, &collision->scheduled, NULL);
2331 : return;
2332 : }
2333 :
2334 : /* claim and dequeue */
2335 86 : debug_work_deactivate(work);
2336 172 : hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2337 86 : worker->current_work = work;
2338 86 : worker->current_func = work->func;
2339 86 : worker->current_pwq = pwq;
2340 86 : work_data = *work_data_bits(work);
2341 86 : worker->current_color = get_work_color(work_data);
2342 :
2343 : /*
2344 : * Record wq name for cmdline and debug reporting, may get
2345 : * overridden through set_worker_desc().
2346 : */
2347 86 : strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
2348 :
2349 172 : list_del_init(&work->entry);
2350 :
2351 : /*
2352 : * CPU intensive works don't participate in concurrency management.
2353 : * They're the scheduler's responsibility. This takes @worker out
2354 : * of concurrency management and the next code block will chain
2355 : * execution of the pending work items.
2356 : */
2357 86 : if (unlikely(cpu_intensive))
2358 0 : worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2359 :
2360 : /*
2361 : * Wake up another worker if necessary. The condition is always
2362 : * false for normal per-cpu workers since nr_running would always
2363 : * be >= 1 at this point. This is used to chain execution of the
2364 : * pending work items for WORKER_NOT_RUNNING workers such as the
2365 : * UNBOUND and CPU_INTENSIVE ones.
2366 : */
2367 86 : if (need_more_worker(pool))
2368 : wake_up_worker(pool);
2369 :
2370 : /*
2371 : * Record the last pool and clear PENDING which should be the last
2372 : * update to @work. Also, do this inside @pool->lock so that
2373 : * PENDING and queued state changes happen together while IRQ is
2374 : * disabled.
2375 : */
2376 172 : set_work_pool_and_clear_pending(work, pool->id);
2377 :
2378 86 : raw_spin_unlock_irq(&pool->lock);
2379 :
2380 : lock_map_acquire(&pwq->wq->lockdep_map);
2381 : lock_map_acquire(&lockdep_map);
2382 : /*
2383 : * Strictly speaking we should mark the invariant state without holding
2384 : * any locks, that is, before these two lock_map_acquire()'s.
2385 : *
2386 : * However, that would result in:
2387 : *
2388 : * A(W1)
2389 : * WFC(C)
2390 : * A(W1)
2391 : * C(C)
2392 : *
2393 : * Which would create W1->C->W1 dependencies, even though there is no
2394 : * actual deadlock possible. There are two solutions, using a
2395 : * read-recursive acquire on the work(queue) 'locks', but this will then
2396 : * hit the lockdep limitation on recursive locks, or simply discard
2397 : * these locks.
2398 : *
2399 : * AFAICT there is no possible deadlock scenario between the
2400 : * flush_work() and complete() primitives (except for single-threaded
2401 : * workqueues), so hiding them isn't a problem.
2402 : */
2403 86 : lockdep_invariant_state(true);
2404 86 : trace_workqueue_execute_start(work);
2405 86 : worker->current_func(work);
2406 : /*
2407 : * While we must be careful to not use "work" after this, the trace
2408 : * point will only record its address.
2409 : */
2410 86 : trace_workqueue_execute_end(work, worker->current_func);
2411 : lock_map_release(&lockdep_map);
2412 : lock_map_release(&pwq->wq->lockdep_map);
2413 :
2414 86 : if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
2415 0 : pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2416 : " last function: %ps\n",
2417 : current->comm, preempt_count(), task_pid_nr(current),
2418 : worker->current_func);
2419 0 : debug_show_held_locks(current);
2420 0 : dump_stack();
2421 : }
2422 :
2423 : /*
2424 : * The following prevents a kworker from hogging CPU on !PREEMPTION
2425 : * kernels, where a requeueing work item waiting for something to
2426 : * happen could deadlock with stop_machine as such work item could
2427 : * indefinitely requeue itself while all other CPUs are trapped in
2428 : * stop_machine. At the same time, report a quiescent RCU state so
2429 : * the same condition doesn't freeze RCU.
2430 : */
2431 86 : cond_resched();
2432 :
2433 86 : raw_spin_lock_irq(&pool->lock);
2434 :
2435 : /* clear cpu intensive status */
2436 86 : if (unlikely(cpu_intensive))
2437 0 : worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2438 :
2439 : /* tag the worker for identification in schedule() */
2440 86 : worker->last_func = worker->current_func;
2441 :
2442 : /* we're done with it, release */
2443 172 : hash_del(&worker->hentry);
2444 86 : worker->current_work = NULL;
2445 86 : worker->current_func = NULL;
2446 86 : worker->current_pwq = NULL;
2447 86 : worker->current_color = INT_MAX;
2448 86 : pwq_dec_nr_in_flight(pwq, work_data);
2449 : }
2450 :
2451 : /**
2452 : * process_scheduled_works - process scheduled works
2453 : * @worker: self
2454 : *
2455 : * Process all scheduled works. Please note that the scheduled list
2456 : * may change while processing a work, so this function repeatedly
2457 : * fetches a work from the top and executes it.
2458 : *
2459 : * CONTEXT:
2460 : * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2461 : * multiple times.
2462 : */
2463 : static void process_scheduled_works(struct worker *worker)
2464 : {
2465 12 : while (!list_empty(&worker->scheduled)) {
2466 4 : struct work_struct *work = list_first_entry(&worker->scheduled,
2467 : struct work_struct, entry);
2468 4 : process_one_work(worker, work);
2469 : }
2470 : }
2471 :
2472 9 : static void set_pf_worker(bool val)
2473 : {
2474 9 : mutex_lock(&wq_pool_attach_mutex);
2475 9 : if (val)
2476 9 : current->flags |= PF_WQ_WORKER;
2477 : else
2478 0 : current->flags &= ~PF_WQ_WORKER;
2479 9 : mutex_unlock(&wq_pool_attach_mutex);
2480 9 : }
2481 :
2482 : /**
2483 : * worker_thread - the worker thread function
2484 : * @__worker: self
2485 : *
2486 : * The worker thread function. All workers belong to a worker_pool -
2487 : * either a per-cpu one or dynamic unbound one. These workers process all
2488 : * work items regardless of their specific target workqueue. The only
2489 : * exception is work items which belong to workqueues with a rescuer which
2490 : * will be explained in rescuer_thread().
2491 : *
2492 : * Return: 0
2493 : */
2494 5 : static int worker_thread(void *__worker)
2495 : {
2496 5 : struct worker *worker = __worker;
2497 5 : struct worker_pool *pool = worker->pool;
2498 :
2499 : /* tell the scheduler that this is a workqueue worker */
2500 5 : set_pf_worker(true);
2501 : woke_up:
2502 88 : raw_spin_lock_irq(&pool->lock);
2503 :
2504 : /* am I supposed to die? */
2505 88 : if (unlikely(worker->flags & WORKER_DIE)) {
2506 0 : raw_spin_unlock_irq(&pool->lock);
2507 0 : set_pf_worker(false);
2508 :
2509 0 : set_task_comm(worker->task, "kworker/dying");
2510 0 : ida_free(&pool->worker_ida, worker->id);
2511 0 : worker_detach_from_pool(worker);
2512 0 : WARN_ON_ONCE(!list_empty(&worker->entry));
2513 0 : kfree(worker);
2514 0 : return 0;
2515 : }
2516 :
2517 88 : worker_leave_idle(worker);
2518 : recheck:
2519 : /* no more worker necessary? */
2520 90 : if (!need_more_worker(pool))
2521 : goto sleep;
2522 :
2523 : /* do we need to manage? */
2524 85 : if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2525 : goto recheck;
2526 :
2527 : /*
2528 : * ->scheduled list can only be filled while a worker is
2529 : * preparing to process a work or actually processing it.
2530 : * Make sure nobody diddled with it while I was sleeping.
2531 : */
2532 166 : WARN_ON_ONCE(!list_empty(&worker->scheduled));
2533 :
2534 : /*
2535 : * Finish PREP stage. We're guaranteed to have at least one idle
2536 : * worker or that someone else has already assumed the manager
2537 : * role. This is where @worker starts participating in concurrency
2538 : * management if applicable and concurrency management is restored
2539 : * after being rebound. See rebind_workers() for details.
2540 : */
2541 83 : worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2542 :
2543 : do {
2544 84 : struct work_struct *work =
2545 84 : list_first_entry(&pool->worklist,
2546 : struct work_struct, entry);
2547 :
2548 84 : pool->watchdog_ts = jiffies;
2549 :
2550 84 : if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2551 : /* optimization path, not strictly necessary */
2552 82 : process_one_work(worker, work);
2553 164 : if (unlikely(!list_empty(&worker->scheduled)))
2554 : process_scheduled_works(worker);
2555 : } else {
2556 2 : move_linked_works(work, &worker->scheduled, NULL);
2557 : process_scheduled_works(worker);
2558 : }
2559 84 : } while (keep_working(pool));
2560 :
2561 83 : worker_set_flags(worker, WORKER_PREP);
2562 : sleep:
2563 : /*
2564 : * pool->lock is held and there's no work to process and no need to
2565 : * manage, sleep. Workers are woken up only while holding
2566 : * pool->lock or from local cpu, so setting the current state
2567 : * before releasing pool->lock is enough to prevent losing any
2568 : * event.
2569 : */
2570 88 : worker_enter_idle(worker);
2571 88 : __set_current_state(TASK_IDLE);
2572 88 : raw_spin_unlock_irq(&pool->lock);
2573 88 : schedule();
2574 83 : goto woke_up;
2575 : }
2576 :
2577 : /**
2578 : * rescuer_thread - the rescuer thread function
2579 : * @__rescuer: self
2580 : *
2581 : * Workqueue rescuer thread function. There's one rescuer for each
2582 : * workqueue which has WQ_MEM_RECLAIM set.
2583 : *
2584 : * Regular work processing on a pool may block trying to create a new
2585 : * worker which uses GFP_KERNEL allocation which has slight chance of
2586 : * developing into deadlock if some works currently on the same queue
2587 : * need to be processed to satisfy the GFP_KERNEL allocation. This is
2588 : * the problem rescuer solves.
2589 : *
2590 : * When such condition is possible, the pool summons rescuers of all
2591 : * workqueues which have works queued on the pool and let them process
2592 : * those works so that forward progress can be guaranteed.
2593 : *
2594 : * This should happen rarely.
2595 : *
2596 : * Return: 0
2597 : */
2598 4 : static int rescuer_thread(void *__rescuer)
2599 : {
2600 4 : struct worker *rescuer = __rescuer;
2601 4 : struct workqueue_struct *wq = rescuer->rescue_wq;
2602 4 : struct list_head *scheduled = &rescuer->scheduled;
2603 : bool should_stop;
2604 :
2605 4 : set_user_nice(current, RESCUER_NICE_LEVEL);
2606 :
2607 : /*
2608 : * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2609 : * doesn't participate in concurrency management.
2610 : */
2611 4 : set_pf_worker(true);
2612 : repeat:
2613 4 : set_current_state(TASK_IDLE);
2614 :
2615 : /*
2616 : * By the time the rescuer is requested to stop, the workqueue
2617 : * shouldn't have any work pending, but @wq->maydays may still have
2618 : * pwq(s) queued. This can happen by non-rescuer workers consuming
2619 : * all the work items before the rescuer got to them. Go through
2620 : * @wq->maydays processing before acting on should_stop so that the
2621 : * list is always empty on exit.
2622 : */
2623 4 : should_stop = kthread_should_stop();
2624 :
2625 : /* see whether any pwq is asking for help */
2626 4 : raw_spin_lock_irq(&wq_mayday_lock);
2627 :
2628 12 : while (!list_empty(&wq->maydays)) {
2629 0 : struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2630 : struct pool_workqueue, mayday_node);
2631 0 : struct worker_pool *pool = pwq->pool;
2632 : struct work_struct *work, *n;
2633 0 : bool first = true;
2634 :
2635 0 : __set_current_state(TASK_RUNNING);
2636 0 : list_del_init(&pwq->mayday_node);
2637 :
2638 0 : raw_spin_unlock_irq(&wq_mayday_lock);
2639 :
2640 0 : worker_attach_to_pool(rescuer, pool);
2641 :
2642 0 : raw_spin_lock_irq(&pool->lock);
2643 :
2644 : /*
2645 : * Slurp in all works issued via this workqueue and
2646 : * process'em.
2647 : */
2648 0 : WARN_ON_ONCE(!list_empty(scheduled));
2649 0 : list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2650 0 : if (get_work_pwq(work) == pwq) {
2651 0 : if (first)
2652 0 : pool->watchdog_ts = jiffies;
2653 : move_linked_works(work, scheduled, &n);
2654 : }
2655 0 : first = false;
2656 : }
2657 :
2658 0 : if (!list_empty(scheduled)) {
2659 0 : process_scheduled_works(rescuer);
2660 :
2661 : /*
2662 : * The above execution of rescued work items could
2663 : * have created more to rescue through
2664 : * pwq_activate_first_inactive() or chained
2665 : * queueing. Let's put @pwq back on mayday list so
2666 : * that such back-to-back work items, which may be
2667 : * being used to relieve memory pressure, don't
2668 : * incur MAYDAY_INTERVAL delay inbetween.
2669 : */
2670 0 : if (pwq->nr_active && need_to_create_worker(pool)) {
2671 0 : raw_spin_lock(&wq_mayday_lock);
2672 : /*
2673 : * Queue iff we aren't racing destruction
2674 : * and somebody else hasn't queued it already.
2675 : */
2676 0 : if (wq->rescuer && list_empty(&pwq->mayday_node)) {
2677 0 : get_pwq(pwq);
2678 0 : list_add_tail(&pwq->mayday_node, &wq->maydays);
2679 : }
2680 0 : raw_spin_unlock(&wq_mayday_lock);
2681 : }
2682 : }
2683 :
2684 : /*
2685 : * Put the reference grabbed by send_mayday(). @pool won't
2686 : * go away while we're still attached to it.
2687 : */
2688 0 : put_pwq(pwq);
2689 :
2690 : /*
2691 : * Leave this pool. If need_more_worker() is %true, notify a
2692 : * regular worker; otherwise, we end up with 0 concurrency
2693 : * and stalling the execution.
2694 : */
2695 0 : if (need_more_worker(pool))
2696 : wake_up_worker(pool);
2697 :
2698 0 : raw_spin_unlock_irq(&pool->lock);
2699 :
2700 0 : worker_detach_from_pool(rescuer);
2701 :
2702 0 : raw_spin_lock_irq(&wq_mayday_lock);
2703 : }
2704 :
2705 4 : raw_spin_unlock_irq(&wq_mayday_lock);
2706 :
2707 4 : if (should_stop) {
2708 0 : __set_current_state(TASK_RUNNING);
2709 0 : set_pf_worker(false);
2710 0 : return 0;
2711 : }
2712 :
2713 : /* rescuers should never participate in concurrency management */
2714 4 : WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2715 4 : schedule();
2716 0 : goto repeat;
2717 : }
2718 :
2719 : /**
2720 : * check_flush_dependency - check for flush dependency sanity
2721 : * @target_wq: workqueue being flushed
2722 : * @target_work: work item being flushed (NULL for workqueue flushes)
2723 : *
2724 : * %current is trying to flush the whole @target_wq or @target_work on it.
2725 : * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2726 : * reclaiming memory or running on a workqueue which doesn't have
2727 : * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2728 : * a deadlock.
2729 : */
2730 2 : static void check_flush_dependency(struct workqueue_struct *target_wq,
2731 : struct work_struct *target_work)
2732 : {
2733 2 : work_func_t target_func = target_work ? target_work->func : NULL;
2734 : struct worker *worker;
2735 :
2736 2 : if (target_wq->flags & WQ_MEM_RECLAIM)
2737 : return;
2738 :
2739 2 : worker = current_wq_worker();
2740 :
2741 2 : WARN_ONCE(current->flags & PF_MEMALLOC,
2742 : "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
2743 : current->pid, current->comm, target_wq->name, target_func);
2744 2 : WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2745 : (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2746 : "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
2747 : worker->current_pwq->wq->name, worker->current_func,
2748 : target_wq->name, target_func);
2749 : }
2750 :
2751 : struct wq_barrier {
2752 : struct work_struct work;
2753 : struct completion done;
2754 : struct task_struct *task; /* purely informational */
2755 : };
2756 :
2757 2 : static void wq_barrier_func(struct work_struct *work)
2758 : {
2759 2 : struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2760 2 : complete(&barr->done);
2761 2 : }
2762 :
2763 : /**
2764 : * insert_wq_barrier - insert a barrier work
2765 : * @pwq: pwq to insert barrier into
2766 : * @barr: wq_barrier to insert
2767 : * @target: target work to attach @barr to
2768 : * @worker: worker currently executing @target, NULL if @target is not executing
2769 : *
2770 : * @barr is linked to @target such that @barr is completed only after
2771 : * @target finishes execution. Please note that the ordering
2772 : * guarantee is observed only with respect to @target and on the local
2773 : * cpu.
2774 : *
2775 : * Currently, a queued barrier can't be canceled. This is because
2776 : * try_to_grab_pending() can't determine whether the work to be
2777 : * grabbed is at the head of the queue and thus can't clear LINKED
2778 : * flag of the previous work while there must be a valid next work
2779 : * after a work with LINKED flag set.
2780 : *
2781 : * Note that when @worker is non-NULL, @target may be modified
2782 : * underneath us, so we can't reliably determine pwq from @target.
2783 : *
2784 : * CONTEXT:
2785 : * raw_spin_lock_irq(pool->lock).
2786 : */
2787 2 : static void insert_wq_barrier(struct pool_workqueue *pwq,
2788 : struct wq_barrier *barr,
2789 : struct work_struct *target, struct worker *worker)
2790 : {
2791 2 : unsigned int work_flags = 0;
2792 : unsigned int work_color;
2793 : struct list_head *head;
2794 :
2795 : /*
2796 : * debugobject calls are safe here even with pool->lock locked
2797 : * as we know for sure that this will not trigger any of the
2798 : * checks and call back into the fixup functions where we
2799 : * might deadlock.
2800 : */
2801 4 : INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2802 2 : __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2803 :
2804 4 : init_completion_map(&barr->done, &target->lockdep_map);
2805 :
2806 2 : barr->task = current;
2807 :
2808 : /* The barrier work item does not participate in pwq->nr_active. */
2809 2 : work_flags |= WORK_STRUCT_INACTIVE;
2810 :
2811 : /*
2812 : * If @target is currently being executed, schedule the
2813 : * barrier to the worker; otherwise, put it after @target.
2814 : */
2815 2 : if (worker) {
2816 0 : head = worker->scheduled.next;
2817 0 : work_color = worker->current_color;
2818 : } else {
2819 2 : unsigned long *bits = work_data_bits(target);
2820 :
2821 2 : head = target->entry.next;
2822 : /* there can already be other linked works, inherit and set */
2823 2 : work_flags |= *bits & WORK_STRUCT_LINKED;
2824 4 : work_color = get_work_color(*bits);
2825 2 : __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2826 : }
2827 :
2828 2 : pwq->nr_in_flight[work_color]++;
2829 4 : work_flags |= work_color_to_flags(work_color);
2830 :
2831 2 : debug_work_activate(&barr->work);
2832 2 : insert_work(pwq, &barr->work, head, work_flags);
2833 2 : }
2834 :
2835 : /**
2836 : * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2837 : * @wq: workqueue being flushed
2838 : * @flush_color: new flush color, < 0 for no-op
2839 : * @work_color: new work color, < 0 for no-op
2840 : *
2841 : * Prepare pwqs for workqueue flushing.
2842 : *
2843 : * If @flush_color is non-negative, flush_color on all pwqs should be
2844 : * -1. If no pwq has in-flight commands at the specified color, all
2845 : * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2846 : * has in flight commands, its pwq->flush_color is set to
2847 : * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2848 : * wakeup logic is armed and %true is returned.
2849 : *
2850 : * The caller should have initialized @wq->first_flusher prior to
2851 : * calling this function with non-negative @flush_color. If
2852 : * @flush_color is negative, no flush color update is done and %false
2853 : * is returned.
2854 : *
2855 : * If @work_color is non-negative, all pwqs should have the same
2856 : * work_color which is previous to @work_color and all will be
2857 : * advanced to @work_color.
2858 : *
2859 : * CONTEXT:
2860 : * mutex_lock(wq->mutex).
2861 : *
2862 : * Return:
2863 : * %true if @flush_color >= 0 and there's something to flush. %false
2864 : * otherwise.
2865 : */
2866 0 : static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2867 : int flush_color, int work_color)
2868 : {
2869 0 : bool wait = false;
2870 : struct pool_workqueue *pwq;
2871 :
2872 0 : if (flush_color >= 0) {
2873 0 : WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2874 0 : atomic_set(&wq->nr_pwqs_to_flush, 1);
2875 : }
2876 :
2877 0 : for_each_pwq(pwq, wq) {
2878 0 : struct worker_pool *pool = pwq->pool;
2879 :
2880 0 : raw_spin_lock_irq(&pool->lock);
2881 :
2882 0 : if (flush_color >= 0) {
2883 0 : WARN_ON_ONCE(pwq->flush_color != -1);
2884 :
2885 0 : if (pwq->nr_in_flight[flush_color]) {
2886 0 : pwq->flush_color = flush_color;
2887 0 : atomic_inc(&wq->nr_pwqs_to_flush);
2888 0 : wait = true;
2889 : }
2890 : }
2891 :
2892 0 : if (work_color >= 0) {
2893 0 : WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2894 0 : pwq->work_color = work_color;
2895 : }
2896 :
2897 0 : raw_spin_unlock_irq(&pool->lock);
2898 : }
2899 :
2900 0 : if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2901 0 : complete(&wq->first_flusher->done);
2902 :
2903 0 : return wait;
2904 : }
2905 :
2906 : /**
2907 : * __flush_workqueue - ensure that any scheduled work has run to completion.
2908 : * @wq: workqueue to flush
2909 : *
2910 : * This function sleeps until all work items which were queued on entry
2911 : * have finished execution, but it is not livelocked by new incoming ones.
2912 : */
2913 0 : void __flush_workqueue(struct workqueue_struct *wq)
2914 : {
2915 0 : struct wq_flusher this_flusher = {
2916 : .list = LIST_HEAD_INIT(this_flusher.list),
2917 : .flush_color = -1,
2918 0 : .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
2919 : };
2920 : int next_color;
2921 :
2922 0 : if (WARN_ON(!wq_online))
2923 0 : return;
2924 :
2925 : lock_map_acquire(&wq->lockdep_map);
2926 : lock_map_release(&wq->lockdep_map);
2927 :
2928 0 : mutex_lock(&wq->mutex);
2929 :
2930 : /*
2931 : * Start-to-wait phase
2932 : */
2933 0 : next_color = work_next_color(wq->work_color);
2934 :
2935 0 : if (next_color != wq->flush_color) {
2936 : /*
2937 : * Color space is not full. The current work_color
2938 : * becomes our flush_color and work_color is advanced
2939 : * by one.
2940 : */
2941 0 : WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2942 0 : this_flusher.flush_color = wq->work_color;
2943 0 : wq->work_color = next_color;
2944 :
2945 0 : if (!wq->first_flusher) {
2946 : /* no flush in progress, become the first flusher */
2947 0 : WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2948 :
2949 0 : wq->first_flusher = &this_flusher;
2950 :
2951 0 : if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2952 : wq->work_color)) {
2953 : /* nothing to flush, done */
2954 0 : wq->flush_color = next_color;
2955 0 : wq->first_flusher = NULL;
2956 0 : goto out_unlock;
2957 : }
2958 : } else {
2959 : /* wait in queue */
2960 0 : WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2961 0 : list_add_tail(&this_flusher.list, &wq->flusher_queue);
2962 0 : flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2963 : }
2964 : } else {
2965 : /*
2966 : * Oops, color space is full, wait on overflow queue.
2967 : * The next flush completion will assign us
2968 : * flush_color and transfer to flusher_queue.
2969 : */
2970 0 : list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2971 : }
2972 :
2973 0 : check_flush_dependency(wq, NULL);
2974 :
2975 0 : mutex_unlock(&wq->mutex);
2976 :
2977 0 : wait_for_completion(&this_flusher.done);
2978 :
2979 : /*
2980 : * Wake-up-and-cascade phase
2981 : *
2982 : * First flushers are responsible for cascading flushes and
2983 : * handling overflow. Non-first flushers can simply return.
2984 : */
2985 0 : if (READ_ONCE(wq->first_flusher) != &this_flusher)
2986 : return;
2987 :
2988 0 : mutex_lock(&wq->mutex);
2989 :
2990 : /* we might have raced, check again with mutex held */
2991 0 : if (wq->first_flusher != &this_flusher)
2992 : goto out_unlock;
2993 :
2994 0 : WRITE_ONCE(wq->first_flusher, NULL);
2995 :
2996 0 : WARN_ON_ONCE(!list_empty(&this_flusher.list));
2997 0 : WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2998 :
2999 0 : while (true) {
3000 : struct wq_flusher *next, *tmp;
3001 :
3002 : /* complete all the flushers sharing the current flush color */
3003 0 : list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
3004 0 : if (next->flush_color != wq->flush_color)
3005 : break;
3006 0 : list_del_init(&next->list);
3007 0 : complete(&next->done);
3008 : }
3009 :
3010 0 : WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
3011 : wq->flush_color != work_next_color(wq->work_color));
3012 :
3013 : /* this flush_color is finished, advance by one */
3014 0 : wq->flush_color = work_next_color(wq->flush_color);
3015 :
3016 : /* one color has been freed, handle overflow queue */
3017 0 : if (!list_empty(&wq->flusher_overflow)) {
3018 : /*
3019 : * Assign the same color to all overflowed
3020 : * flushers, advance work_color and append to
3021 : * flusher_queue. This is the start-to-wait
3022 : * phase for these overflowed flushers.
3023 : */
3024 0 : list_for_each_entry(tmp, &wq->flusher_overflow, list)
3025 0 : tmp->flush_color = wq->work_color;
3026 :
3027 0 : wq->work_color = work_next_color(wq->work_color);
3028 :
3029 0 : list_splice_tail_init(&wq->flusher_overflow,
3030 : &wq->flusher_queue);
3031 0 : flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
3032 : }
3033 :
3034 0 : if (list_empty(&wq->flusher_queue)) {
3035 0 : WARN_ON_ONCE(wq->flush_color != wq->work_color);
3036 : break;
3037 : }
3038 :
3039 : /*
3040 : * Need to flush more colors. Make the next flusher
3041 : * the new first flusher and arm pwqs.
3042 : */
3043 0 : WARN_ON_ONCE(wq->flush_color == wq->work_color);
3044 0 : WARN_ON_ONCE(wq->flush_color != next->flush_color);
3045 :
3046 0 : list_del_init(&next->list);
3047 0 : wq->first_flusher = next;
3048 :
3049 0 : if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
3050 : break;
3051 :
3052 : /*
3053 : * Meh... this color is already done, clear first
3054 : * flusher and repeat cascading.
3055 : */
3056 0 : wq->first_flusher = NULL;
3057 : }
3058 :
3059 : out_unlock:
3060 0 : mutex_unlock(&wq->mutex);
3061 : }
3062 : EXPORT_SYMBOL(__flush_workqueue);
3063 :
3064 : /**
3065 : * drain_workqueue - drain a workqueue
3066 : * @wq: workqueue to drain
3067 : *
3068 : * Wait until the workqueue becomes empty. While draining is in progress,
3069 : * only chain queueing is allowed. IOW, only currently pending or running
3070 : * work items on @wq can queue further work items on it. @wq is flushed
3071 : * repeatedly until it becomes empty. The number of flushing is determined
3072 : * by the depth of chaining and should be relatively short. Whine if it
3073 : * takes too long.
3074 : */
3075 0 : void drain_workqueue(struct workqueue_struct *wq)
3076 : {
3077 0 : unsigned int flush_cnt = 0;
3078 : struct pool_workqueue *pwq;
3079 :
3080 : /*
3081 : * __queue_work() needs to test whether there are drainers, is much
3082 : * hotter than drain_workqueue() and already looks at @wq->flags.
3083 : * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
3084 : */
3085 0 : mutex_lock(&wq->mutex);
3086 0 : if (!wq->nr_drainers++)
3087 0 : wq->flags |= __WQ_DRAINING;
3088 0 : mutex_unlock(&wq->mutex);
3089 : reflush:
3090 0 : __flush_workqueue(wq);
3091 :
3092 0 : mutex_lock(&wq->mutex);
3093 :
3094 0 : for_each_pwq(pwq, wq) {
3095 : bool drained;
3096 :
3097 0 : raw_spin_lock_irq(&pwq->pool->lock);
3098 0 : drained = !pwq->nr_active && list_empty(&pwq->inactive_works);
3099 0 : raw_spin_unlock_irq(&pwq->pool->lock);
3100 :
3101 0 : if (drained)
3102 0 : continue;
3103 :
3104 0 : if (++flush_cnt == 10 ||
3105 0 : (flush_cnt % 100 == 0 && flush_cnt <= 1000))
3106 0 : pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
3107 : wq->name, __func__, flush_cnt);
3108 :
3109 0 : mutex_unlock(&wq->mutex);
3110 0 : goto reflush;
3111 : }
3112 :
3113 0 : if (!--wq->nr_drainers)
3114 0 : wq->flags &= ~__WQ_DRAINING;
3115 0 : mutex_unlock(&wq->mutex);
3116 0 : }
3117 : EXPORT_SYMBOL_GPL(drain_workqueue);
3118 :
3119 26 : static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
3120 : bool from_cancel)
3121 : {
3122 26 : struct worker *worker = NULL;
3123 : struct worker_pool *pool;
3124 : struct pool_workqueue *pwq;
3125 :
3126 : might_sleep();
3127 :
3128 : rcu_read_lock();
3129 26 : pool = get_work_pool(work);
3130 26 : if (!pool) {
3131 : rcu_read_unlock();
3132 22 : return false;
3133 : }
3134 :
3135 4 : raw_spin_lock_irq(&pool->lock);
3136 : /* see the comment in try_to_grab_pending() with the same code */
3137 4 : pwq = get_work_pwq(work);
3138 4 : if (pwq) {
3139 2 : if (unlikely(pwq->pool != pool))
3140 : goto already_gone;
3141 : } else {
3142 2 : worker = find_worker_executing_work(pool, work);
3143 2 : if (!worker)
3144 : goto already_gone;
3145 0 : pwq = worker->current_pwq;
3146 : }
3147 :
3148 2 : check_flush_dependency(pwq->wq, work);
3149 :
3150 2 : insert_wq_barrier(pwq, barr, work, worker);
3151 2 : raw_spin_unlock_irq(&pool->lock);
3152 :
3153 : /*
3154 : * Force a lock recursion deadlock when using flush_work() inside a
3155 : * single-threaded or rescuer equipped workqueue.
3156 : *
3157 : * For single threaded workqueues the deadlock happens when the work
3158 : * is after the work issuing the flush_work(). For rescuer equipped
3159 : * workqueues the deadlock happens when the rescuer stalls, blocking
3160 : * forward progress.
3161 : */
3162 : if (!from_cancel &&
3163 : (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {
3164 : lock_map_acquire(&pwq->wq->lockdep_map);
3165 : lock_map_release(&pwq->wq->lockdep_map);
3166 : }
3167 : rcu_read_unlock();
3168 2 : return true;
3169 : already_gone:
3170 2 : raw_spin_unlock_irq(&pool->lock);
3171 : rcu_read_unlock();
3172 2 : return false;
3173 : }
3174 :
3175 26 : static bool __flush_work(struct work_struct *work, bool from_cancel)
3176 : {
3177 : struct wq_barrier barr;
3178 :
3179 26 : if (WARN_ON(!wq_online))
3180 : return false;
3181 :
3182 26 : if (WARN_ON(!work->func))
3183 : return false;
3184 :
3185 : lock_map_acquire(&work->lockdep_map);
3186 : lock_map_release(&work->lockdep_map);
3187 :
3188 26 : if (start_flush_work(work, &barr, from_cancel)) {
3189 2 : wait_for_completion(&barr.done);
3190 2 : destroy_work_on_stack(&barr.work);
3191 2 : return true;
3192 : } else {
3193 : return false;
3194 : }
3195 : }
3196 :
3197 : /**
3198 : * flush_work - wait for a work to finish executing the last queueing instance
3199 : * @work: the work to flush
3200 : *
3201 : * Wait until @work has finished execution. @work is guaranteed to be idle
3202 : * on return if it hasn't been requeued since flush started.
3203 : *
3204 : * Return:
3205 : * %true if flush_work() waited for the work to finish execution,
3206 : * %false if it was already idle.
3207 : */
3208 26 : bool flush_work(struct work_struct *work)
3209 : {
3210 26 : return __flush_work(work, false);
3211 : }
3212 : EXPORT_SYMBOL_GPL(flush_work);
3213 :
3214 : struct cwt_wait {
3215 : wait_queue_entry_t wait;
3216 : struct work_struct *work;
3217 : };
3218 :
3219 0 : static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
3220 : {
3221 0 : struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
3222 :
3223 0 : if (cwait->work != key)
3224 : return 0;
3225 0 : return autoremove_wake_function(wait, mode, sync, key);
3226 : }
3227 :
3228 0 : static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
3229 : {
3230 : static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
3231 : unsigned long flags;
3232 : int ret;
3233 :
3234 : do {
3235 0 : ret = try_to_grab_pending(work, is_dwork, &flags);
3236 : /*
3237 : * If someone else is already canceling, wait for it to
3238 : * finish. flush_work() doesn't work for PREEMPT_NONE
3239 : * because we may get scheduled between @work's completion
3240 : * and the other canceling task resuming and clearing
3241 : * CANCELING - flush_work() will return false immediately
3242 : * as @work is no longer busy, try_to_grab_pending() will
3243 : * return -ENOENT as @work is still being canceled and the
3244 : * other canceling task won't be able to clear CANCELING as
3245 : * we're hogging the CPU.
3246 : *
3247 : * Let's wait for completion using a waitqueue. As this
3248 : * may lead to the thundering herd problem, use a custom
3249 : * wake function which matches @work along with exclusive
3250 : * wait and wakeup.
3251 : */
3252 0 : if (unlikely(ret == -ENOENT)) {
3253 : struct cwt_wait cwait;
3254 :
3255 0 : init_wait(&cwait.wait);
3256 0 : cwait.wait.func = cwt_wakefn;
3257 0 : cwait.work = work;
3258 :
3259 0 : prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
3260 : TASK_UNINTERRUPTIBLE);
3261 0 : if (work_is_canceling(work))
3262 0 : schedule();
3263 0 : finish_wait(&cancel_waitq, &cwait.wait);
3264 : }
3265 0 : } while (unlikely(ret < 0));
3266 :
3267 : /* tell other tasks trying to grab @work to back off */
3268 0 : mark_work_canceling(work);
3269 0 : local_irq_restore(flags);
3270 :
3271 : /*
3272 : * This allows canceling during early boot. We know that @work
3273 : * isn't executing.
3274 : */
3275 0 : if (wq_online)
3276 0 : __flush_work(work, true);
3277 :
3278 0 : clear_work_data(work);
3279 :
3280 : /*
3281 : * Paired with prepare_to_wait() above so that either
3282 : * waitqueue_active() is visible here or !work_is_canceling() is
3283 : * visible there.
3284 : */
3285 0 : smp_mb();
3286 0 : if (waitqueue_active(&cancel_waitq))
3287 0 : __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
3288 :
3289 0 : return ret;
3290 : }
3291 :
3292 : /**
3293 : * cancel_work_sync - cancel a work and wait for it to finish
3294 : * @work: the work to cancel
3295 : *
3296 : * Cancel @work and wait for its execution to finish. This function
3297 : * can be used even if the work re-queues itself or migrates to
3298 : * another workqueue. On return from this function, @work is
3299 : * guaranteed to be not pending or executing on any CPU.
3300 : *
3301 : * cancel_work_sync(&delayed_work->work) must not be used for
3302 : * delayed_work's. Use cancel_delayed_work_sync() instead.
3303 : *
3304 : * The caller must ensure that the workqueue on which @work was last
3305 : * queued can't be destroyed before this function returns.
3306 : *
3307 : * Return:
3308 : * %true if @work was pending, %false otherwise.
3309 : */
3310 0 : bool cancel_work_sync(struct work_struct *work)
3311 : {
3312 0 : return __cancel_work_timer(work, false);
3313 : }
3314 : EXPORT_SYMBOL_GPL(cancel_work_sync);
3315 :
3316 : /**
3317 : * flush_delayed_work - wait for a dwork to finish executing the last queueing
3318 : * @dwork: the delayed work to flush
3319 : *
3320 : * Delayed timer is cancelled and the pending work is queued for
3321 : * immediate execution. Like flush_work(), this function only
3322 : * considers the last queueing instance of @dwork.
3323 : *
3324 : * Return:
3325 : * %true if flush_work() waited for the work to finish execution,
3326 : * %false if it was already idle.
3327 : */
3328 0 : bool flush_delayed_work(struct delayed_work *dwork)
3329 : {
3330 : local_irq_disable();
3331 0 : if (del_timer_sync(&dwork->timer))
3332 0 : __queue_work(dwork->cpu, dwork->wq, &dwork->work);
3333 : local_irq_enable();
3334 0 : return flush_work(&dwork->work);
3335 : }
3336 : EXPORT_SYMBOL(flush_delayed_work);
3337 :
3338 : /**
3339 : * flush_rcu_work - wait for a rwork to finish executing the last queueing
3340 : * @rwork: the rcu work to flush
3341 : *
3342 : * Return:
3343 : * %true if flush_rcu_work() waited for the work to finish execution,
3344 : * %false if it was already idle.
3345 : */
3346 0 : bool flush_rcu_work(struct rcu_work *rwork)
3347 : {
3348 0 : if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
3349 0 : rcu_barrier();
3350 0 : flush_work(&rwork->work);
3351 0 : return true;
3352 : } else {
3353 0 : return flush_work(&rwork->work);
3354 : }
3355 : }
3356 : EXPORT_SYMBOL(flush_rcu_work);
3357 :
3358 38 : static bool __cancel_work(struct work_struct *work, bool is_dwork)
3359 : {
3360 : unsigned long flags;
3361 : int ret;
3362 :
3363 : do {
3364 38 : ret = try_to_grab_pending(work, is_dwork, &flags);
3365 38 : } while (unlikely(ret == -EAGAIN));
3366 :
3367 38 : if (unlikely(ret < 0))
3368 : return false;
3369 :
3370 76 : set_work_pool_and_clear_pending(work, get_work_pool_id(work));
3371 76 : local_irq_restore(flags);
3372 38 : return ret;
3373 : }
3374 :
3375 : /*
3376 : * See cancel_delayed_work()
3377 : */
3378 0 : bool cancel_work(struct work_struct *work)
3379 : {
3380 0 : return __cancel_work(work, false);
3381 : }
3382 : EXPORT_SYMBOL(cancel_work);
3383 :
3384 : /**
3385 : * cancel_delayed_work - cancel a delayed work
3386 : * @dwork: delayed_work to cancel
3387 : *
3388 : * Kill off a pending delayed_work.
3389 : *
3390 : * Return: %true if @dwork was pending and canceled; %false if it wasn't
3391 : * pending.
3392 : *
3393 : * Note:
3394 : * The work callback function may still be running on return, unless
3395 : * it returns %true and the work doesn't re-arm itself. Explicitly flush or
3396 : * use cancel_delayed_work_sync() to wait on it.
3397 : *
3398 : * This function is safe to call from any context including IRQ handler.
3399 : */
3400 38 : bool cancel_delayed_work(struct delayed_work *dwork)
3401 : {
3402 38 : return __cancel_work(&dwork->work, true);
3403 : }
3404 : EXPORT_SYMBOL(cancel_delayed_work);
3405 :
3406 : /**
3407 : * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3408 : * @dwork: the delayed work cancel
3409 : *
3410 : * This is cancel_work_sync() for delayed works.
3411 : *
3412 : * Return:
3413 : * %true if @dwork was pending, %false otherwise.
3414 : */
3415 0 : bool cancel_delayed_work_sync(struct delayed_work *dwork)
3416 : {
3417 0 : return __cancel_work_timer(&dwork->work, true);
3418 : }
3419 : EXPORT_SYMBOL(cancel_delayed_work_sync);
3420 :
3421 : /**
3422 : * schedule_on_each_cpu - execute a function synchronously on each online CPU
3423 : * @func: the function to call
3424 : *
3425 : * schedule_on_each_cpu() executes @func on each online CPU using the
3426 : * system workqueue and blocks until all CPUs have completed.
3427 : * schedule_on_each_cpu() is very slow.
3428 : *
3429 : * Return:
3430 : * 0 on success, -errno on failure.
3431 : */
3432 0 : int schedule_on_each_cpu(work_func_t func)
3433 : {
3434 : int cpu;
3435 : struct work_struct __percpu *works;
3436 :
3437 0 : works = alloc_percpu(struct work_struct);
3438 0 : if (!works)
3439 : return -ENOMEM;
3440 :
3441 : cpus_read_lock();
3442 :
3443 0 : for_each_online_cpu(cpu) {
3444 0 : struct work_struct *work = per_cpu_ptr(works, cpu);
3445 :
3446 0 : INIT_WORK(work, func);
3447 0 : schedule_work_on(cpu, work);
3448 : }
3449 :
3450 0 : for_each_online_cpu(cpu)
3451 0 : flush_work(per_cpu_ptr(works, cpu));
3452 :
3453 : cpus_read_unlock();
3454 0 : free_percpu(works);
3455 0 : return 0;
3456 : }
3457 :
3458 : /**
3459 : * execute_in_process_context - reliably execute the routine with user context
3460 : * @fn: the function to execute
3461 : * @ew: guaranteed storage for the execute work structure (must
3462 : * be available when the work executes)
3463 : *
3464 : * Executes the function immediately if process context is available,
3465 : * otherwise schedules the function for delayed execution.
3466 : *
3467 : * Return: 0 - function was executed
3468 : * 1 - function was scheduled for execution
3469 : */
3470 0 : int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3471 : {
3472 0 : if (!in_interrupt()) {
3473 0 : fn(&ew->work);
3474 0 : return 0;
3475 : }
3476 :
3477 0 : INIT_WORK(&ew->work, fn);
3478 0 : schedule_work(&ew->work);
3479 :
3480 0 : return 1;
3481 : }
3482 : EXPORT_SYMBOL_GPL(execute_in_process_context);
3483 :
3484 : /**
3485 : * free_workqueue_attrs - free a workqueue_attrs
3486 : * @attrs: workqueue_attrs to free
3487 : *
3488 : * Undo alloc_workqueue_attrs().
3489 : */
3490 0 : void free_workqueue_attrs(struct workqueue_attrs *attrs)
3491 : {
3492 3 : if (attrs) {
3493 6 : free_cpumask_var(attrs->cpumask);
3494 6 : kfree(attrs);
3495 : }
3496 0 : }
3497 :
3498 : /**
3499 : * alloc_workqueue_attrs - allocate a workqueue_attrs
3500 : *
3501 : * Allocate a new workqueue_attrs, initialize with default settings and
3502 : * return it.
3503 : *
3504 : * Return: The allocated new workqueue_attr on success. %NULL on failure.
3505 : */
3506 16 : struct workqueue_attrs *alloc_workqueue_attrs(void)
3507 : {
3508 : struct workqueue_attrs *attrs;
3509 :
3510 16 : attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3511 16 : if (!attrs)
3512 : goto fail;
3513 16 : if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
3514 : goto fail;
3515 :
3516 32 : cpumask_copy(attrs->cpumask, cpu_possible_mask);
3517 16 : return attrs;
3518 : fail:
3519 : free_workqueue_attrs(attrs);
3520 : return NULL;
3521 : }
3522 :
3523 : static void copy_workqueue_attrs(struct workqueue_attrs *to,
3524 : const struct workqueue_attrs *from)
3525 : {
3526 13 : to->nice = from->nice;
3527 26 : cpumask_copy(to->cpumask, from->cpumask);
3528 : /*
3529 : * Unlike hash and equality test, this function doesn't ignore
3530 : * ->no_numa as it is used for both pool and wq attrs. Instead,
3531 : * get_unbound_pool() explicitly clears ->no_numa after copying.
3532 : */
3533 13 : to->no_numa = from->no_numa;
3534 : }
3535 :
3536 : /* hash value of the content of @attr */
3537 3 : static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3538 : {
3539 3 : u32 hash = 0;
3540 :
3541 6 : hash = jhash_1word(attrs->nice, hash);
3542 3 : hash = jhash(cpumask_bits(attrs->cpumask),
3543 : BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3544 3 : return hash;
3545 : }
3546 :
3547 : /* content equality test */
3548 : static bool wqattrs_equal(const struct workqueue_attrs *a,
3549 : const struct workqueue_attrs *b)
3550 : {
3551 2 : if (a->nice != b->nice)
3552 : return false;
3553 4 : if (!cpumask_equal(a->cpumask, b->cpumask))
3554 : return false;
3555 : return true;
3556 : }
3557 :
3558 : /**
3559 : * init_worker_pool - initialize a newly zalloc'd worker_pool
3560 : * @pool: worker_pool to initialize
3561 : *
3562 : * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3563 : *
3564 : * Return: 0 on success, -errno on failure. Even on failure, all fields
3565 : * inside @pool proper are initialized and put_unbound_pool() can be called
3566 : * on @pool safely to release it.
3567 : */
3568 3 : static int init_worker_pool(struct worker_pool *pool)
3569 : {
3570 : raw_spin_lock_init(&pool->lock);
3571 3 : pool->id = -1;
3572 3 : pool->cpu = -1;
3573 3 : pool->node = NUMA_NO_NODE;
3574 3 : pool->flags |= POOL_DISASSOCIATED;
3575 3 : pool->watchdog_ts = jiffies;
3576 6 : INIT_LIST_HEAD(&pool->worklist);
3577 6 : INIT_LIST_HEAD(&pool->idle_list);
3578 6 : hash_init(pool->busy_hash);
3579 :
3580 3 : timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
3581 6 : INIT_WORK(&pool->idle_cull_work, idle_cull_fn);
3582 :
3583 3 : timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
3584 :
3585 6 : INIT_LIST_HEAD(&pool->workers);
3586 6 : INIT_LIST_HEAD(&pool->dying_workers);
3587 :
3588 6 : ida_init(&pool->worker_ida);
3589 6 : INIT_HLIST_NODE(&pool->hash_node);
3590 3 : pool->refcnt = 1;
3591 :
3592 : /* shouldn't fail above this point */
3593 3 : pool->attrs = alloc_workqueue_attrs();
3594 3 : if (!pool->attrs)
3595 : return -ENOMEM;
3596 3 : return 0;
3597 : }
3598 :
3599 : #ifdef CONFIG_LOCKDEP
3600 : static void wq_init_lockdep(struct workqueue_struct *wq)
3601 : {
3602 : char *lock_name;
3603 :
3604 : lockdep_register_key(&wq->key);
3605 : lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
3606 : if (!lock_name)
3607 : lock_name = wq->name;
3608 :
3609 : wq->lock_name = lock_name;
3610 : lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
3611 : }
3612 :
3613 : static void wq_unregister_lockdep(struct workqueue_struct *wq)
3614 : {
3615 : lockdep_unregister_key(&wq->key);
3616 : }
3617 :
3618 : static void wq_free_lockdep(struct workqueue_struct *wq)
3619 : {
3620 : if (wq->lock_name != wq->name)
3621 : kfree(wq->lock_name);
3622 : }
3623 : #else
3624 : static void wq_init_lockdep(struct workqueue_struct *wq)
3625 : {
3626 : }
3627 :
3628 : static void wq_unregister_lockdep(struct workqueue_struct *wq)
3629 : {
3630 : }
3631 :
3632 : static void wq_free_lockdep(struct workqueue_struct *wq)
3633 : {
3634 : }
3635 : #endif
3636 :
3637 0 : static void rcu_free_wq(struct rcu_head *rcu)
3638 : {
3639 0 : struct workqueue_struct *wq =
3640 0 : container_of(rcu, struct workqueue_struct, rcu);
3641 :
3642 0 : wq_free_lockdep(wq);
3643 :
3644 0 : if (!(wq->flags & WQ_UNBOUND))
3645 0 : free_percpu(wq->cpu_pwqs);
3646 : else
3647 0 : free_workqueue_attrs(wq->unbound_attrs);
3648 :
3649 0 : kfree(wq);
3650 0 : }
3651 :
3652 0 : static void rcu_free_pool(struct rcu_head *rcu)
3653 : {
3654 0 : struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3655 :
3656 0 : ida_destroy(&pool->worker_ida);
3657 0 : free_workqueue_attrs(pool->attrs);
3658 0 : kfree(pool);
3659 0 : }
3660 :
3661 : /**
3662 : * put_unbound_pool - put a worker_pool
3663 : * @pool: worker_pool to put
3664 : *
3665 : * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
3666 : * safe manner. get_unbound_pool() calls this function on its failure path
3667 : * and this function should be able to release pools which went through,
3668 : * successfully or not, init_worker_pool().
3669 : *
3670 : * Should be called with wq_pool_mutex held.
3671 : */
3672 0 : static void put_unbound_pool(struct worker_pool *pool)
3673 : {
3674 0 : DECLARE_COMPLETION_ONSTACK(detach_completion);
3675 : struct list_head cull_list;
3676 : struct worker *worker;
3677 :
3678 0 : INIT_LIST_HEAD(&cull_list);
3679 :
3680 : lockdep_assert_held(&wq_pool_mutex);
3681 :
3682 0 : if (--pool->refcnt)
3683 0 : return;
3684 :
3685 : /* sanity checks */
3686 0 : if (WARN_ON(!(pool->cpu < 0)) ||
3687 0 : WARN_ON(!list_empty(&pool->worklist)))
3688 : return;
3689 :
3690 : /* release id and unhash */
3691 0 : if (pool->id >= 0)
3692 0 : idr_remove(&worker_pool_idr, pool->id);
3693 0 : hash_del(&pool->hash_node);
3694 :
3695 : /*
3696 : * Become the manager and destroy all workers. This prevents
3697 : * @pool's workers from blocking on attach_mutex. We're the last
3698 : * manager and @pool gets freed with the flag set.
3699 : *
3700 : * Having a concurrent manager is quite unlikely to happen as we can
3701 : * only get here with
3702 : * pwq->refcnt == pool->refcnt == 0
3703 : * which implies no work queued to the pool, which implies no worker can
3704 : * become the manager. However a worker could have taken the role of
3705 : * manager before the refcnts dropped to 0, since maybe_create_worker()
3706 : * drops pool->lock
3707 : */
3708 : while (true) {
3709 0 : rcuwait_wait_event(&manager_wait,
3710 : !(pool->flags & POOL_MANAGER_ACTIVE),
3711 : TASK_UNINTERRUPTIBLE);
3712 :
3713 0 : mutex_lock(&wq_pool_attach_mutex);
3714 0 : raw_spin_lock_irq(&pool->lock);
3715 0 : if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
3716 0 : pool->flags |= POOL_MANAGER_ACTIVE;
3717 : break;
3718 : }
3719 0 : raw_spin_unlock_irq(&pool->lock);
3720 0 : mutex_unlock(&wq_pool_attach_mutex);
3721 : }
3722 :
3723 0 : while ((worker = first_idle_worker(pool)))
3724 0 : set_worker_dying(worker, &cull_list);
3725 0 : WARN_ON(pool->nr_workers || pool->nr_idle);
3726 0 : raw_spin_unlock_irq(&pool->lock);
3727 :
3728 0 : wake_dying_workers(&cull_list);
3729 :
3730 0 : if (!list_empty(&pool->workers) || !list_empty(&pool->dying_workers))
3731 0 : pool->detach_completion = &detach_completion;
3732 0 : mutex_unlock(&wq_pool_attach_mutex);
3733 :
3734 0 : if (pool->detach_completion)
3735 0 : wait_for_completion(pool->detach_completion);
3736 :
3737 : /* shut down the timers */
3738 0 : del_timer_sync(&pool->idle_timer);
3739 0 : cancel_work_sync(&pool->idle_cull_work);
3740 0 : del_timer_sync(&pool->mayday_timer);
3741 :
3742 : /* RCU protected to allow dereferences from get_work_pool() */
3743 0 : call_rcu(&pool->rcu, rcu_free_pool);
3744 : }
3745 :
3746 : /**
3747 : * get_unbound_pool - get a worker_pool with the specified attributes
3748 : * @attrs: the attributes of the worker_pool to get
3749 : *
3750 : * Obtain a worker_pool which has the same attributes as @attrs, bump the
3751 : * reference count and return it. If there already is a matching
3752 : * worker_pool, it will be used; otherwise, this function attempts to
3753 : * create a new one.
3754 : *
3755 : * Should be called with wq_pool_mutex held.
3756 : *
3757 : * Return: On success, a worker_pool with the same attributes as @attrs.
3758 : * On failure, %NULL.
3759 : */
3760 3 : static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3761 : {
3762 3 : u32 hash = wqattrs_hash(attrs);
3763 : struct worker_pool *pool;
3764 : int node;
3765 3 : int target_node = NUMA_NO_NODE;
3766 :
3767 : lockdep_assert_held(&wq_pool_mutex);
3768 :
3769 : /* do we already have a matching pool? */
3770 3 : hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3771 4 : if (wqattrs_equal(pool->attrs, attrs)) {
3772 2 : pool->refcnt++;
3773 2 : return pool;
3774 : }
3775 : }
3776 :
3777 : /* if cpumask is contained inside a NUMA node, we belong to that node */
3778 1 : if (wq_numa_enabled) {
3779 0 : for_each_node(node) {
3780 0 : if (cpumask_subset(attrs->cpumask,
3781 0 : wq_numa_possible_cpumask[node])) {
3782 : target_node = node;
3783 : break;
3784 : }
3785 : }
3786 : }
3787 :
3788 : /* nope, create a new one */
3789 1 : pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
3790 1 : if (!pool || init_worker_pool(pool) < 0)
3791 : goto fail;
3792 :
3793 : lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3794 2 : copy_workqueue_attrs(pool->attrs, attrs);
3795 1 : pool->node = target_node;
3796 :
3797 : /*
3798 : * no_numa isn't a worker_pool attribute, always clear it. See
3799 : * 'struct workqueue_attrs' comments for detail.
3800 : */
3801 1 : pool->attrs->no_numa = false;
3802 :
3803 1 : if (worker_pool_assign_id(pool) < 0)
3804 : goto fail;
3805 :
3806 : /* create and start the initial worker */
3807 1 : if (wq_online && !create_worker(pool))
3808 : goto fail;
3809 :
3810 : /* install */
3811 2 : hash_add(unbound_pool_hash, &pool->hash_node, hash);
3812 :
3813 1 : return pool;
3814 : fail:
3815 0 : if (pool)
3816 0 : put_unbound_pool(pool);
3817 : return NULL;
3818 : }
3819 :
3820 0 : static void rcu_free_pwq(struct rcu_head *rcu)
3821 : {
3822 0 : kmem_cache_free(pwq_cache,
3823 0 : container_of(rcu, struct pool_workqueue, rcu));
3824 0 : }
3825 :
3826 : /*
3827 : * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3828 : * and needs to be destroyed.
3829 : */
3830 0 : static void pwq_unbound_release_workfn(struct work_struct *work)
3831 : {
3832 0 : struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3833 : unbound_release_work);
3834 0 : struct workqueue_struct *wq = pwq->wq;
3835 0 : struct worker_pool *pool = pwq->pool;
3836 0 : bool is_last = false;
3837 :
3838 : /*
3839 : * when @pwq is not linked, it doesn't hold any reference to the
3840 : * @wq, and @wq is invalid to access.
3841 : */
3842 0 : if (!list_empty(&pwq->pwqs_node)) {
3843 0 : if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3844 : return;
3845 :
3846 0 : mutex_lock(&wq->mutex);
3847 0 : list_del_rcu(&pwq->pwqs_node);
3848 0 : is_last = list_empty(&wq->pwqs);
3849 0 : mutex_unlock(&wq->mutex);
3850 : }
3851 :
3852 0 : mutex_lock(&wq_pool_mutex);
3853 0 : put_unbound_pool(pool);
3854 0 : mutex_unlock(&wq_pool_mutex);
3855 :
3856 0 : call_rcu(&pwq->rcu, rcu_free_pwq);
3857 :
3858 : /*
3859 : * If we're the last pwq going away, @wq is already dead and no one
3860 : * is gonna access it anymore. Schedule RCU free.
3861 : */
3862 0 : if (is_last) {
3863 0 : wq_unregister_lockdep(wq);
3864 0 : call_rcu(&wq->rcu, rcu_free_wq);
3865 : }
3866 : }
3867 :
3868 : /**
3869 : * pwq_adjust_max_active - update a pwq's max_active to the current setting
3870 : * @pwq: target pool_workqueue
3871 : *
3872 : * If @pwq isn't freezing, set @pwq->max_active to the associated
3873 : * workqueue's saved_max_active and activate inactive work items
3874 : * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3875 : */
3876 28 : static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3877 : {
3878 28 : struct workqueue_struct *wq = pwq->wq;
3879 28 : bool freezable = wq->flags & WQ_FREEZABLE;
3880 : unsigned long flags;
3881 :
3882 : /* for @wq->saved_max_active */
3883 : lockdep_assert_held(&wq->mutex);
3884 :
3885 : /* fast exit for non-freezable wqs */
3886 28 : if (!freezable && pwq->max_active == wq->saved_max_active)
3887 : return;
3888 :
3889 : /* this function can be called during early boot w/ irq disabled */
3890 17 : raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3891 :
3892 : /*
3893 : * During [un]freezing, the caller is responsible for ensuring that
3894 : * this function is called at least once after @workqueue_freezing
3895 : * is updated and visible.
3896 : */
3897 17 : if (!freezable || !workqueue_freezing) {
3898 17 : bool kick = false;
3899 :
3900 17 : pwq->max_active = wq->saved_max_active;
3901 :
3902 51 : while (!list_empty(&pwq->inactive_works) &&
3903 0 : pwq->nr_active < pwq->max_active) {
3904 0 : pwq_activate_first_inactive(pwq);
3905 0 : kick = true;
3906 : }
3907 :
3908 : /*
3909 : * Need to kick a worker after thawed or an unbound wq's
3910 : * max_active is bumped. In realtime scenarios, always kicking a
3911 : * worker will cause interference on the isolated cpu cores, so
3912 : * let's kick iff work items were activated.
3913 : */
3914 17 : if (kick)
3915 0 : wake_up_worker(pwq->pool);
3916 : } else {
3917 0 : pwq->max_active = 0;
3918 : }
3919 :
3920 34 : raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3921 : }
3922 :
3923 : /* initialize newly allocated @pwq which is associated with @wq and @pool */
3924 14 : static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3925 : struct worker_pool *pool)
3926 : {
3927 14 : BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3928 :
3929 14 : memset(pwq, 0, sizeof(*pwq));
3930 :
3931 14 : pwq->pool = pool;
3932 14 : pwq->wq = wq;
3933 14 : pwq->flush_color = -1;
3934 14 : pwq->refcnt = 1;
3935 28 : INIT_LIST_HEAD(&pwq->inactive_works);
3936 28 : INIT_LIST_HEAD(&pwq->pwqs_node);
3937 28 : INIT_LIST_HEAD(&pwq->mayday_node);
3938 28 : INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3939 14 : }
3940 :
3941 : /* sync @pwq with the current state of its associated wq and link it */
3942 17 : static void link_pwq(struct pool_workqueue *pwq)
3943 : {
3944 17 : struct workqueue_struct *wq = pwq->wq;
3945 :
3946 : lockdep_assert_held(&wq->mutex);
3947 :
3948 : /* may be called multiple times, ignore if already linked */
3949 34 : if (!list_empty(&pwq->pwqs_node))
3950 : return;
3951 :
3952 : /* set the matching work_color */
3953 14 : pwq->work_color = wq->work_color;
3954 :
3955 : /* sync max_active to the current setting */
3956 14 : pwq_adjust_max_active(pwq);
3957 :
3958 : /* link in @pwq */
3959 14 : list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3960 : }
3961 :
3962 : /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3963 3 : static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3964 : const struct workqueue_attrs *attrs)
3965 : {
3966 : struct worker_pool *pool;
3967 : struct pool_workqueue *pwq;
3968 :
3969 : lockdep_assert_held(&wq_pool_mutex);
3970 :
3971 3 : pool = get_unbound_pool(attrs);
3972 3 : if (!pool)
3973 : return NULL;
3974 :
3975 3 : pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3976 3 : if (!pwq) {
3977 0 : put_unbound_pool(pool);
3978 0 : return NULL;
3979 : }
3980 :
3981 3 : init_pwq(pwq, wq, pool);
3982 3 : return pwq;
3983 : }
3984 :
3985 : /**
3986 : * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3987 : * @attrs: the wq_attrs of the default pwq of the target workqueue
3988 : * @node: the target NUMA node
3989 : * @cpu_going_down: if >= 0, the CPU to consider as offline
3990 : * @cpumask: outarg, the resulting cpumask
3991 : *
3992 : * Calculate the cpumask a workqueue with @attrs should use on @node. If
3993 : * @cpu_going_down is >= 0, that cpu is considered offline during
3994 : * calculation. The result is stored in @cpumask.
3995 : *
3996 : * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3997 : * enabled and @node has online CPUs requested by @attrs, the returned
3998 : * cpumask is the intersection of the possible CPUs of @node and
3999 : * @attrs->cpumask.
4000 : *
4001 : * The caller is responsible for ensuring that the cpumask of @node stays
4002 : * stable.
4003 : *
4004 : * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
4005 : * %false if equal.
4006 : */
4007 3 : static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
4008 : int cpu_going_down, cpumask_t *cpumask)
4009 : {
4010 3 : if (!wq_numa_enabled || attrs->no_numa)
4011 : goto use_dfl;
4012 :
4013 : /* does @node have any online CPUs @attrs wants? */
4014 0 : cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
4015 0 : if (cpu_going_down >= 0)
4016 : cpumask_clear_cpu(cpu_going_down, cpumask);
4017 :
4018 0 : if (cpumask_empty(cpumask))
4019 : goto use_dfl;
4020 :
4021 : /* yeap, return possible CPUs in @node that @attrs wants */
4022 0 : cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
4023 :
4024 0 : if (cpumask_empty(cpumask)) {
4025 0 : pr_warn_once("WARNING: workqueue cpumask: online intersect > "
4026 : "possible intersect\n");
4027 : return false;
4028 : }
4029 :
4030 0 : return !cpumask_equal(cpumask, attrs->cpumask);
4031 :
4032 : use_dfl:
4033 6 : cpumask_copy(cpumask, attrs->cpumask);
4034 3 : return false;
4035 : }
4036 :
4037 : /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
4038 : static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
4039 : int node,
4040 : struct pool_workqueue *pwq)
4041 : {
4042 : struct pool_workqueue *old_pwq;
4043 :
4044 : lockdep_assert_held(&wq_pool_mutex);
4045 : lockdep_assert_held(&wq->mutex);
4046 :
4047 : /* link_pwq() can handle duplicate calls */
4048 3 : link_pwq(pwq);
4049 :
4050 3 : old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4051 3 : rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
4052 : return old_pwq;
4053 : }
4054 :
4055 : /* context to store the prepared attrs & pwqs before applying */
4056 : struct apply_wqattrs_ctx {
4057 : struct workqueue_struct *wq; /* target workqueue */
4058 : struct workqueue_attrs *attrs; /* attrs to apply */
4059 : struct list_head list; /* queued for batching commit */
4060 : struct pool_workqueue *dfl_pwq;
4061 : struct pool_workqueue *pwq_tbl[];
4062 : };
4063 :
4064 : /* free the resources after success or abort */
4065 3 : static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
4066 : {
4067 3 : if (ctx) {
4068 : int node;
4069 :
4070 3 : for_each_node(node)
4071 3 : put_pwq_unlocked(ctx->pwq_tbl[node]);
4072 3 : put_pwq_unlocked(ctx->dfl_pwq);
4073 :
4074 6 : free_workqueue_attrs(ctx->attrs);
4075 :
4076 3 : kfree(ctx);
4077 : }
4078 3 : }
4079 :
4080 : /* allocate the attrs and pwqs for later installation */
4081 : static struct apply_wqattrs_ctx *
4082 3 : apply_wqattrs_prepare(struct workqueue_struct *wq,
4083 : const struct workqueue_attrs *attrs,
4084 : const cpumask_var_t unbound_cpumask)
4085 : {
4086 : struct apply_wqattrs_ctx *ctx;
4087 : struct workqueue_attrs *new_attrs, *tmp_attrs;
4088 : int node;
4089 :
4090 : lockdep_assert_held(&wq_pool_mutex);
4091 :
4092 3 : ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
4093 :
4094 3 : new_attrs = alloc_workqueue_attrs();
4095 3 : tmp_attrs = alloc_workqueue_attrs();
4096 3 : if (!ctx || !new_attrs || !tmp_attrs)
4097 : goto out_free;
4098 :
4099 : /*
4100 : * Calculate the attrs of the default pwq with unbound_cpumask
4101 : * which is wq_unbound_cpumask or to set to wq_unbound_cpumask.
4102 : * If the user configured cpumask doesn't overlap with the
4103 : * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
4104 : */
4105 3 : copy_workqueue_attrs(new_attrs, attrs);
4106 6 : cpumask_and(new_attrs->cpumask, new_attrs->cpumask, unbound_cpumask);
4107 6 : if (unlikely(cpumask_empty(new_attrs->cpumask)))
4108 0 : cpumask_copy(new_attrs->cpumask, unbound_cpumask);
4109 :
4110 : /*
4111 : * We may create multiple pwqs with differing cpumasks. Make a
4112 : * copy of @new_attrs which will be modified and used to obtain
4113 : * pools.
4114 : */
4115 3 : copy_workqueue_attrs(tmp_attrs, new_attrs);
4116 :
4117 : /*
4118 : * If something goes wrong during CPU up/down, we'll fall back to
4119 : * the default pwq covering whole @attrs->cpumask. Always create
4120 : * it even if we don't use it immediately.
4121 : */
4122 3 : ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
4123 3 : if (!ctx->dfl_pwq)
4124 : goto out_free;
4125 :
4126 6 : for_each_node(node) {
4127 3 : if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
4128 0 : ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
4129 0 : if (!ctx->pwq_tbl[node])
4130 : goto out_free;
4131 : } else {
4132 3 : ctx->dfl_pwq->refcnt++;
4133 3 : ctx->pwq_tbl[node] = ctx->dfl_pwq;
4134 : }
4135 : }
4136 :
4137 : /* save the user configured attrs and sanitize it. */
4138 3 : copy_workqueue_attrs(new_attrs, attrs);
4139 6 : cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
4140 3 : ctx->attrs = new_attrs;
4141 :
4142 3 : ctx->wq = wq;
4143 3 : free_workqueue_attrs(tmp_attrs);
4144 3 : return ctx;
4145 :
4146 : out_free:
4147 0 : free_workqueue_attrs(tmp_attrs);
4148 0 : free_workqueue_attrs(new_attrs);
4149 0 : apply_wqattrs_cleanup(ctx);
4150 0 : return NULL;
4151 : }
4152 :
4153 : /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
4154 3 : static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
4155 : {
4156 : int node;
4157 :
4158 : /* all pwqs have been created successfully, let's install'em */
4159 3 : mutex_lock(&ctx->wq->mutex);
4160 :
4161 6 : copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4162 :
4163 : /* save the previous pwq and install the new one */
4164 6 : for_each_node(node)
4165 6 : ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
4166 : ctx->pwq_tbl[node]);
4167 :
4168 : /* @dfl_pwq might not have been used, ensure it's linked */
4169 3 : link_pwq(ctx->dfl_pwq);
4170 3 : swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
4171 :
4172 3 : mutex_unlock(&ctx->wq->mutex);
4173 3 : }
4174 :
4175 : static void apply_wqattrs_lock(void)
4176 : {
4177 : /* CPUs should stay stable across pwq creations and installations */
4178 : cpus_read_lock();
4179 0 : mutex_lock(&wq_pool_mutex);
4180 : }
4181 :
4182 : static void apply_wqattrs_unlock(void)
4183 : {
4184 0 : mutex_unlock(&wq_pool_mutex);
4185 : cpus_read_unlock();
4186 : }
4187 :
4188 3 : static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
4189 : const struct workqueue_attrs *attrs)
4190 : {
4191 : struct apply_wqattrs_ctx *ctx;
4192 :
4193 : /* only unbound workqueues can change attributes */
4194 3 : if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
4195 : return -EINVAL;
4196 :
4197 : /* creating multiple pwqs breaks ordering guarantee */
4198 6 : if (!list_empty(&wq->pwqs)) {
4199 0 : if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4200 : return -EINVAL;
4201 :
4202 0 : wq->flags &= ~__WQ_ORDERED;
4203 : }
4204 :
4205 3 : ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
4206 3 : if (!ctx)
4207 : return -ENOMEM;
4208 :
4209 : /* the ctx has been prepared successfully, let's commit it */
4210 3 : apply_wqattrs_commit(ctx);
4211 3 : apply_wqattrs_cleanup(ctx);
4212 :
4213 3 : return 0;
4214 : }
4215 :
4216 : /**
4217 : * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4218 : * @wq: the target workqueue
4219 : * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
4220 : *
4221 : * Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
4222 : * machines, this function maps a separate pwq to each NUMA node with
4223 : * possibles CPUs in @attrs->cpumask so that work items are affine to the
4224 : * NUMA node it was issued on. Older pwqs are released as in-flight work
4225 : * items finish. Note that a work item which repeatedly requeues itself
4226 : * back-to-back will stay on its current pwq.
4227 : *
4228 : * Performs GFP_KERNEL allocations.
4229 : *
4230 : * Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
4231 : *
4232 : * Return: 0 on success and -errno on failure.
4233 : */
4234 3 : int apply_workqueue_attrs(struct workqueue_struct *wq,
4235 : const struct workqueue_attrs *attrs)
4236 : {
4237 : int ret;
4238 :
4239 : lockdep_assert_cpus_held();
4240 :
4241 3 : mutex_lock(&wq_pool_mutex);
4242 3 : ret = apply_workqueue_attrs_locked(wq, attrs);
4243 3 : mutex_unlock(&wq_pool_mutex);
4244 :
4245 3 : return ret;
4246 : }
4247 :
4248 : /**
4249 : * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4250 : * @wq: the target workqueue
4251 : * @cpu: the CPU coming up or going down
4252 : * @online: whether @cpu is coming up or going down
4253 : *
4254 : * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4255 : * %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
4256 : * @wq accordingly.
4257 : *
4258 : * If NUMA affinity can't be adjusted due to memory allocation failure, it
4259 : * falls back to @wq->dfl_pwq which may not be optimal but is always
4260 : * correct.
4261 : *
4262 : * Note that when the last allowed CPU of a NUMA node goes offline for a
4263 : * workqueue with a cpumask spanning multiple nodes, the workers which were
4264 : * already executing the work items for the workqueue will lose their CPU
4265 : * affinity and may execute on any CPU. This is similar to how per-cpu
4266 : * workqueues behave on CPU_DOWN. If a workqueue user wants strict
4267 : * affinity, it's the user's responsibility to flush the work item from
4268 : * CPU_DOWN_PREPARE.
4269 : */
4270 8 : static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4271 : bool online)
4272 : {
4273 8 : int node = cpu_to_node(cpu);
4274 8 : int cpu_off = online ? -1 : cpu;
4275 8 : struct pool_workqueue *old_pwq = NULL, *pwq;
4276 : struct workqueue_attrs *target_attrs;
4277 : cpumask_t *cpumask;
4278 :
4279 : lockdep_assert_held(&wq_pool_mutex);
4280 :
4281 8 : if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||
4282 0 : wq->unbound_attrs->no_numa)
4283 : return;
4284 :
4285 : /*
4286 : * We don't wanna alloc/free wq_attrs for each wq for each CPU.
4287 : * Let's use a preallocated one. The following buf is protected by
4288 : * CPU hotplug exclusion.
4289 : */
4290 0 : target_attrs = wq_update_unbound_numa_attrs_buf;
4291 0 : cpumask = target_attrs->cpumask;
4292 :
4293 0 : copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4294 0 : pwq = unbound_pwq_by_node(wq, node);
4295 :
4296 : /*
4297 : * Let's determine what needs to be done. If the target cpumask is
4298 : * different from the default pwq's, we need to compare it to @pwq's
4299 : * and create a new one if they don't match. If the target cpumask
4300 : * equals the default pwq's, the default pwq should be used.
4301 : */
4302 0 : if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
4303 0 : if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4304 : return;
4305 : } else {
4306 : goto use_dfl_pwq;
4307 : }
4308 :
4309 : /* create a new pwq */
4310 0 : pwq = alloc_unbound_pwq(wq, target_attrs);
4311 0 : if (!pwq) {
4312 0 : pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4313 : wq->name);
4314 0 : goto use_dfl_pwq;
4315 : }
4316 :
4317 : /* Install the new pwq. */
4318 0 : mutex_lock(&wq->mutex);
4319 0 : old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4320 0 : goto out_unlock;
4321 :
4322 : use_dfl_pwq:
4323 0 : mutex_lock(&wq->mutex);
4324 0 : raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4325 0 : get_pwq(wq->dfl_pwq);
4326 0 : raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4327 0 : old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4328 : out_unlock:
4329 0 : mutex_unlock(&wq->mutex);
4330 0 : put_pwq_unlocked(old_pwq);
4331 : }
4332 :
4333 14 : static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4334 : {
4335 14 : bool highpri = wq->flags & WQ_HIGHPRI;
4336 : int cpu, ret;
4337 :
4338 14 : if (!(wq->flags & WQ_UNBOUND)) {
4339 11 : wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4340 11 : if (!wq->cpu_pwqs)
4341 : return -ENOMEM;
4342 :
4343 11 : for_each_possible_cpu(cpu) {
4344 11 : struct pool_workqueue *pwq =
4345 11 : per_cpu_ptr(wq->cpu_pwqs, cpu);
4346 11 : struct worker_pool *cpu_pools =
4347 11 : per_cpu(cpu_worker_pools, cpu);
4348 :
4349 11 : init_pwq(pwq, wq, &cpu_pools[highpri]);
4350 :
4351 11 : mutex_lock(&wq->mutex);
4352 11 : link_pwq(pwq);
4353 11 : mutex_unlock(&wq->mutex);
4354 : }
4355 : return 0;
4356 : }
4357 :
4358 : cpus_read_lock();
4359 3 : if (wq->flags & __WQ_ORDERED) {
4360 1 : ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4361 : /* there should only be single pwq for ordering guarantee */
4362 1 : WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
4363 : wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4364 : "ordering guarantee broken for workqueue %s\n", wq->name);
4365 : } else {
4366 2 : ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4367 : }
4368 : cpus_read_unlock();
4369 :
4370 : return ret;
4371 : }
4372 :
4373 14 : static int wq_clamp_max_active(int max_active, unsigned int flags,
4374 : const char *name)
4375 : {
4376 14 : int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4377 :
4378 14 : if (max_active < 1 || max_active > lim)
4379 0 : pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4380 : max_active, name, 1, lim);
4381 :
4382 14 : return clamp_val(max_active, 1, lim);
4383 : }
4384 :
4385 : /*
4386 : * Workqueues which may be used during memory reclaim should have a rescuer
4387 : * to guarantee forward progress.
4388 : */
4389 14 : static int init_rescuer(struct workqueue_struct *wq)
4390 : {
4391 : struct worker *rescuer;
4392 : int ret;
4393 :
4394 14 : if (!(wq->flags & WQ_MEM_RECLAIM))
4395 : return 0;
4396 :
4397 4 : rescuer = alloc_worker(NUMA_NO_NODE);
4398 4 : if (!rescuer) {
4399 0 : pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
4400 : wq->name);
4401 0 : return -ENOMEM;
4402 : }
4403 :
4404 4 : rescuer->rescue_wq = wq;
4405 4 : rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4406 8 : if (IS_ERR(rescuer->task)) {
4407 0 : ret = PTR_ERR(rescuer->task);
4408 0 : pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
4409 : wq->name, ERR_PTR(ret));
4410 0 : kfree(rescuer);
4411 0 : return ret;
4412 : }
4413 :
4414 4 : wq->rescuer = rescuer;
4415 4 : kthread_bind_mask(rescuer->task, cpu_possible_mask);
4416 4 : wake_up_process(rescuer->task);
4417 :
4418 4 : return 0;
4419 : }
4420 :
4421 : __printf(1, 4)
4422 14 : struct workqueue_struct *alloc_workqueue(const char *fmt,
4423 : unsigned int flags,
4424 : int max_active, ...)
4425 : {
4426 14 : size_t tbl_size = 0;
4427 : va_list args;
4428 : struct workqueue_struct *wq;
4429 : struct pool_workqueue *pwq;
4430 :
4431 : /*
4432 : * Unbound && max_active == 1 used to imply ordered, which is no
4433 : * longer the case on NUMA machines due to per-node pools. While
4434 : * alloc_ordered_workqueue() is the right way to create an ordered
4435 : * workqueue, keep the previous behavior to avoid subtle breakages
4436 : * on NUMA.
4437 : */
4438 14 : if ((flags & WQ_UNBOUND) && max_active == 1)
4439 1 : flags |= __WQ_ORDERED;
4440 :
4441 : /* see the comment above the definition of WQ_POWER_EFFICIENT */
4442 14 : if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4443 0 : flags |= WQ_UNBOUND;
4444 :
4445 : /* allocate wq and format name */
4446 14 : if (flags & WQ_UNBOUND)
4447 3 : tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
4448 :
4449 14 : wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4450 14 : if (!wq)
4451 : return NULL;
4452 :
4453 14 : if (flags & WQ_UNBOUND) {
4454 3 : wq->unbound_attrs = alloc_workqueue_attrs();
4455 3 : if (!wq->unbound_attrs)
4456 : goto err_free_wq;
4457 : }
4458 :
4459 14 : va_start(args, max_active);
4460 14 : vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4461 14 : va_end(args);
4462 :
4463 14 : max_active = max_active ?: WQ_DFL_ACTIVE;
4464 14 : max_active = wq_clamp_max_active(max_active, flags, wq->name);
4465 :
4466 : /* init wq */
4467 14 : wq->flags = flags;
4468 14 : wq->saved_max_active = max_active;
4469 14 : mutex_init(&wq->mutex);
4470 28 : atomic_set(&wq->nr_pwqs_to_flush, 0);
4471 28 : INIT_LIST_HEAD(&wq->pwqs);
4472 28 : INIT_LIST_HEAD(&wq->flusher_queue);
4473 28 : INIT_LIST_HEAD(&wq->flusher_overflow);
4474 28 : INIT_LIST_HEAD(&wq->maydays);
4475 :
4476 14 : wq_init_lockdep(wq);
4477 28 : INIT_LIST_HEAD(&wq->list);
4478 :
4479 14 : if (alloc_and_link_pwqs(wq) < 0)
4480 : goto err_unreg_lockdep;
4481 :
4482 14 : if (wq_online && init_rescuer(wq) < 0)
4483 : goto err_destroy;
4484 :
4485 14 : if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4486 : goto err_destroy;
4487 :
4488 : /*
4489 : * wq_pool_mutex protects global freeze state and workqueues list.
4490 : * Grab it, adjust max_active and add the new @wq to workqueues
4491 : * list.
4492 : */
4493 14 : mutex_lock(&wq_pool_mutex);
4494 :
4495 14 : mutex_lock(&wq->mutex);
4496 28 : for_each_pwq(pwq, wq)
4497 14 : pwq_adjust_max_active(pwq);
4498 14 : mutex_unlock(&wq->mutex);
4499 :
4500 28 : list_add_tail_rcu(&wq->list, &workqueues);
4501 :
4502 14 : mutex_unlock(&wq_pool_mutex);
4503 :
4504 14 : return wq;
4505 :
4506 : err_unreg_lockdep:
4507 : wq_unregister_lockdep(wq);
4508 : wq_free_lockdep(wq);
4509 : err_free_wq:
4510 0 : free_workqueue_attrs(wq->unbound_attrs);
4511 0 : kfree(wq);
4512 0 : return NULL;
4513 : err_destroy:
4514 0 : destroy_workqueue(wq);
4515 0 : return NULL;
4516 : }
4517 : EXPORT_SYMBOL_GPL(alloc_workqueue);
4518 :
4519 : static bool pwq_busy(struct pool_workqueue *pwq)
4520 : {
4521 : int i;
4522 :
4523 0 : for (i = 0; i < WORK_NR_COLORS; i++)
4524 0 : if (pwq->nr_in_flight[i])
4525 : return true;
4526 :
4527 0 : if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
4528 : return true;
4529 0 : if (pwq->nr_active || !list_empty(&pwq->inactive_works))
4530 : return true;
4531 :
4532 : return false;
4533 : }
4534 :
4535 : /**
4536 : * destroy_workqueue - safely terminate a workqueue
4537 : * @wq: target workqueue
4538 : *
4539 : * Safely destroy a workqueue. All work currently pending will be done first.
4540 : */
4541 0 : void destroy_workqueue(struct workqueue_struct *wq)
4542 : {
4543 : struct pool_workqueue *pwq;
4544 : int node;
4545 :
4546 : /*
4547 : * Remove it from sysfs first so that sanity check failure doesn't
4548 : * lead to sysfs name conflicts.
4549 : */
4550 0 : workqueue_sysfs_unregister(wq);
4551 :
4552 : /* mark the workqueue destruction is in progress */
4553 0 : mutex_lock(&wq->mutex);
4554 0 : wq->flags |= __WQ_DESTROYING;
4555 0 : mutex_unlock(&wq->mutex);
4556 :
4557 : /* drain it before proceeding with destruction */
4558 0 : drain_workqueue(wq);
4559 :
4560 : /* kill rescuer, if sanity checks fail, leave it w/o rescuer */
4561 0 : if (wq->rescuer) {
4562 0 : struct worker *rescuer = wq->rescuer;
4563 :
4564 : /* this prevents new queueing */
4565 0 : raw_spin_lock_irq(&wq_mayday_lock);
4566 0 : wq->rescuer = NULL;
4567 0 : raw_spin_unlock_irq(&wq_mayday_lock);
4568 :
4569 : /* rescuer will empty maydays list before exiting */
4570 0 : kthread_stop(rescuer->task);
4571 0 : kfree(rescuer);
4572 : }
4573 :
4574 : /*
4575 : * Sanity checks - grab all the locks so that we wait for all
4576 : * in-flight operations which may do put_pwq().
4577 : */
4578 0 : mutex_lock(&wq_pool_mutex);
4579 0 : mutex_lock(&wq->mutex);
4580 0 : for_each_pwq(pwq, wq) {
4581 0 : raw_spin_lock_irq(&pwq->pool->lock);
4582 0 : if (WARN_ON(pwq_busy(pwq))) {
4583 0 : pr_warn("%s: %s has the following busy pwq\n",
4584 : __func__, wq->name);
4585 0 : show_pwq(pwq);
4586 0 : raw_spin_unlock_irq(&pwq->pool->lock);
4587 0 : mutex_unlock(&wq->mutex);
4588 0 : mutex_unlock(&wq_pool_mutex);
4589 0 : show_one_workqueue(wq);
4590 0 : return;
4591 : }
4592 0 : raw_spin_unlock_irq(&pwq->pool->lock);
4593 : }
4594 0 : mutex_unlock(&wq->mutex);
4595 :
4596 : /*
4597 : * wq list is used to freeze wq, remove from list after
4598 : * flushing is complete in case freeze races us.
4599 : */
4600 0 : list_del_rcu(&wq->list);
4601 0 : mutex_unlock(&wq_pool_mutex);
4602 :
4603 0 : if (!(wq->flags & WQ_UNBOUND)) {
4604 0 : wq_unregister_lockdep(wq);
4605 : /*
4606 : * The base ref is never dropped on per-cpu pwqs. Directly
4607 : * schedule RCU free.
4608 : */
4609 0 : call_rcu(&wq->rcu, rcu_free_wq);
4610 : } else {
4611 : /*
4612 : * We're the sole accessor of @wq at this point. Directly
4613 : * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4614 : * @wq will be freed when the last pwq is released.
4615 : */
4616 0 : for_each_node(node) {
4617 0 : pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4618 0 : RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4619 0 : put_pwq_unlocked(pwq);
4620 : }
4621 :
4622 : /*
4623 : * Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4624 : * put. Don't access it afterwards.
4625 : */
4626 0 : pwq = wq->dfl_pwq;
4627 0 : wq->dfl_pwq = NULL;
4628 0 : put_pwq_unlocked(pwq);
4629 : }
4630 : }
4631 : EXPORT_SYMBOL_GPL(destroy_workqueue);
4632 :
4633 : /**
4634 : * workqueue_set_max_active - adjust max_active of a workqueue
4635 : * @wq: target workqueue
4636 : * @max_active: new max_active value.
4637 : *
4638 : * Set max_active of @wq to @max_active.
4639 : *
4640 : * CONTEXT:
4641 : * Don't call from IRQ context.
4642 : */
4643 0 : void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4644 : {
4645 : struct pool_workqueue *pwq;
4646 :
4647 : /* disallow meddling with max_active for ordered workqueues */
4648 0 : if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4649 : return;
4650 :
4651 0 : max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4652 :
4653 0 : mutex_lock(&wq->mutex);
4654 :
4655 0 : wq->flags &= ~__WQ_ORDERED;
4656 0 : wq->saved_max_active = max_active;
4657 :
4658 0 : for_each_pwq(pwq, wq)
4659 0 : pwq_adjust_max_active(pwq);
4660 :
4661 0 : mutex_unlock(&wq->mutex);
4662 : }
4663 : EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4664 :
4665 : /**
4666 : * current_work - retrieve %current task's work struct
4667 : *
4668 : * Determine if %current task is a workqueue worker and what it's working on.
4669 : * Useful to find out the context that the %current task is running in.
4670 : *
4671 : * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
4672 : */
4673 0 : struct work_struct *current_work(void)
4674 : {
4675 0 : struct worker *worker = current_wq_worker();
4676 :
4677 0 : return worker ? worker->current_work : NULL;
4678 : }
4679 : EXPORT_SYMBOL(current_work);
4680 :
4681 : /**
4682 : * current_is_workqueue_rescuer - is %current workqueue rescuer?
4683 : *
4684 : * Determine whether %current is a workqueue rescuer. Can be used from
4685 : * work functions to determine whether it's being run off the rescuer task.
4686 : *
4687 : * Return: %true if %current is a workqueue rescuer. %false otherwise.
4688 : */
4689 0 : bool current_is_workqueue_rescuer(void)
4690 : {
4691 0 : struct worker *worker = current_wq_worker();
4692 :
4693 0 : return worker && worker->rescue_wq;
4694 : }
4695 :
4696 : /**
4697 : * workqueue_congested - test whether a workqueue is congested
4698 : * @cpu: CPU in question
4699 : * @wq: target workqueue
4700 : *
4701 : * Test whether @wq's cpu workqueue for @cpu is congested. There is
4702 : * no synchronization around this function and the test result is
4703 : * unreliable and only useful as advisory hints or for debugging.
4704 : *
4705 : * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4706 : * Note that both per-cpu and unbound workqueues may be associated with
4707 : * multiple pool_workqueues which have separate congested states. A
4708 : * workqueue being congested on one CPU doesn't mean the workqueue is also
4709 : * contested on other CPUs / NUMA nodes.
4710 : *
4711 : * Return:
4712 : * %true if congested, %false otherwise.
4713 : */
4714 0 : bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4715 : {
4716 : struct pool_workqueue *pwq;
4717 : bool ret;
4718 :
4719 : rcu_read_lock();
4720 0 : preempt_disable();
4721 :
4722 : if (cpu == WORK_CPU_UNBOUND)
4723 : cpu = smp_processor_id();
4724 :
4725 0 : if (!(wq->flags & WQ_UNBOUND))
4726 0 : pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4727 : else
4728 0 : pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4729 :
4730 0 : ret = !list_empty(&pwq->inactive_works);
4731 0 : preempt_enable();
4732 : rcu_read_unlock();
4733 :
4734 0 : return ret;
4735 : }
4736 : EXPORT_SYMBOL_GPL(workqueue_congested);
4737 :
4738 : /**
4739 : * work_busy - test whether a work is currently pending or running
4740 : * @work: the work to be tested
4741 : *
4742 : * Test whether @work is currently pending or running. There is no
4743 : * synchronization around this function and the test result is
4744 : * unreliable and only useful as advisory hints or for debugging.
4745 : *
4746 : * Return:
4747 : * OR'd bitmask of WORK_BUSY_* bits.
4748 : */
4749 0 : unsigned int work_busy(struct work_struct *work)
4750 : {
4751 : struct worker_pool *pool;
4752 : unsigned long flags;
4753 0 : unsigned int ret = 0;
4754 :
4755 0 : if (work_pending(work))
4756 0 : ret |= WORK_BUSY_PENDING;
4757 :
4758 : rcu_read_lock();
4759 0 : pool = get_work_pool(work);
4760 0 : if (pool) {
4761 0 : raw_spin_lock_irqsave(&pool->lock, flags);
4762 0 : if (find_worker_executing_work(pool, work))
4763 0 : ret |= WORK_BUSY_RUNNING;
4764 0 : raw_spin_unlock_irqrestore(&pool->lock, flags);
4765 : }
4766 : rcu_read_unlock();
4767 :
4768 0 : return ret;
4769 : }
4770 : EXPORT_SYMBOL_GPL(work_busy);
4771 :
4772 : /**
4773 : * set_worker_desc - set description for the current work item
4774 : * @fmt: printf-style format string
4775 : * @...: arguments for the format string
4776 : *
4777 : * This function can be called by a running work function to describe what
4778 : * the work item is about. If the worker task gets dumped, this
4779 : * information will be printed out together to help debugging. The
4780 : * description can be at most WORKER_DESC_LEN including the trailing '\0'.
4781 : */
4782 0 : void set_worker_desc(const char *fmt, ...)
4783 : {
4784 0 : struct worker *worker = current_wq_worker();
4785 : va_list args;
4786 :
4787 0 : if (worker) {
4788 0 : va_start(args, fmt);
4789 0 : vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4790 0 : va_end(args);
4791 : }
4792 0 : }
4793 : EXPORT_SYMBOL_GPL(set_worker_desc);
4794 :
4795 : /**
4796 : * print_worker_info - print out worker information and description
4797 : * @log_lvl: the log level to use when printing
4798 : * @task: target task
4799 : *
4800 : * If @task is a worker and currently executing a work item, print out the
4801 : * name of the workqueue being serviced and worker description set with
4802 : * set_worker_desc() by the currently executing work item.
4803 : *
4804 : * This function can be safely called on any task as long as the
4805 : * task_struct itself is accessible. While safe, this function isn't
4806 : * synchronized and may print out mixups or garbages of limited length.
4807 : */
4808 5 : void print_worker_info(const char *log_lvl, struct task_struct *task)
4809 : {
4810 5 : work_func_t *fn = NULL;
4811 5 : char name[WQ_NAME_LEN] = { };
4812 5 : char desc[WORKER_DESC_LEN] = { };
4813 5 : struct pool_workqueue *pwq = NULL;
4814 5 : struct workqueue_struct *wq = NULL;
4815 : struct worker *worker;
4816 :
4817 5 : if (!(task->flags & PF_WQ_WORKER))
4818 5 : return;
4819 :
4820 : /*
4821 : * This function is called without any synchronization and @task
4822 : * could be in any state. Be careful with dereferences.
4823 : */
4824 0 : worker = kthread_probe_data(task);
4825 :
4826 : /*
4827 : * Carefully copy the associated workqueue's workfn, name and desc.
4828 : * Keep the original last '\0' in case the original is garbage.
4829 : */
4830 0 : copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
4831 0 : copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
4832 0 : copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
4833 0 : copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
4834 0 : copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
4835 :
4836 0 : if (fn || name[0] || desc[0]) {
4837 0 : printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
4838 0 : if (strcmp(name, desc))
4839 0 : pr_cont(" (%s)", desc);
4840 0 : pr_cont("\n");
4841 : }
4842 : }
4843 :
4844 0 : static void pr_cont_pool_info(struct worker_pool *pool)
4845 : {
4846 0 : pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4847 0 : if (pool->node != NUMA_NO_NODE)
4848 0 : pr_cont(" node=%d", pool->node);
4849 0 : pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4850 0 : }
4851 :
4852 : struct pr_cont_work_struct {
4853 : bool comma;
4854 : work_func_t func;
4855 : long ctr;
4856 : };
4857 :
4858 0 : static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
4859 : {
4860 0 : if (!pcwsp->ctr)
4861 : goto out_record;
4862 0 : if (func == pcwsp->func) {
4863 0 : pcwsp->ctr++;
4864 0 : return;
4865 : }
4866 0 : if (pcwsp->ctr == 1)
4867 0 : pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
4868 : else
4869 0 : pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
4870 0 : pcwsp->ctr = 0;
4871 : out_record:
4872 0 : if ((long)func == -1L)
4873 : return;
4874 0 : pcwsp->comma = comma;
4875 0 : pcwsp->func = func;
4876 0 : pcwsp->ctr = 1;
4877 : }
4878 :
4879 0 : static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp)
4880 : {
4881 0 : if (work->func == wq_barrier_func) {
4882 : struct wq_barrier *barr;
4883 :
4884 0 : barr = container_of(work, struct wq_barrier, work);
4885 :
4886 0 : pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
4887 0 : pr_cont("%s BAR(%d)", comma ? "," : "",
4888 : task_pid_nr(barr->task));
4889 : } else {
4890 0 : if (!comma)
4891 0 : pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
4892 0 : pr_cont_work_flush(comma, work->func, pcwsp);
4893 : }
4894 0 : }
4895 :
4896 0 : static void show_pwq(struct pool_workqueue *pwq)
4897 : {
4898 0 : struct pr_cont_work_struct pcws = { .ctr = 0, };
4899 0 : struct worker_pool *pool = pwq->pool;
4900 : struct work_struct *work;
4901 : struct worker *worker;
4902 0 : bool has_in_flight = false, has_pending = false;
4903 : int bkt;
4904 :
4905 0 : pr_info(" pwq %d:", pool->id);
4906 0 : pr_cont_pool_info(pool);
4907 :
4908 0 : pr_cont(" active=%d/%d refcnt=%d%s\n",
4909 : pwq->nr_active, pwq->max_active, pwq->refcnt,
4910 : !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4911 :
4912 0 : hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4913 0 : if (worker->current_pwq == pwq) {
4914 : has_in_flight = true;
4915 : break;
4916 : }
4917 : }
4918 0 : if (has_in_flight) {
4919 0 : bool comma = false;
4920 :
4921 0 : pr_info(" in-flight:");
4922 0 : hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4923 0 : if (worker->current_pwq != pwq)
4924 0 : continue;
4925 :
4926 0 : pr_cont("%s %d%s:%ps", comma ? "," : "",
4927 : task_pid_nr(worker->task),
4928 : worker->rescue_wq ? "(RESCUER)" : "",
4929 : worker->current_func);
4930 0 : list_for_each_entry(work, &worker->scheduled, entry)
4931 0 : pr_cont_work(false, work, &pcws);
4932 0 : pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
4933 0 : comma = true;
4934 : }
4935 0 : pr_cont("\n");
4936 : }
4937 :
4938 0 : list_for_each_entry(work, &pool->worklist, entry) {
4939 0 : if (get_work_pwq(work) == pwq) {
4940 : has_pending = true;
4941 : break;
4942 : }
4943 : }
4944 0 : if (has_pending) {
4945 0 : bool comma = false;
4946 :
4947 0 : pr_info(" pending:");
4948 0 : list_for_each_entry(work, &pool->worklist, entry) {
4949 0 : if (get_work_pwq(work) != pwq)
4950 0 : continue;
4951 :
4952 0 : pr_cont_work(comma, work, &pcws);
4953 0 : comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4954 : }
4955 0 : pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
4956 0 : pr_cont("\n");
4957 : }
4958 :
4959 0 : if (!list_empty(&pwq->inactive_works)) {
4960 0 : bool comma = false;
4961 :
4962 0 : pr_info(" inactive:");
4963 0 : list_for_each_entry(work, &pwq->inactive_works, entry) {
4964 0 : pr_cont_work(comma, work, &pcws);
4965 0 : comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4966 : }
4967 0 : pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
4968 0 : pr_cont("\n");
4969 : }
4970 0 : }
4971 :
4972 : /**
4973 : * show_one_workqueue - dump state of specified workqueue
4974 : * @wq: workqueue whose state will be printed
4975 : */
4976 0 : void show_one_workqueue(struct workqueue_struct *wq)
4977 : {
4978 : struct pool_workqueue *pwq;
4979 0 : bool idle = true;
4980 : unsigned long flags;
4981 :
4982 0 : for_each_pwq(pwq, wq) {
4983 0 : if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
4984 : idle = false;
4985 : break;
4986 : }
4987 : }
4988 0 : if (idle) /* Nothing to print for idle workqueue */
4989 : return;
4990 :
4991 0 : pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4992 :
4993 0 : for_each_pwq(pwq, wq) {
4994 0 : raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4995 0 : if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
4996 : /*
4997 : * Defer printing to avoid deadlocks in console
4998 : * drivers that queue work while holding locks
4999 : * also taken in their write paths.
5000 : */
5001 0 : printk_deferred_enter();
5002 0 : show_pwq(pwq);
5003 0 : printk_deferred_exit();
5004 : }
5005 0 : raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
5006 : /*
5007 : * We could be printing a lot from atomic context, e.g.
5008 : * sysrq-t -> show_all_workqueues(). Avoid triggering
5009 : * hard lockup.
5010 : */
5011 : touch_nmi_watchdog();
5012 : }
5013 :
5014 : }
5015 :
5016 : /**
5017 : * show_one_worker_pool - dump state of specified worker pool
5018 : * @pool: worker pool whose state will be printed
5019 : */
5020 0 : static void show_one_worker_pool(struct worker_pool *pool)
5021 : {
5022 : struct worker *worker;
5023 0 : bool first = true;
5024 : unsigned long flags;
5025 0 : unsigned long hung = 0;
5026 :
5027 0 : raw_spin_lock_irqsave(&pool->lock, flags);
5028 0 : if (pool->nr_workers == pool->nr_idle)
5029 : goto next_pool;
5030 :
5031 : /* How long the first pending work is waiting for a worker. */
5032 0 : if (!list_empty(&pool->worklist))
5033 0 : hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;
5034 :
5035 : /*
5036 : * Defer printing to avoid deadlocks in console drivers that
5037 : * queue work while holding locks also taken in their write
5038 : * paths.
5039 : */
5040 0 : printk_deferred_enter();
5041 0 : pr_info("pool %d:", pool->id);
5042 0 : pr_cont_pool_info(pool);
5043 0 : pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
5044 0 : if (pool->manager)
5045 0 : pr_cont(" manager: %d",
5046 : task_pid_nr(pool->manager->task));
5047 0 : list_for_each_entry(worker, &pool->idle_list, entry) {
5048 0 : pr_cont(" %s%d", first ? "idle: " : "",
5049 : task_pid_nr(worker->task));
5050 0 : first = false;
5051 : }
5052 0 : pr_cont("\n");
5053 0 : printk_deferred_exit();
5054 : next_pool:
5055 0 : raw_spin_unlock_irqrestore(&pool->lock, flags);
5056 : /*
5057 : * We could be printing a lot from atomic context, e.g.
5058 : * sysrq-t -> show_all_workqueues(). Avoid triggering
5059 : * hard lockup.
5060 : */
5061 : touch_nmi_watchdog();
5062 :
5063 0 : }
5064 :
5065 : /**
5066 : * show_all_workqueues - dump workqueue state
5067 : *
5068 : * Called from a sysrq handler and prints out all busy workqueues and pools.
5069 : */
5070 0 : void show_all_workqueues(void)
5071 : {
5072 : struct workqueue_struct *wq;
5073 : struct worker_pool *pool;
5074 : int pi;
5075 :
5076 : rcu_read_lock();
5077 :
5078 0 : pr_info("Showing busy workqueues and worker pools:\n");
5079 :
5080 0 : list_for_each_entry_rcu(wq, &workqueues, list)
5081 0 : show_one_workqueue(wq);
5082 :
5083 0 : for_each_pool(pool, pi)
5084 0 : show_one_worker_pool(pool);
5085 :
5086 : rcu_read_unlock();
5087 0 : }
5088 :
5089 : /**
5090 : * show_freezable_workqueues - dump freezable workqueue state
5091 : *
5092 : * Called from try_to_freeze_tasks() and prints out all freezable workqueues
5093 : * still busy.
5094 : */
5095 0 : void show_freezable_workqueues(void)
5096 : {
5097 : struct workqueue_struct *wq;
5098 :
5099 : rcu_read_lock();
5100 :
5101 0 : pr_info("Showing freezable workqueues that are still busy:\n");
5102 :
5103 0 : list_for_each_entry_rcu(wq, &workqueues, list) {
5104 0 : if (!(wq->flags & WQ_FREEZABLE))
5105 0 : continue;
5106 0 : show_one_workqueue(wq);
5107 : }
5108 :
5109 : rcu_read_unlock();
5110 0 : }
5111 :
5112 : /* used to show worker information through /proc/PID/{comm,stat,status} */
5113 0 : void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
5114 : {
5115 : int off;
5116 :
5117 : /* always show the actual comm */
5118 0 : off = strscpy(buf, task->comm, size);
5119 0 : if (off < 0)
5120 : return;
5121 :
5122 : /* stabilize PF_WQ_WORKER and worker pool association */
5123 0 : mutex_lock(&wq_pool_attach_mutex);
5124 :
5125 0 : if (task->flags & PF_WQ_WORKER) {
5126 0 : struct worker *worker = kthread_data(task);
5127 0 : struct worker_pool *pool = worker->pool;
5128 :
5129 0 : if (pool) {
5130 0 : raw_spin_lock_irq(&pool->lock);
5131 : /*
5132 : * ->desc tracks information (wq name or
5133 : * set_worker_desc()) for the latest execution. If
5134 : * current, prepend '+', otherwise '-'.
5135 : */
5136 0 : if (worker->desc[0] != '\0') {
5137 0 : if (worker->current_work)
5138 0 : scnprintf(buf + off, size - off, "+%s",
5139 0 : worker->desc);
5140 : else
5141 0 : scnprintf(buf + off, size - off, "-%s",
5142 0 : worker->desc);
5143 : }
5144 0 : raw_spin_unlock_irq(&pool->lock);
5145 : }
5146 : }
5147 :
5148 0 : mutex_unlock(&wq_pool_attach_mutex);
5149 : }
5150 :
5151 : #ifdef CONFIG_SMP
5152 :
5153 : /*
5154 : * CPU hotplug.
5155 : *
5156 : * There are two challenges in supporting CPU hotplug. Firstly, there
5157 : * are a lot of assumptions on strong associations among work, pwq and
5158 : * pool which make migrating pending and scheduled works very
5159 : * difficult to implement without impacting hot paths. Secondly,
5160 : * worker pools serve mix of short, long and very long running works making
5161 : * blocked draining impractical.
5162 : *
5163 : * This is solved by allowing the pools to be disassociated from the CPU
5164 : * running as an unbound one and allowing it to be reattached later if the
5165 : * cpu comes back online.
5166 : */
5167 :
5168 : static void unbind_workers(int cpu)
5169 : {
5170 : struct worker_pool *pool;
5171 : struct worker *worker;
5172 :
5173 : for_each_cpu_worker_pool(pool, cpu) {
5174 : mutex_lock(&wq_pool_attach_mutex);
5175 : raw_spin_lock_irq(&pool->lock);
5176 :
5177 : /*
5178 : * We've blocked all attach/detach operations. Make all workers
5179 : * unbound and set DISASSOCIATED. Before this, all workers
5180 : * must be on the cpu. After this, they may become diasporas.
5181 : * And the preemption disabled section in their sched callbacks
5182 : * are guaranteed to see WORKER_UNBOUND since the code here
5183 : * is on the same cpu.
5184 : */
5185 : for_each_pool_worker(worker, pool)
5186 : worker->flags |= WORKER_UNBOUND;
5187 :
5188 : pool->flags |= POOL_DISASSOCIATED;
5189 :
5190 : /*
5191 : * The handling of nr_running in sched callbacks are disabled
5192 : * now. Zap nr_running. After this, nr_running stays zero and
5193 : * need_more_worker() and keep_working() are always true as
5194 : * long as the worklist is not empty. This pool now behaves as
5195 : * an unbound (in terms of concurrency management) pool which
5196 : * are served by workers tied to the pool.
5197 : */
5198 : pool->nr_running = 0;
5199 :
5200 : /*
5201 : * With concurrency management just turned off, a busy
5202 : * worker blocking could lead to lengthy stalls. Kick off
5203 : * unbound chain execution of currently pending work items.
5204 : */
5205 : wake_up_worker(pool);
5206 :
5207 : raw_spin_unlock_irq(&pool->lock);
5208 :
5209 : for_each_pool_worker(worker, pool)
5210 : unbind_worker(worker);
5211 :
5212 : mutex_unlock(&wq_pool_attach_mutex);
5213 : }
5214 : }
5215 :
5216 : /**
5217 : * rebind_workers - rebind all workers of a pool to the associated CPU
5218 : * @pool: pool of interest
5219 : *
5220 : * @pool->cpu is coming online. Rebind all workers to the CPU.
5221 : */
5222 : static void rebind_workers(struct worker_pool *pool)
5223 : {
5224 : struct worker *worker;
5225 :
5226 : lockdep_assert_held(&wq_pool_attach_mutex);
5227 :
5228 : /*
5229 : * Restore CPU affinity of all workers. As all idle workers should
5230 : * be on the run-queue of the associated CPU before any local
5231 : * wake-ups for concurrency management happen, restore CPU affinity
5232 : * of all workers first and then clear UNBOUND. As we're called
5233 : * from CPU_ONLINE, the following shouldn't fail.
5234 : */
5235 : for_each_pool_worker(worker, pool) {
5236 : kthread_set_per_cpu(worker->task, pool->cpu);
5237 : WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
5238 : pool->attrs->cpumask) < 0);
5239 : }
5240 :
5241 : raw_spin_lock_irq(&pool->lock);
5242 :
5243 : pool->flags &= ~POOL_DISASSOCIATED;
5244 :
5245 : for_each_pool_worker(worker, pool) {
5246 : unsigned int worker_flags = worker->flags;
5247 :
5248 : /*
5249 : * We want to clear UNBOUND but can't directly call
5250 : * worker_clr_flags() or adjust nr_running. Atomically
5251 : * replace UNBOUND with another NOT_RUNNING flag REBOUND.
5252 : * @worker will clear REBOUND using worker_clr_flags() when
5253 : * it initiates the next execution cycle thus restoring
5254 : * concurrency management. Note that when or whether
5255 : * @worker clears REBOUND doesn't affect correctness.
5256 : *
5257 : * WRITE_ONCE() is necessary because @worker->flags may be
5258 : * tested without holding any lock in
5259 : * wq_worker_running(). Without it, NOT_RUNNING test may
5260 : * fail incorrectly leading to premature concurrency
5261 : * management operations.
5262 : */
5263 : WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
5264 : worker_flags |= WORKER_REBOUND;
5265 : worker_flags &= ~WORKER_UNBOUND;
5266 : WRITE_ONCE(worker->flags, worker_flags);
5267 : }
5268 :
5269 : raw_spin_unlock_irq(&pool->lock);
5270 : }
5271 :
5272 : /**
5273 : * restore_unbound_workers_cpumask - restore cpumask of unbound workers
5274 : * @pool: unbound pool of interest
5275 : * @cpu: the CPU which is coming up
5276 : *
5277 : * An unbound pool may end up with a cpumask which doesn't have any online
5278 : * CPUs. When a worker of such pool get scheduled, the scheduler resets
5279 : * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
5280 : * online CPU before, cpus_allowed of all its workers should be restored.
5281 : */
5282 : static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
5283 : {
5284 : static cpumask_t cpumask;
5285 : struct worker *worker;
5286 :
5287 : lockdep_assert_held(&wq_pool_attach_mutex);
5288 :
5289 : /* is @cpu allowed for @pool? */
5290 : if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
5291 : return;
5292 :
5293 : cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
5294 :
5295 : /* as we're called from CPU_ONLINE, the following shouldn't fail */
5296 : for_each_pool_worker(worker, pool)
5297 : WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
5298 : }
5299 :
5300 : int workqueue_prepare_cpu(unsigned int cpu)
5301 : {
5302 : struct worker_pool *pool;
5303 :
5304 : for_each_cpu_worker_pool(pool, cpu) {
5305 : if (pool->nr_workers)
5306 : continue;
5307 : if (!create_worker(pool))
5308 : return -ENOMEM;
5309 : }
5310 : return 0;
5311 : }
5312 :
5313 : int workqueue_online_cpu(unsigned int cpu)
5314 : {
5315 : struct worker_pool *pool;
5316 : struct workqueue_struct *wq;
5317 : int pi;
5318 :
5319 : mutex_lock(&wq_pool_mutex);
5320 :
5321 : for_each_pool(pool, pi) {
5322 : mutex_lock(&wq_pool_attach_mutex);
5323 :
5324 : if (pool->cpu == cpu)
5325 : rebind_workers(pool);
5326 : else if (pool->cpu < 0)
5327 : restore_unbound_workers_cpumask(pool, cpu);
5328 :
5329 : mutex_unlock(&wq_pool_attach_mutex);
5330 : }
5331 :
5332 : /* update NUMA affinity of unbound workqueues */
5333 : list_for_each_entry(wq, &workqueues, list)
5334 : wq_update_unbound_numa(wq, cpu, true);
5335 :
5336 : mutex_unlock(&wq_pool_mutex);
5337 : return 0;
5338 : }
5339 :
5340 : int workqueue_offline_cpu(unsigned int cpu)
5341 : {
5342 : struct workqueue_struct *wq;
5343 :
5344 : /* unbinding per-cpu workers should happen on the local CPU */
5345 : if (WARN_ON(cpu != smp_processor_id()))
5346 : return -1;
5347 :
5348 : unbind_workers(cpu);
5349 :
5350 : /* update NUMA affinity of unbound workqueues */
5351 : mutex_lock(&wq_pool_mutex);
5352 : list_for_each_entry(wq, &workqueues, list)
5353 : wq_update_unbound_numa(wq, cpu, false);
5354 : mutex_unlock(&wq_pool_mutex);
5355 :
5356 : return 0;
5357 : }
5358 :
5359 : struct work_for_cpu {
5360 : struct work_struct work;
5361 : long (*fn)(void *);
5362 : void *arg;
5363 : long ret;
5364 : };
5365 :
5366 : static void work_for_cpu_fn(struct work_struct *work)
5367 : {
5368 : struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
5369 :
5370 : wfc->ret = wfc->fn(wfc->arg);
5371 : }
5372 :
5373 : /**
5374 : * work_on_cpu - run a function in thread context on a particular cpu
5375 : * @cpu: the cpu to run on
5376 : * @fn: the function to run
5377 : * @arg: the function arg
5378 : *
5379 : * It is up to the caller to ensure that the cpu doesn't go offline.
5380 : * The caller must not hold any locks which would prevent @fn from completing.
5381 : *
5382 : * Return: The value @fn returns.
5383 : */
5384 : long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
5385 : {
5386 : struct work_for_cpu wfc = { .fn = fn, .arg = arg };
5387 :
5388 : INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
5389 : schedule_work_on(cpu, &wfc.work);
5390 : flush_work(&wfc.work);
5391 : destroy_work_on_stack(&wfc.work);
5392 : return wfc.ret;
5393 : }
5394 : EXPORT_SYMBOL_GPL(work_on_cpu);
5395 :
5396 : /**
5397 : * work_on_cpu_safe - run a function in thread context on a particular cpu
5398 : * @cpu: the cpu to run on
5399 : * @fn: the function to run
5400 : * @arg: the function argument
5401 : *
5402 : * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
5403 : * any locks which would prevent @fn from completing.
5404 : *
5405 : * Return: The value @fn returns.
5406 : */
5407 : long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg)
5408 : {
5409 : long ret = -ENODEV;
5410 :
5411 : cpus_read_lock();
5412 : if (cpu_online(cpu))
5413 : ret = work_on_cpu(cpu, fn, arg);
5414 : cpus_read_unlock();
5415 : return ret;
5416 : }
5417 : EXPORT_SYMBOL_GPL(work_on_cpu_safe);
5418 : #endif /* CONFIG_SMP */
5419 :
5420 : #ifdef CONFIG_FREEZER
5421 :
5422 : /**
5423 : * freeze_workqueues_begin - begin freezing workqueues
5424 : *
5425 : * Start freezing workqueues. After this function returns, all freezable
5426 : * workqueues will queue new works to their inactive_works list instead of
5427 : * pool->worklist.
5428 : *
5429 : * CONTEXT:
5430 : * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5431 : */
5432 0 : void freeze_workqueues_begin(void)
5433 : {
5434 : struct workqueue_struct *wq;
5435 : struct pool_workqueue *pwq;
5436 :
5437 0 : mutex_lock(&wq_pool_mutex);
5438 :
5439 0 : WARN_ON_ONCE(workqueue_freezing);
5440 0 : workqueue_freezing = true;
5441 :
5442 0 : list_for_each_entry(wq, &workqueues, list) {
5443 0 : mutex_lock(&wq->mutex);
5444 0 : for_each_pwq(pwq, wq)
5445 0 : pwq_adjust_max_active(pwq);
5446 0 : mutex_unlock(&wq->mutex);
5447 : }
5448 :
5449 0 : mutex_unlock(&wq_pool_mutex);
5450 0 : }
5451 :
5452 : /**
5453 : * freeze_workqueues_busy - are freezable workqueues still busy?
5454 : *
5455 : * Check whether freezing is complete. This function must be called
5456 : * between freeze_workqueues_begin() and thaw_workqueues().
5457 : *
5458 : * CONTEXT:
5459 : * Grabs and releases wq_pool_mutex.
5460 : *
5461 : * Return:
5462 : * %true if some freezable workqueues are still busy. %false if freezing
5463 : * is complete.
5464 : */
5465 0 : bool freeze_workqueues_busy(void)
5466 : {
5467 0 : bool busy = false;
5468 : struct workqueue_struct *wq;
5469 : struct pool_workqueue *pwq;
5470 :
5471 0 : mutex_lock(&wq_pool_mutex);
5472 :
5473 0 : WARN_ON_ONCE(!workqueue_freezing);
5474 :
5475 0 : list_for_each_entry(wq, &workqueues, list) {
5476 0 : if (!(wq->flags & WQ_FREEZABLE))
5477 0 : continue;
5478 : /*
5479 : * nr_active is monotonically decreasing. It's safe
5480 : * to peek without lock.
5481 : */
5482 : rcu_read_lock();
5483 0 : for_each_pwq(pwq, wq) {
5484 0 : WARN_ON_ONCE(pwq->nr_active < 0);
5485 0 : if (pwq->nr_active) {
5486 0 : busy = true;
5487 : rcu_read_unlock();
5488 : goto out_unlock;
5489 : }
5490 : }
5491 : rcu_read_unlock();
5492 : }
5493 : out_unlock:
5494 0 : mutex_unlock(&wq_pool_mutex);
5495 0 : return busy;
5496 : }
5497 :
5498 : /**
5499 : * thaw_workqueues - thaw workqueues
5500 : *
5501 : * Thaw workqueues. Normal queueing is restored and all collected
5502 : * frozen works are transferred to their respective pool worklists.
5503 : *
5504 : * CONTEXT:
5505 : * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5506 : */
5507 0 : void thaw_workqueues(void)
5508 : {
5509 : struct workqueue_struct *wq;
5510 : struct pool_workqueue *pwq;
5511 :
5512 0 : mutex_lock(&wq_pool_mutex);
5513 :
5514 0 : if (!workqueue_freezing)
5515 : goto out_unlock;
5516 :
5517 0 : workqueue_freezing = false;
5518 :
5519 : /* restore max_active and repopulate worklist */
5520 0 : list_for_each_entry(wq, &workqueues, list) {
5521 0 : mutex_lock(&wq->mutex);
5522 0 : for_each_pwq(pwq, wq)
5523 0 : pwq_adjust_max_active(pwq);
5524 0 : mutex_unlock(&wq->mutex);
5525 : }
5526 :
5527 : out_unlock:
5528 0 : mutex_unlock(&wq_pool_mutex);
5529 0 : }
5530 : #endif /* CONFIG_FREEZER */
5531 :
5532 0 : static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
5533 : {
5534 0 : LIST_HEAD(ctxs);
5535 0 : int ret = 0;
5536 : struct workqueue_struct *wq;
5537 : struct apply_wqattrs_ctx *ctx, *n;
5538 :
5539 : lockdep_assert_held(&wq_pool_mutex);
5540 :
5541 0 : list_for_each_entry(wq, &workqueues, list) {
5542 0 : if (!(wq->flags & WQ_UNBOUND))
5543 0 : continue;
5544 : /* creating multiple pwqs breaks ordering guarantee */
5545 0 : if (wq->flags & __WQ_ORDERED)
5546 0 : continue;
5547 :
5548 0 : ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
5549 0 : if (!ctx) {
5550 : ret = -ENOMEM;
5551 : break;
5552 : }
5553 :
5554 0 : list_add_tail(&ctx->list, &ctxs);
5555 : }
5556 :
5557 0 : list_for_each_entry_safe(ctx, n, &ctxs, list) {
5558 0 : if (!ret)
5559 0 : apply_wqattrs_commit(ctx);
5560 0 : apply_wqattrs_cleanup(ctx);
5561 : }
5562 :
5563 0 : if (!ret) {
5564 0 : mutex_lock(&wq_pool_attach_mutex);
5565 0 : cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
5566 0 : mutex_unlock(&wq_pool_attach_mutex);
5567 : }
5568 0 : return ret;
5569 : }
5570 :
5571 : /**
5572 : * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5573 : * @cpumask: the cpumask to set
5574 : *
5575 : * The low-level workqueues cpumask is a global cpumask that limits
5576 : * the affinity of all unbound workqueues. This function check the @cpumask
5577 : * and apply it to all unbound workqueues and updates all pwqs of them.
5578 : *
5579 : * Return: 0 - Success
5580 : * -EINVAL - Invalid @cpumask
5581 : * -ENOMEM - Failed to allocate memory for attrs or pwqs.
5582 : */
5583 0 : int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
5584 : {
5585 0 : int ret = -EINVAL;
5586 :
5587 : /*
5588 : * Not excluding isolated cpus on purpose.
5589 : * If the user wishes to include them, we allow that.
5590 : */
5591 0 : cpumask_and(cpumask, cpumask, cpu_possible_mask);
5592 0 : if (!cpumask_empty(cpumask)) {
5593 0 : apply_wqattrs_lock();
5594 0 : if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
5595 : ret = 0;
5596 : goto out_unlock;
5597 : }
5598 :
5599 0 : ret = workqueue_apply_unbound_cpumask(cpumask);
5600 :
5601 : out_unlock:
5602 : apply_wqattrs_unlock();
5603 : }
5604 :
5605 0 : return ret;
5606 : }
5607 :
5608 : #ifdef CONFIG_SYSFS
5609 : /*
5610 : * Workqueues with WQ_SYSFS flag set is visible to userland via
5611 : * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
5612 : * following attributes.
5613 : *
5614 : * per_cpu RO bool : whether the workqueue is per-cpu or unbound
5615 : * max_active RW int : maximum number of in-flight work items
5616 : *
5617 : * Unbound workqueues have the following extra attributes.
5618 : *
5619 : * pool_ids RO int : the associated pool IDs for each node
5620 : * nice RW int : nice value of the workers
5621 : * cpumask RW mask : bitmask of allowed CPUs for the workers
5622 : * numa RW bool : whether enable NUMA affinity
5623 : */
5624 : struct wq_device {
5625 : struct workqueue_struct *wq;
5626 : struct device dev;
5627 : };
5628 :
5629 : static struct workqueue_struct *dev_to_wq(struct device *dev)
5630 : {
5631 0 : struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5632 :
5633 0 : return wq_dev->wq;
5634 : }
5635 :
5636 0 : static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
5637 : char *buf)
5638 : {
5639 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5640 :
5641 0 : return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
5642 : }
5643 : static DEVICE_ATTR_RO(per_cpu);
5644 :
5645 0 : static ssize_t max_active_show(struct device *dev,
5646 : struct device_attribute *attr, char *buf)
5647 : {
5648 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5649 :
5650 0 : return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
5651 : }
5652 :
5653 0 : static ssize_t max_active_store(struct device *dev,
5654 : struct device_attribute *attr, const char *buf,
5655 : size_t count)
5656 : {
5657 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5658 : int val;
5659 :
5660 0 : if (sscanf(buf, "%d", &val) != 1 || val <= 0)
5661 : return -EINVAL;
5662 :
5663 0 : workqueue_set_max_active(wq, val);
5664 0 : return count;
5665 : }
5666 : static DEVICE_ATTR_RW(max_active);
5667 :
5668 : static struct attribute *wq_sysfs_attrs[] = {
5669 : &dev_attr_per_cpu.attr,
5670 : &dev_attr_max_active.attr,
5671 : NULL,
5672 : };
5673 : ATTRIBUTE_GROUPS(wq_sysfs);
5674 :
5675 0 : static ssize_t wq_pool_ids_show(struct device *dev,
5676 : struct device_attribute *attr, char *buf)
5677 : {
5678 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5679 0 : const char *delim = "";
5680 0 : int node, written = 0;
5681 :
5682 : cpus_read_lock();
5683 : rcu_read_lock();
5684 0 : for_each_node(node) {
5685 0 : written += scnprintf(buf + written, PAGE_SIZE - written,
5686 : "%s%d:%d", delim, node,
5687 0 : unbound_pwq_by_node(wq, node)->pool->id);
5688 0 : delim = " ";
5689 : }
5690 0 : written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5691 : rcu_read_unlock();
5692 : cpus_read_unlock();
5693 :
5694 0 : return written;
5695 : }
5696 :
5697 0 : static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
5698 : char *buf)
5699 : {
5700 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5701 : int written;
5702 :
5703 0 : mutex_lock(&wq->mutex);
5704 0 : written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
5705 0 : mutex_unlock(&wq->mutex);
5706 :
5707 0 : return written;
5708 : }
5709 :
5710 : /* prepare workqueue_attrs for sysfs store operations */
5711 0 : static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
5712 : {
5713 : struct workqueue_attrs *attrs;
5714 :
5715 : lockdep_assert_held(&wq_pool_mutex);
5716 :
5717 0 : attrs = alloc_workqueue_attrs();
5718 0 : if (!attrs)
5719 : return NULL;
5720 :
5721 0 : copy_workqueue_attrs(attrs, wq->unbound_attrs);
5722 0 : return attrs;
5723 : }
5724 :
5725 0 : static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
5726 : const char *buf, size_t count)
5727 : {
5728 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5729 : struct workqueue_attrs *attrs;
5730 0 : int ret = -ENOMEM;
5731 :
5732 : apply_wqattrs_lock();
5733 :
5734 0 : attrs = wq_sysfs_prep_attrs(wq);
5735 0 : if (!attrs)
5736 : goto out_unlock;
5737 :
5738 0 : if (sscanf(buf, "%d", &attrs->nice) == 1 &&
5739 0 : attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
5740 0 : ret = apply_workqueue_attrs_locked(wq, attrs);
5741 : else
5742 : ret = -EINVAL;
5743 :
5744 : out_unlock:
5745 0 : apply_wqattrs_unlock();
5746 0 : free_workqueue_attrs(attrs);
5747 0 : return ret ?: count;
5748 : }
5749 :
5750 0 : static ssize_t wq_cpumask_show(struct device *dev,
5751 : struct device_attribute *attr, char *buf)
5752 : {
5753 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5754 : int written;
5755 :
5756 0 : mutex_lock(&wq->mutex);
5757 0 : written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5758 0 : cpumask_pr_args(wq->unbound_attrs->cpumask));
5759 0 : mutex_unlock(&wq->mutex);
5760 0 : return written;
5761 : }
5762 :
5763 0 : static ssize_t wq_cpumask_store(struct device *dev,
5764 : struct device_attribute *attr,
5765 : const char *buf, size_t count)
5766 : {
5767 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5768 : struct workqueue_attrs *attrs;
5769 0 : int ret = -ENOMEM;
5770 :
5771 : apply_wqattrs_lock();
5772 :
5773 0 : attrs = wq_sysfs_prep_attrs(wq);
5774 0 : if (!attrs)
5775 : goto out_unlock;
5776 :
5777 0 : ret = cpumask_parse(buf, attrs->cpumask);
5778 0 : if (!ret)
5779 0 : ret = apply_workqueue_attrs_locked(wq, attrs);
5780 :
5781 : out_unlock:
5782 0 : apply_wqattrs_unlock();
5783 0 : free_workqueue_attrs(attrs);
5784 0 : return ret ?: count;
5785 : }
5786 :
5787 0 : static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
5788 : char *buf)
5789 : {
5790 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5791 : int written;
5792 :
5793 0 : mutex_lock(&wq->mutex);
5794 0 : written = scnprintf(buf, PAGE_SIZE, "%d\n",
5795 0 : !wq->unbound_attrs->no_numa);
5796 0 : mutex_unlock(&wq->mutex);
5797 :
5798 0 : return written;
5799 : }
5800 :
5801 0 : static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
5802 : const char *buf, size_t count)
5803 : {
5804 0 : struct workqueue_struct *wq = dev_to_wq(dev);
5805 : struct workqueue_attrs *attrs;
5806 0 : int v, ret = -ENOMEM;
5807 :
5808 : apply_wqattrs_lock();
5809 :
5810 0 : attrs = wq_sysfs_prep_attrs(wq);
5811 0 : if (!attrs)
5812 : goto out_unlock;
5813 :
5814 0 : ret = -EINVAL;
5815 0 : if (sscanf(buf, "%d", &v) == 1) {
5816 0 : attrs->no_numa = !v;
5817 0 : ret = apply_workqueue_attrs_locked(wq, attrs);
5818 : }
5819 :
5820 : out_unlock:
5821 0 : apply_wqattrs_unlock();
5822 0 : free_workqueue_attrs(attrs);
5823 0 : return ret ?: count;
5824 : }
5825 :
5826 : static struct device_attribute wq_sysfs_unbound_attrs[] = {
5827 : __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5828 : __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5829 : __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5830 : __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5831 : __ATTR_NULL,
5832 : };
5833 :
5834 : static struct bus_type wq_subsys = {
5835 : .name = "workqueue",
5836 : .dev_groups = wq_sysfs_groups,
5837 : };
5838 :
5839 0 : static ssize_t wq_unbound_cpumask_show(struct device *dev,
5840 : struct device_attribute *attr, char *buf)
5841 : {
5842 : int written;
5843 :
5844 0 : mutex_lock(&wq_pool_mutex);
5845 0 : written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5846 : cpumask_pr_args(wq_unbound_cpumask));
5847 0 : mutex_unlock(&wq_pool_mutex);
5848 :
5849 0 : return written;
5850 : }
5851 :
5852 0 : static ssize_t wq_unbound_cpumask_store(struct device *dev,
5853 : struct device_attribute *attr, const char *buf, size_t count)
5854 : {
5855 : cpumask_var_t cpumask;
5856 : int ret;
5857 :
5858 0 : if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5859 : return -ENOMEM;
5860 :
5861 0 : ret = cpumask_parse(buf, cpumask);
5862 0 : if (!ret)
5863 0 : ret = workqueue_set_unbound_cpumask(cpumask);
5864 :
5865 0 : free_cpumask_var(cpumask);
5866 0 : return ret ? ret : count;
5867 : }
5868 :
5869 : static struct device_attribute wq_sysfs_cpumask_attr =
5870 : __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5871 : wq_unbound_cpumask_store);
5872 :
5873 1 : static int __init wq_sysfs_init(void)
5874 : {
5875 : struct device *dev_root;
5876 : int err;
5877 :
5878 1 : err = subsys_virtual_register(&wq_subsys, NULL);
5879 1 : if (err)
5880 : return err;
5881 :
5882 1 : dev_root = bus_get_dev_root(&wq_subsys);
5883 1 : if (dev_root) {
5884 1 : err = device_create_file(dev_root, &wq_sysfs_cpumask_attr);
5885 1 : put_device(dev_root);
5886 : }
5887 : return err;
5888 : }
5889 : core_initcall(wq_sysfs_init);
5890 :
5891 0 : static void wq_device_release(struct device *dev)
5892 : {
5893 0 : struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5894 :
5895 0 : kfree(wq_dev);
5896 0 : }
5897 :
5898 : /**
5899 : * workqueue_sysfs_register - make a workqueue visible in sysfs
5900 : * @wq: the workqueue to register
5901 : *
5902 : * Expose @wq in sysfs under /sys/bus/workqueue/devices.
5903 : * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5904 : * which is the preferred method.
5905 : *
5906 : * Workqueue user should use this function directly iff it wants to apply
5907 : * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5908 : * apply_workqueue_attrs() may race against userland updating the
5909 : * attributes.
5910 : *
5911 : * Return: 0 on success, -errno on failure.
5912 : */
5913 1 : int workqueue_sysfs_register(struct workqueue_struct *wq)
5914 : {
5915 : struct wq_device *wq_dev;
5916 : int ret;
5917 :
5918 : /*
5919 : * Adjusting max_active or creating new pwqs by applying
5920 : * attributes breaks ordering guarantee. Disallow exposing ordered
5921 : * workqueues.
5922 : */
5923 1 : if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
5924 : return -EINVAL;
5925 :
5926 1 : wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5927 1 : if (!wq_dev)
5928 : return -ENOMEM;
5929 :
5930 1 : wq_dev->wq = wq;
5931 1 : wq_dev->dev.bus = &wq_subsys;
5932 1 : wq_dev->dev.release = wq_device_release;
5933 1 : dev_set_name(&wq_dev->dev, "%s", wq->name);
5934 :
5935 : /*
5936 : * unbound_attrs are created separately. Suppress uevent until
5937 : * everything is ready.
5938 : */
5939 2 : dev_set_uevent_suppress(&wq_dev->dev, true);
5940 :
5941 1 : ret = device_register(&wq_dev->dev);
5942 1 : if (ret) {
5943 0 : put_device(&wq_dev->dev);
5944 0 : wq->wq_dev = NULL;
5945 0 : return ret;
5946 : }
5947 :
5948 1 : if (wq->flags & WQ_UNBOUND) {
5949 : struct device_attribute *attr;
5950 :
5951 4 : for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5952 4 : ret = device_create_file(&wq_dev->dev, attr);
5953 4 : if (ret) {
5954 0 : device_unregister(&wq_dev->dev);
5955 0 : wq->wq_dev = NULL;
5956 0 : return ret;
5957 : }
5958 : }
5959 : }
5960 :
5961 2 : dev_set_uevent_suppress(&wq_dev->dev, false);
5962 1 : kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5963 1 : return 0;
5964 : }
5965 :
5966 : /**
5967 : * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5968 : * @wq: the workqueue to unregister
5969 : *
5970 : * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
5971 : */
5972 : static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
5973 : {
5974 0 : struct wq_device *wq_dev = wq->wq_dev;
5975 :
5976 0 : if (!wq->wq_dev)
5977 : return;
5978 :
5979 0 : wq->wq_dev = NULL;
5980 0 : device_unregister(&wq_dev->dev);
5981 : }
5982 : #else /* CONFIG_SYSFS */
5983 : static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
5984 : #endif /* CONFIG_SYSFS */
5985 :
5986 : /*
5987 : * Workqueue watchdog.
5988 : *
5989 : * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5990 : * flush dependency, a concurrency managed work item which stays RUNNING
5991 : * indefinitely. Workqueue stalls can be very difficult to debug as the
5992 : * usual warning mechanisms don't trigger and internal workqueue state is
5993 : * largely opaque.
5994 : *
5995 : * Workqueue watchdog monitors all worker pools periodically and dumps
5996 : * state if some pools failed to make forward progress for a while where
5997 : * forward progress is defined as the first item on ->worklist changing.
5998 : *
5999 : * This mechanism is controlled through the kernel parameter
6000 : * "workqueue.watchdog_thresh" which can be updated at runtime through the
6001 : * corresponding sysfs parameter file.
6002 : */
6003 : #ifdef CONFIG_WQ_WATCHDOG
6004 :
6005 : static unsigned long wq_watchdog_thresh = 30;
6006 : static struct timer_list wq_watchdog_timer;
6007 :
6008 : static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
6009 : static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
6010 :
6011 : /*
6012 : * Show workers that might prevent the processing of pending work items.
6013 : * The only candidates are CPU-bound workers in the running state.
6014 : * Pending work items should be handled by another idle worker
6015 : * in all other situations.
6016 : */
6017 : static void show_cpu_pool_hog(struct worker_pool *pool)
6018 : {
6019 : struct worker *worker;
6020 : unsigned long flags;
6021 : int bkt;
6022 :
6023 : raw_spin_lock_irqsave(&pool->lock, flags);
6024 :
6025 : hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6026 : if (task_is_running(worker->task)) {
6027 : /*
6028 : * Defer printing to avoid deadlocks in console
6029 : * drivers that queue work while holding locks
6030 : * also taken in their write paths.
6031 : */
6032 : printk_deferred_enter();
6033 :
6034 : pr_info("pool %d:\n", pool->id);
6035 : sched_show_task(worker->task);
6036 :
6037 : printk_deferred_exit();
6038 : }
6039 : }
6040 :
6041 : raw_spin_unlock_irqrestore(&pool->lock, flags);
6042 : }
6043 :
6044 : static void show_cpu_pools_hogs(void)
6045 : {
6046 : struct worker_pool *pool;
6047 : int pi;
6048 :
6049 : pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");
6050 :
6051 : rcu_read_lock();
6052 :
6053 : for_each_pool(pool, pi) {
6054 : if (pool->cpu_stall)
6055 : show_cpu_pool_hog(pool);
6056 :
6057 : }
6058 :
6059 : rcu_read_unlock();
6060 : }
6061 :
6062 : static void wq_watchdog_reset_touched(void)
6063 : {
6064 : int cpu;
6065 :
6066 : wq_watchdog_touched = jiffies;
6067 : for_each_possible_cpu(cpu)
6068 : per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
6069 : }
6070 :
6071 : static void wq_watchdog_timer_fn(struct timer_list *unused)
6072 : {
6073 : unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
6074 : bool lockup_detected = false;
6075 : bool cpu_pool_stall = false;
6076 : unsigned long now = jiffies;
6077 : struct worker_pool *pool;
6078 : int pi;
6079 :
6080 : if (!thresh)
6081 : return;
6082 :
6083 : rcu_read_lock();
6084 :
6085 : for_each_pool(pool, pi) {
6086 : unsigned long pool_ts, touched, ts;
6087 :
6088 : pool->cpu_stall = false;
6089 : if (list_empty(&pool->worklist))
6090 : continue;
6091 :
6092 : /*
6093 : * If a virtual machine is stopped by the host it can look to
6094 : * the watchdog like a stall.
6095 : */
6096 : kvm_check_and_clear_guest_paused();
6097 :
6098 : /* get the latest of pool and touched timestamps */
6099 : if (pool->cpu >= 0)
6100 : touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
6101 : else
6102 : touched = READ_ONCE(wq_watchdog_touched);
6103 : pool_ts = READ_ONCE(pool->watchdog_ts);
6104 :
6105 : if (time_after(pool_ts, touched))
6106 : ts = pool_ts;
6107 : else
6108 : ts = touched;
6109 :
6110 : /* did we stall? */
6111 : if (time_after(now, ts + thresh)) {
6112 : lockup_detected = true;
6113 : if (pool->cpu >= 0) {
6114 : pool->cpu_stall = true;
6115 : cpu_pool_stall = true;
6116 : }
6117 : pr_emerg("BUG: workqueue lockup - pool");
6118 : pr_cont_pool_info(pool);
6119 : pr_cont(" stuck for %us!\n",
6120 : jiffies_to_msecs(now - pool_ts) / 1000);
6121 : }
6122 :
6123 :
6124 : }
6125 :
6126 : rcu_read_unlock();
6127 :
6128 : if (lockup_detected)
6129 : show_all_workqueues();
6130 :
6131 : if (cpu_pool_stall)
6132 : show_cpu_pools_hogs();
6133 :
6134 : wq_watchdog_reset_touched();
6135 : mod_timer(&wq_watchdog_timer, jiffies + thresh);
6136 : }
6137 :
6138 : notrace void wq_watchdog_touch(int cpu)
6139 : {
6140 : if (cpu >= 0)
6141 : per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
6142 :
6143 : wq_watchdog_touched = jiffies;
6144 : }
6145 :
6146 : static void wq_watchdog_set_thresh(unsigned long thresh)
6147 : {
6148 : wq_watchdog_thresh = 0;
6149 : del_timer_sync(&wq_watchdog_timer);
6150 :
6151 : if (thresh) {
6152 : wq_watchdog_thresh = thresh;
6153 : wq_watchdog_reset_touched();
6154 : mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
6155 : }
6156 : }
6157 :
6158 : static int wq_watchdog_param_set_thresh(const char *val,
6159 : const struct kernel_param *kp)
6160 : {
6161 : unsigned long thresh;
6162 : int ret;
6163 :
6164 : ret = kstrtoul(val, 0, &thresh);
6165 : if (ret)
6166 : return ret;
6167 :
6168 : if (system_wq)
6169 : wq_watchdog_set_thresh(thresh);
6170 : else
6171 : wq_watchdog_thresh = thresh;
6172 :
6173 : return 0;
6174 : }
6175 :
6176 : static const struct kernel_param_ops wq_watchdog_thresh_ops = {
6177 : .set = wq_watchdog_param_set_thresh,
6178 : .get = param_get_ulong,
6179 : };
6180 :
6181 : module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
6182 : 0644);
6183 :
6184 : static void wq_watchdog_init(void)
6185 : {
6186 : timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
6187 : wq_watchdog_set_thresh(wq_watchdog_thresh);
6188 : }
6189 :
6190 : #else /* CONFIG_WQ_WATCHDOG */
6191 :
6192 : static inline void wq_watchdog_init(void) { }
6193 :
6194 : #endif /* CONFIG_WQ_WATCHDOG */
6195 :
6196 : static void __init wq_numa_init(void)
6197 : {
6198 : cpumask_var_t *tbl;
6199 : int node, cpu;
6200 :
6201 1 : if (num_possible_nodes() <= 1)
6202 : return;
6203 :
6204 : if (wq_disable_numa) {
6205 : pr_info("workqueue: NUMA affinity support disabled\n");
6206 : return;
6207 : }
6208 :
6209 : for_each_possible_cpu(cpu) {
6210 : if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
6211 : pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
6212 : return;
6213 : }
6214 : }
6215 :
6216 : wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
6217 : BUG_ON(!wq_update_unbound_numa_attrs_buf);
6218 :
6219 : /*
6220 : * We want masks of possible CPUs of each node which isn't readily
6221 : * available. Build one from cpu_to_node() which should have been
6222 : * fully initialized by now.
6223 : */
6224 : tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);
6225 : BUG_ON(!tbl);
6226 :
6227 : for_each_node(node)
6228 : BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
6229 : node_online(node) ? node : NUMA_NO_NODE));
6230 :
6231 : for_each_possible_cpu(cpu) {
6232 : node = cpu_to_node(cpu);
6233 : cpumask_set_cpu(cpu, tbl[node]);
6234 : }
6235 :
6236 : wq_numa_possible_cpumask = tbl;
6237 : wq_numa_enabled = true;
6238 : }
6239 :
6240 : /**
6241 : * workqueue_init_early - early init for workqueue subsystem
6242 : *
6243 : * This is the first half of two-staged workqueue subsystem initialization
6244 : * and invoked as soon as the bare basics - memory allocation, cpumasks and
6245 : * idr are up. It sets up all the data structures and system workqueues
6246 : * and allows early boot code to create workqueues and queue/cancel work
6247 : * items. Actual work item execution starts only after kthreads can be
6248 : * created and scheduled right before early initcalls.
6249 : */
6250 1 : void __init workqueue_init_early(void)
6251 : {
6252 1 : int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
6253 : int i, cpu;
6254 :
6255 : BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
6256 :
6257 1 : BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
6258 2 : cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
6259 2 : cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
6260 :
6261 1 : pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
6262 :
6263 : /* initialize CPU pools */
6264 2 : for_each_possible_cpu(cpu) {
6265 : struct worker_pool *pool;
6266 :
6267 : i = 0;
6268 2 : for_each_cpu_worker_pool(pool, cpu) {
6269 2 : BUG_ON(init_worker_pool(pool));
6270 2 : pool->cpu = cpu;
6271 6 : cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
6272 2 : pool->attrs->nice = std_nice[i++];
6273 2 : pool->node = cpu_to_node(cpu);
6274 :
6275 : /* alloc pool ID */
6276 2 : mutex_lock(&wq_pool_mutex);
6277 2 : BUG_ON(worker_pool_assign_id(pool));
6278 2 : mutex_unlock(&wq_pool_mutex);
6279 : }
6280 : }
6281 :
6282 : /* create default unbound and ordered wq attrs */
6283 2 : for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
6284 : struct workqueue_attrs *attrs;
6285 :
6286 2 : BUG_ON(!(attrs = alloc_workqueue_attrs()));
6287 2 : attrs->nice = std_nice[i];
6288 2 : unbound_std_wq_attrs[i] = attrs;
6289 :
6290 : /*
6291 : * An ordered wq should have only one pwq as ordering is
6292 : * guaranteed by max_active which is enforced by pwqs.
6293 : * Turn off NUMA so that dfl_pwq is used for all nodes.
6294 : */
6295 2 : BUG_ON(!(attrs = alloc_workqueue_attrs()));
6296 2 : attrs->nice = std_nice[i];
6297 2 : attrs->no_numa = true;
6298 2 : ordered_wq_attrs[i] = attrs;
6299 : }
6300 :
6301 1 : system_wq = alloc_workqueue("events", 0, 0);
6302 1 : system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
6303 1 : system_long_wq = alloc_workqueue("events_long", 0, 0);
6304 1 : system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
6305 : WQ_UNBOUND_MAX_ACTIVE);
6306 1 : system_freezable_wq = alloc_workqueue("events_freezable",
6307 : WQ_FREEZABLE, 0);
6308 1 : system_power_efficient_wq = alloc_workqueue("events_power_efficient",
6309 : WQ_POWER_EFFICIENT, 0);
6310 1 : system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
6311 : WQ_FREEZABLE | WQ_POWER_EFFICIENT,
6312 : 0);
6313 1 : BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
6314 : !system_unbound_wq || !system_freezable_wq ||
6315 : !system_power_efficient_wq ||
6316 : !system_freezable_power_efficient_wq);
6317 1 : }
6318 :
6319 : /**
6320 : * workqueue_init - bring workqueue subsystem fully online
6321 : *
6322 : * This is the latter half of two-staged workqueue subsystem initialization
6323 : * and invoked as soon as kthreads can be created and scheduled.
6324 : * Workqueues have been created and work items queued on them, but there
6325 : * are no kworkers executing the work items yet. Populate the worker pools
6326 : * with the initial workers and enable future kworker creations.
6327 : */
6328 1 : void __init workqueue_init(void)
6329 : {
6330 : struct workqueue_struct *wq;
6331 : struct worker_pool *pool;
6332 : int cpu, bkt;
6333 :
6334 : /*
6335 : * It'd be simpler to initialize NUMA in workqueue_init_early() but
6336 : * CPU to node mapping may not be available that early on some
6337 : * archs such as power and arm64. As per-cpu pools created
6338 : * previously could be missing node hint and unbound pools NUMA
6339 : * affinity, fix them up.
6340 : *
6341 : * Also, while iterating workqueues, create rescuers if requested.
6342 : */
6343 : wq_numa_init();
6344 :
6345 1 : mutex_lock(&wq_pool_mutex);
6346 :
6347 2 : for_each_possible_cpu(cpu) {
6348 2 : for_each_cpu_worker_pool(pool, cpu) {
6349 2 : pool->node = cpu_to_node(cpu);
6350 : }
6351 : }
6352 :
6353 9 : list_for_each_entry(wq, &workqueues, list) {
6354 8 : wq_update_unbound_numa(wq, smp_processor_id(), true);
6355 8 : WARN(init_rescuer(wq),
6356 : "workqueue: failed to create early rescuer for %s",
6357 : wq->name);
6358 : }
6359 :
6360 1 : mutex_unlock(&wq_pool_mutex);
6361 :
6362 : /* create the initial workers */
6363 2 : for_each_online_cpu(cpu) {
6364 2 : for_each_cpu_worker_pool(pool, cpu) {
6365 2 : pool->flags &= ~POOL_DISASSOCIATED;
6366 2 : BUG_ON(!create_worker(pool));
6367 : }
6368 : }
6369 :
6370 65 : hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
6371 1 : BUG_ON(!create_worker(pool));
6372 :
6373 1 : wq_online = true;
6374 : wq_watchdog_init();
6375 1 : }
6376 :
6377 : /*
6378 : * Despite the naming, this is a no-op function which is here only for avoiding
6379 : * link error. Since compile-time warning may fail to catch, we will need to
6380 : * emit run-time warning from __flush_workqueue().
6381 : */
6382 0 : void __warn_flushing_systemwide_wq(void) { }
6383 : EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
|
