Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * async.c: Asynchronous function calls for boot performance
4 : *
5 : * (C) Copyright 2009 Intel Corporation
6 : * Author: Arjan van de Ven <arjan@linux.intel.com>
7 : */
8 :
9 :
10 : /*
11 :
12 : Goals and Theory of Operation
13 :
14 : The primary goal of this feature is to reduce the kernel boot time,
15 : by doing various independent hardware delays and discovery operations
16 : decoupled and not strictly serialized.
17 :
18 : More specifically, the asynchronous function call concept allows
19 : certain operations (primarily during system boot) to happen
20 : asynchronously, out of order, while these operations still
21 : have their externally visible parts happen sequentially and in-order.
22 : (not unlike how out-of-order CPUs retire their instructions in order)
23 :
24 : Key to the asynchronous function call implementation is the concept of
25 : a "sequence cookie" (which, although it has an abstracted type, can be
26 : thought of as a monotonically incrementing number).
27 :
28 : The async core will assign each scheduled event such a sequence cookie and
29 : pass this to the called functions.
30 :
31 : The asynchronously called function should before doing a globally visible
32 : operation, such as registering device numbers, call the
33 : async_synchronize_cookie() function and pass in its own cookie. The
34 : async_synchronize_cookie() function will make sure that all asynchronous
35 : operations that were scheduled prior to the operation corresponding with the
36 : cookie have completed.
37 :
38 : Subsystem/driver initialization code that scheduled asynchronous probe
39 : functions, but which shares global resources with other drivers/subsystems
40 : that do not use the asynchronous call feature, need to do a full
41 : synchronization with the async_synchronize_full() function, before returning
42 : from their init function. This is to maintain strict ordering between the
43 : asynchronous and synchronous parts of the kernel.
44 :
45 : */
46 :
47 : #include <linux/async.h>
48 : #include <linux/atomic.h>
49 : #include <linux/ktime.h>
50 : #include <linux/export.h>
51 : #include <linux/wait.h>
52 : #include <linux/sched.h>
53 : #include <linux/slab.h>
54 : #include <linux/workqueue.h>
55 :
56 : #include "workqueue_internal.h"
57 :
58 : static async_cookie_t next_cookie = 1;
59 :
60 : #define MAX_WORK 32768
61 : #define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
62 :
63 : static LIST_HEAD(async_global_pending); /* pending from all registered doms */
64 : static ASYNC_DOMAIN(async_dfl_domain);
65 : static DEFINE_SPINLOCK(async_lock);
66 :
67 : struct async_entry {
68 : struct list_head domain_list;
69 : struct list_head global_list;
70 : struct work_struct work;
71 : async_cookie_t cookie;
72 : async_func_t func;
73 : void *data;
74 : struct async_domain *domain;
75 : };
76 :
77 : static DECLARE_WAIT_QUEUE_HEAD(async_done);
78 :
79 : static atomic_t entry_count;
80 :
81 : static long long microseconds_since(ktime_t start)
82 : {
83 : ktime_t now = ktime_get();
84 : return ktime_to_ns(ktime_sub(now, start)) >> 10;
85 : }
86 :
87 2 : static async_cookie_t lowest_in_progress(struct async_domain *domain)
88 : {
89 2 : struct async_entry *first = NULL;
90 2 : async_cookie_t ret = ASYNC_COOKIE_MAX;
91 : unsigned long flags;
92 :
93 2 : spin_lock_irqsave(&async_lock, flags);
94 :
95 2 : if (domain) {
96 0 : if (!list_empty(&domain->pending))
97 0 : first = list_first_entry(&domain->pending,
98 : struct async_entry, domain_list);
99 : } else {
100 2 : if (!list_empty(&async_global_pending))
101 0 : first = list_first_entry(&async_global_pending,
102 : struct async_entry, global_list);
103 : }
104 :
105 2 : if (first)
106 0 : ret = first->cookie;
107 :
108 2 : spin_unlock_irqrestore(&async_lock, flags);
109 2 : return ret;
110 : }
111 :
112 : /*
113 : * pick the first pending entry and run it
114 : */
115 0 : static void async_run_entry_fn(struct work_struct *work)
116 : {
117 0 : struct async_entry *entry =
118 0 : container_of(work, struct async_entry, work);
119 : unsigned long flags;
120 : ktime_t calltime;
121 :
122 : /* 1) run (and print duration) */
123 : pr_debug("calling %lli_%pS @ %i\n", (long long)entry->cookie,
124 : entry->func, task_pid_nr(current));
125 0 : calltime = ktime_get();
126 :
127 0 : entry->func(entry->data, entry->cookie);
128 :
129 : pr_debug("initcall %lli_%pS returned after %lld usecs\n",
130 : (long long)entry->cookie, entry->func,
131 : microseconds_since(calltime));
132 :
133 : /* 2) remove self from the pending queues */
134 0 : spin_lock_irqsave(&async_lock, flags);
135 0 : list_del_init(&entry->domain_list);
136 0 : list_del_init(&entry->global_list);
137 :
138 : /* 3) free the entry */
139 0 : kfree(entry);
140 0 : atomic_dec(&entry_count);
141 :
142 0 : spin_unlock_irqrestore(&async_lock, flags);
143 :
144 : /* 4) wake up any waiters */
145 0 : wake_up(&async_done);
146 0 : }
147 :
148 : /**
149 : * async_schedule_node_domain - NUMA specific version of async_schedule_domain
150 : * @func: function to execute asynchronously
151 : * @data: data pointer to pass to the function
152 : * @node: NUMA node that we want to schedule this on or close to
153 : * @domain: the domain
154 : *
155 : * Returns an async_cookie_t that may be used for checkpointing later.
156 : * @domain may be used in the async_synchronize_*_domain() functions to
157 : * wait within a certain synchronization domain rather than globally.
158 : *
159 : * Note: This function may be called from atomic or non-atomic contexts.
160 : *
161 : * The node requested will be honored on a best effort basis. If the node
162 : * has no CPUs associated with it then the work is distributed among all
163 : * available CPUs.
164 : */
165 0 : async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
166 : int node, struct async_domain *domain)
167 : {
168 : struct async_entry *entry;
169 : unsigned long flags;
170 : async_cookie_t newcookie;
171 :
172 : /* allow irq-off callers */
173 0 : entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
174 :
175 : /*
176 : * If we're out of memory or if there's too much work
177 : * pending already, we execute synchronously.
178 : */
179 0 : if (!entry || atomic_read(&entry_count) > MAX_WORK) {
180 0 : kfree(entry);
181 0 : spin_lock_irqsave(&async_lock, flags);
182 0 : newcookie = next_cookie++;
183 0 : spin_unlock_irqrestore(&async_lock, flags);
184 :
185 : /* low on memory.. run synchronously */
186 0 : func(data, newcookie);
187 0 : return newcookie;
188 : }
189 0 : INIT_LIST_HEAD(&entry->domain_list);
190 0 : INIT_LIST_HEAD(&entry->global_list);
191 0 : INIT_WORK(&entry->work, async_run_entry_fn);
192 0 : entry->func = func;
193 0 : entry->data = data;
194 0 : entry->domain = domain;
195 :
196 0 : spin_lock_irqsave(&async_lock, flags);
197 :
198 : /* allocate cookie and queue */
199 0 : newcookie = entry->cookie = next_cookie++;
200 :
201 0 : list_add_tail(&entry->domain_list, &domain->pending);
202 0 : if (domain->registered)
203 0 : list_add_tail(&entry->global_list, &async_global_pending);
204 :
205 0 : atomic_inc(&entry_count);
206 0 : spin_unlock_irqrestore(&async_lock, flags);
207 :
208 : /* schedule for execution */
209 0 : queue_work_node(node, system_unbound_wq, &entry->work);
210 :
211 0 : return newcookie;
212 : }
213 : EXPORT_SYMBOL_GPL(async_schedule_node_domain);
214 :
215 : /**
216 : * async_schedule_node - NUMA specific version of async_schedule
217 : * @func: function to execute asynchronously
218 : * @data: data pointer to pass to the function
219 : * @node: NUMA node that we want to schedule this on or close to
220 : *
221 : * Returns an async_cookie_t that may be used for checkpointing later.
222 : * Note: This function may be called from atomic or non-atomic contexts.
223 : *
224 : * The node requested will be honored on a best effort basis. If the node
225 : * has no CPUs associated with it then the work is distributed among all
226 : * available CPUs.
227 : */
228 0 : async_cookie_t async_schedule_node(async_func_t func, void *data, int node)
229 : {
230 0 : return async_schedule_node_domain(func, data, node, &async_dfl_domain);
231 : }
232 : EXPORT_SYMBOL_GPL(async_schedule_node);
233 :
234 : /**
235 : * async_synchronize_full - synchronize all asynchronous function calls
236 : *
237 : * This function waits until all asynchronous function calls have been done.
238 : */
239 2 : void async_synchronize_full(void)
240 : {
241 2 : async_synchronize_full_domain(NULL);
242 2 : }
243 : EXPORT_SYMBOL_GPL(async_synchronize_full);
244 :
245 : /**
246 : * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
247 : * @domain: the domain to synchronize
248 : *
249 : * This function waits until all asynchronous function calls for the
250 : * synchronization domain specified by @domain have been done.
251 : */
252 0 : void async_synchronize_full_domain(struct async_domain *domain)
253 : {
254 2 : async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
255 0 : }
256 : EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
257 :
258 : /**
259 : * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
260 : * @cookie: async_cookie_t to use as checkpoint
261 : * @domain: the domain to synchronize (%NULL for all registered domains)
262 : *
263 : * This function waits until all asynchronous function calls for the
264 : * synchronization domain specified by @domain submitted prior to @cookie
265 : * have been done.
266 : */
267 2 : void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
268 : {
269 : ktime_t starttime;
270 :
271 : pr_debug("async_waiting @ %i\n", task_pid_nr(current));
272 2 : starttime = ktime_get();
273 :
274 2 : wait_event(async_done, lowest_in_progress(domain) >= cookie);
275 :
276 : pr_debug("async_continuing @ %i after %lli usec\n", task_pid_nr(current),
277 : microseconds_since(starttime));
278 2 : }
279 : EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
280 :
281 : /**
282 : * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
283 : * @cookie: async_cookie_t to use as checkpoint
284 : *
285 : * This function waits until all asynchronous function calls prior to @cookie
286 : * have been done.
287 : */
288 0 : void async_synchronize_cookie(async_cookie_t cookie)
289 : {
290 0 : async_synchronize_cookie_domain(cookie, &async_dfl_domain);
291 0 : }
292 : EXPORT_SYMBOL_GPL(async_synchronize_cookie);
293 :
294 : /**
295 : * current_is_async - is %current an async worker task?
296 : *
297 : * Returns %true if %current is an async worker task.
298 : */
299 0 : bool current_is_async(void)
300 : {
301 0 : struct worker *worker = current_wq_worker();
302 :
303 0 : return worker && worker->current_func == async_run_entry_fn;
304 : }
305 : EXPORT_SYMBOL_GPL(current_is_async);
|
