Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : #define CREATE_TRACE_POINTS
3 : #include <trace/events/mmap_lock.h>
4 :
5 : #include <linux/mm.h>
6 : #include <linux/cgroup.h>
7 : #include <linux/memcontrol.h>
8 : #include <linux/mmap_lock.h>
9 : #include <linux/mutex.h>
10 : #include <linux/percpu.h>
11 : #include <linux/rcupdate.h>
12 : #include <linux/smp.h>
13 : #include <linux/trace_events.h>
14 : #include <linux/local_lock.h>
15 :
16 : EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking);
17 : EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned);
18 : EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released);
19 :
20 : #ifdef CONFIG_MEMCG
21 :
22 : /*
23 : * Our various events all share the same buffer (because we don't want or need
24 : * to allocate a set of buffers *per event type*), so we need to protect against
25 : * concurrent _reg() and _unreg() calls, and count how many _reg() calls have
26 : * been made.
27 : */
28 : static DEFINE_MUTEX(reg_lock);
29 : static int reg_refcount; /* Protected by reg_lock. */
30 :
31 : /*
32 : * Size of the buffer for memcg path names. Ignoring stack trace support,
33 : * trace_events_hist.c uses MAX_FILTER_STR_VAL for this, so we also use it.
34 : */
35 : #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL
36 :
37 : /*
38 : * How many contexts our trace events might be called in: normal, softirq, irq,
39 : * and NMI.
40 : */
41 : #define CONTEXT_COUNT 4
42 :
43 : struct memcg_path {
44 : local_lock_t lock;
45 : char __rcu *buf;
46 : local_t buf_idx;
47 : };
48 : static DEFINE_PER_CPU(struct memcg_path, memcg_paths) = {
49 : .lock = INIT_LOCAL_LOCK(lock),
50 : .buf_idx = LOCAL_INIT(0),
51 : };
52 :
53 : static char **tmp_bufs;
54 :
55 : /* Called with reg_lock held. */
56 : static void free_memcg_path_bufs(void)
57 : {
58 : struct memcg_path *memcg_path;
59 : int cpu;
60 : char **old = tmp_bufs;
61 :
62 : for_each_possible_cpu(cpu) {
63 : memcg_path = per_cpu_ptr(&memcg_paths, cpu);
64 : *(old++) = rcu_dereference_protected(memcg_path->buf,
65 : lockdep_is_held(®_lock));
66 : rcu_assign_pointer(memcg_path->buf, NULL);
67 : }
68 :
69 : /* Wait for inflight memcg_path_buf users to finish. */
70 : synchronize_rcu();
71 :
72 : old = tmp_bufs;
73 : for_each_possible_cpu(cpu) {
74 : kfree(*(old++));
75 : }
76 :
77 : kfree(tmp_bufs);
78 : tmp_bufs = NULL;
79 : }
80 :
81 : int trace_mmap_lock_reg(void)
82 : {
83 : int cpu;
84 : char *new;
85 :
86 : mutex_lock(®_lock);
87 :
88 : /* If the refcount is going 0->1, proceed with allocating buffers. */
89 : if (reg_refcount++)
90 : goto out;
91 :
92 : tmp_bufs = kmalloc_array(num_possible_cpus(), sizeof(*tmp_bufs),
93 : GFP_KERNEL);
94 : if (tmp_bufs == NULL)
95 : goto out_fail;
96 :
97 : for_each_possible_cpu(cpu) {
98 : new = kmalloc(MEMCG_PATH_BUF_SIZE * CONTEXT_COUNT, GFP_KERNEL);
99 : if (new == NULL)
100 : goto out_fail_free;
101 : rcu_assign_pointer(per_cpu_ptr(&memcg_paths, cpu)->buf, new);
102 : /* Don't need to wait for inflights, they'd have gotten NULL. */
103 : }
104 :
105 : out:
106 : mutex_unlock(®_lock);
107 : return 0;
108 :
109 : out_fail_free:
110 : free_memcg_path_bufs();
111 : out_fail:
112 : /* Since we failed, undo the earlier ref increment. */
113 : --reg_refcount;
114 :
115 : mutex_unlock(®_lock);
116 : return -ENOMEM;
117 : }
118 :
119 : void trace_mmap_lock_unreg(void)
120 : {
121 : mutex_lock(®_lock);
122 :
123 : /* If the refcount is going 1->0, proceed with freeing buffers. */
124 : if (--reg_refcount)
125 : goto out;
126 :
127 : free_memcg_path_bufs();
128 :
129 : out:
130 : mutex_unlock(®_lock);
131 : }
132 :
133 : static inline char *get_memcg_path_buf(void)
134 : {
135 : struct memcg_path *memcg_path = this_cpu_ptr(&memcg_paths);
136 : char *buf;
137 : int idx;
138 :
139 : rcu_read_lock();
140 : buf = rcu_dereference(memcg_path->buf);
141 : if (buf == NULL) {
142 : rcu_read_unlock();
143 : return NULL;
144 : }
145 : idx = local_add_return(MEMCG_PATH_BUF_SIZE, &memcg_path->buf_idx) -
146 : MEMCG_PATH_BUF_SIZE;
147 : return &buf[idx];
148 : }
149 :
150 : static inline void put_memcg_path_buf(void)
151 : {
152 : local_sub(MEMCG_PATH_BUF_SIZE, &this_cpu_ptr(&memcg_paths)->buf_idx);
153 : rcu_read_unlock();
154 : }
155 :
156 : #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
157 : do { \
158 : const char *memcg_path; \
159 : local_lock(&memcg_paths.lock); \
160 : memcg_path = get_mm_memcg_path(mm); \
161 : trace_mmap_lock_##type(mm, \
162 : memcg_path != NULL ? memcg_path : "", \
163 : ##__VA_ARGS__); \
164 : if (likely(memcg_path != NULL)) \
165 : put_memcg_path_buf(); \
166 : local_unlock(&memcg_paths.lock); \
167 : } while (0)
168 :
169 : #else /* !CONFIG_MEMCG */
170 :
171 0 : int trace_mmap_lock_reg(void)
172 : {
173 0 : return 0;
174 : }
175 :
176 0 : void trace_mmap_lock_unreg(void)
177 : {
178 0 : }
179 :
180 : #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
181 : trace_mmap_lock_##type(mm, "", ##__VA_ARGS__)
182 :
183 : #endif /* CONFIG_MEMCG */
184 :
185 : #ifdef CONFIG_TRACING
186 : #ifdef CONFIG_MEMCG
187 : /*
188 : * Write the given mm_struct's memcg path to a percpu buffer, and return a
189 : * pointer to it. If the path cannot be determined, or no buffer was available
190 : * (because the trace event is being unregistered), NULL is returned.
191 : *
192 : * Note: buffers are allocated per-cpu to avoid locking, so preemption must be
193 : * disabled by the caller before calling us, and re-enabled only after the
194 : * caller is done with the pointer.
195 : *
196 : * The caller must call put_memcg_path_buf() once the buffer is no longer
197 : * needed. This must be done while preemption is still disabled.
198 : */
199 : static const char *get_mm_memcg_path(struct mm_struct *mm)
200 : {
201 : char *buf = NULL;
202 : struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
203 :
204 : if (memcg == NULL)
205 : goto out;
206 : if (unlikely(memcg->css.cgroup == NULL))
207 : goto out_put;
208 :
209 : buf = get_memcg_path_buf();
210 : if (buf == NULL)
211 : goto out_put;
212 :
213 : cgroup_path(memcg->css.cgroup, buf, MEMCG_PATH_BUF_SIZE);
214 :
215 : out_put:
216 : css_put(&memcg->css);
217 : out:
218 : return buf;
219 : }
220 :
221 : #endif /* CONFIG_MEMCG */
222 :
223 : /*
224 : * Trace calls must be in a separate file, as otherwise there's a circular
225 : * dependency between linux/mmap_lock.h and trace/events/mmap_lock.h.
226 : */
227 :
228 : void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write)
229 : {
230 : TRACE_MMAP_LOCK_EVENT(start_locking, mm, write);
231 : }
232 : EXPORT_SYMBOL(__mmap_lock_do_trace_start_locking);
233 :
234 : void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
235 : bool success)
236 : {
237 : TRACE_MMAP_LOCK_EVENT(acquire_returned, mm, write, success);
238 : }
239 : EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_returned);
240 :
241 : void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write)
242 : {
243 : TRACE_MMAP_LOCK_EVENT(released, mm, write);
244 : }
245 : EXPORT_SYMBOL(__mmap_lock_do_trace_released);
246 : #endif /* CONFIG_TRACING */
|
