Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef _LINUX_VMALLOC_H
3 : #define _LINUX_VMALLOC_H
4 :
5 : #include <linux/spinlock.h>
6 : #include <linux/init.h>
7 : #include <linux/list.h>
8 : #include <linux/llist.h>
9 : #include <asm/page.h> /* pgprot_t */
10 : #include <linux/rbtree.h>
11 : #include <linux/overflow.h>
12 :
13 : #include <asm/vmalloc.h>
14 :
15 : struct vm_area_struct; /* vma defining user mapping in mm_types.h */
16 : struct notifier_block; /* in notifier.h */
17 : struct iov_iter; /* in uio.h */
18 :
19 : /* bits in flags of vmalloc's vm_struct below */
20 : #define VM_IOREMAP 0x00000001 /* ioremap() and friends */
21 : #define VM_ALLOC 0x00000002 /* vmalloc() */
22 : #define VM_MAP 0x00000004 /* vmap()ed pages */
23 : #define VM_USERMAP 0x00000008 /* suitable for remap_vmalloc_range */
24 : #define VM_DMA_COHERENT 0x00000010 /* dma_alloc_coherent */
25 : #define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */
26 : #define VM_NO_GUARD 0x00000040 /* ***DANGEROUS*** don't add guard page */
27 : #define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */
28 : #define VM_FLUSH_RESET_PERMS 0x00000100 /* reset direct map and flush TLB on unmap, can't be freed in atomic context */
29 : #define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */
30 : #define VM_ALLOW_HUGE_VMAP 0x00000400 /* Allow for huge pages on archs with HAVE_ARCH_HUGE_VMALLOC */
31 :
32 : #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
33 : !defined(CONFIG_KASAN_VMALLOC)
34 : #define VM_DEFER_KMEMLEAK 0x00000800 /* defer kmemleak object creation */
35 : #else
36 : #define VM_DEFER_KMEMLEAK 0
37 : #endif
38 :
39 : /* bits [20..32] reserved for arch specific ioremap internals */
40 :
41 : /*
42 : * Maximum alignment for ioremap() regions.
43 : * Can be overridden by arch-specific value.
44 : */
45 : #ifndef IOREMAP_MAX_ORDER
46 : #define IOREMAP_MAX_ORDER (7 + PAGE_SHIFT) /* 128 pages */
47 : #endif
48 :
49 : struct vm_struct {
50 : struct vm_struct *next;
51 : void *addr;
52 : unsigned long size;
53 : unsigned long flags;
54 : struct page **pages;
55 : #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
56 : unsigned int page_order;
57 : #endif
58 : unsigned int nr_pages;
59 : phys_addr_t phys_addr;
60 : const void *caller;
61 : };
62 :
63 : struct vmap_area {
64 : unsigned long va_start;
65 : unsigned long va_end;
66 :
67 : struct rb_node rb_node; /* address sorted rbtree */
68 : struct list_head list; /* address sorted list */
69 :
70 : /*
71 : * The following two variables can be packed, because
72 : * a vmap_area object can be either:
73 : * 1) in "free" tree (root is free_vmap_area_root)
74 : * 2) or "busy" tree (root is vmap_area_root)
75 : */
76 : union {
77 : unsigned long subtree_max_size; /* in "free" tree */
78 : struct vm_struct *vm; /* in "busy" tree */
79 : };
80 : unsigned long flags; /* mark type of vm_map_ram area */
81 : };
82 :
83 : /* archs that select HAVE_ARCH_HUGE_VMAP should override one or more of these */
84 : #ifndef arch_vmap_p4d_supported
85 : static inline bool arch_vmap_p4d_supported(pgprot_t prot)
86 : {
87 : return false;
88 : }
89 : #endif
90 :
91 : #ifndef arch_vmap_pud_supported
92 : static inline bool arch_vmap_pud_supported(pgprot_t prot)
93 : {
94 : return false;
95 : }
96 : #endif
97 :
98 : #ifndef arch_vmap_pmd_supported
99 : static inline bool arch_vmap_pmd_supported(pgprot_t prot)
100 : {
101 : return false;
102 : }
103 : #endif
104 :
105 : #ifndef arch_vmap_pte_range_map_size
106 : static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr, unsigned long end,
107 : u64 pfn, unsigned int max_page_shift)
108 : {
109 : return PAGE_SIZE;
110 : }
111 : #endif
112 :
113 : #ifndef arch_vmap_pte_supported_shift
114 : static inline int arch_vmap_pte_supported_shift(unsigned long size)
115 : {
116 : return PAGE_SHIFT;
117 : }
118 : #endif
119 :
120 : #ifndef arch_vmap_pgprot_tagged
121 : static inline pgprot_t arch_vmap_pgprot_tagged(pgprot_t prot)
122 : {
123 : return prot;
124 : }
125 : #endif
126 :
127 : /*
128 : * Highlevel APIs for driver use
129 : */
130 : extern void vm_unmap_ram(const void *mem, unsigned int count);
131 : extern void *vm_map_ram(struct page **pages, unsigned int count, int node);
132 : extern void vm_unmap_aliases(void);
133 :
134 : #ifdef CONFIG_MMU
135 : extern unsigned long vmalloc_nr_pages(void);
136 : #else
137 : static inline unsigned long vmalloc_nr_pages(void) { return 0; }
138 : #endif
139 :
140 : extern void *vmalloc(unsigned long size) __alloc_size(1);
141 : extern void *vzalloc(unsigned long size) __alloc_size(1);
142 : extern void *vmalloc_user(unsigned long size) __alloc_size(1);
143 : extern void *vmalloc_node(unsigned long size, int node) __alloc_size(1);
144 : extern void *vzalloc_node(unsigned long size, int node) __alloc_size(1);
145 : extern void *vmalloc_32(unsigned long size) __alloc_size(1);
146 : extern void *vmalloc_32_user(unsigned long size) __alloc_size(1);
147 : extern void *__vmalloc(unsigned long size, gfp_t gfp_mask) __alloc_size(1);
148 : extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
149 : unsigned long start, unsigned long end, gfp_t gfp_mask,
150 : pgprot_t prot, unsigned long vm_flags, int node,
151 : const void *caller) __alloc_size(1);
152 : void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask,
153 : int node, const void *caller) __alloc_size(1);
154 : void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) __alloc_size(1);
155 :
156 : extern void *__vmalloc_array(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2);
157 : extern void *vmalloc_array(size_t n, size_t size) __alloc_size(1, 2);
158 : extern void *__vcalloc(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2);
159 : extern void *vcalloc(size_t n, size_t size) __alloc_size(1, 2);
160 :
161 : extern void vfree(const void *addr);
162 : extern void vfree_atomic(const void *addr);
163 :
164 : extern void *vmap(struct page **pages, unsigned int count,
165 : unsigned long flags, pgprot_t prot);
166 : void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot);
167 : extern void vunmap(const void *addr);
168 :
169 : extern int remap_vmalloc_range_partial(struct vm_area_struct *vma,
170 : unsigned long uaddr, void *kaddr,
171 : unsigned long pgoff, unsigned long size);
172 :
173 : extern int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
174 : unsigned long pgoff);
175 :
176 : /*
177 : * Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values
178 : * and let generic vmalloc and ioremap code know when arch_sync_kernel_mappings()
179 : * needs to be called.
180 : */
181 : #ifndef ARCH_PAGE_TABLE_SYNC_MASK
182 : #define ARCH_PAGE_TABLE_SYNC_MASK 0
183 : #endif
184 :
185 : /*
186 : * There is no default implementation for arch_sync_kernel_mappings(). It is
187 : * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK
188 : * is 0.
189 : */
190 : void arch_sync_kernel_mappings(unsigned long start, unsigned long end);
191 :
192 : /*
193 : * Lowlevel-APIs (not for driver use!)
194 : */
195 :
196 : static inline size_t get_vm_area_size(const struct vm_struct *area)
197 : {
198 16 : if (!(area->flags & VM_NO_GUARD))
199 : /* return actual size without guard page */
200 16 : return area->size - PAGE_SIZE;
201 : else
202 : return area->size;
203 :
204 : }
205 :
206 : extern struct vm_struct *get_vm_area(unsigned long size, unsigned long flags);
207 : extern struct vm_struct *get_vm_area_caller(unsigned long size,
208 : unsigned long flags, const void *caller);
209 : extern struct vm_struct *__get_vm_area_caller(unsigned long size,
210 : unsigned long flags,
211 : unsigned long start, unsigned long end,
212 : const void *caller);
213 : void free_vm_area(struct vm_struct *area);
214 : extern struct vm_struct *remove_vm_area(const void *addr);
215 : extern struct vm_struct *find_vm_area(const void *addr);
216 : struct vmap_area *find_vmap_area(unsigned long addr);
217 :
218 : static inline bool is_vm_area_hugepages(const void *addr)
219 : {
220 : /*
221 : * This may not 100% tell if the area is mapped with > PAGE_SIZE
222 : * page table entries, if for some reason the architecture indicates
223 : * larger sizes are available but decides not to use them, nothing
224 : * prevents that. This only indicates the size of the physical page
225 : * allocated in the vmalloc layer.
226 : */
227 : #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
228 : return find_vm_area(addr)->page_order > 0;
229 : #else
230 : return false;
231 : #endif
232 : }
233 :
234 : #ifdef CONFIG_MMU
235 : void vunmap_range(unsigned long addr, unsigned long end);
236 : static inline void set_vm_flush_reset_perms(void *addr)
237 : {
238 : struct vm_struct *vm = find_vm_area(addr);
239 :
240 : if (vm)
241 : vm->flags |= VM_FLUSH_RESET_PERMS;
242 : }
243 :
244 : #else
245 : static inline void set_vm_flush_reset_perms(void *addr)
246 : {
247 : }
248 : #endif
249 :
250 : /* for /proc/kcore */
251 : extern long vread_iter(struct iov_iter *iter, const char *addr, size_t count);
252 :
253 : /*
254 : * Internals. Don't use..
255 : */
256 : extern struct list_head vmap_area_list;
257 : extern __init void vm_area_add_early(struct vm_struct *vm);
258 : extern __init void vm_area_register_early(struct vm_struct *vm, size_t align);
259 :
260 : #ifdef CONFIG_SMP
261 : # ifdef CONFIG_MMU
262 : struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
263 : const size_t *sizes, int nr_vms,
264 : size_t align);
265 :
266 : void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms);
267 : # else
268 : static inline struct vm_struct **
269 : pcpu_get_vm_areas(const unsigned long *offsets,
270 : const size_t *sizes, int nr_vms,
271 : size_t align)
272 : {
273 : return NULL;
274 : }
275 :
276 : static inline void
277 : pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
278 : {
279 : }
280 : # endif
281 : #endif
282 :
283 : #ifdef CONFIG_MMU
284 : #define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START)
285 : #else
286 : #define VMALLOC_TOTAL 0UL
287 : #endif
288 :
289 : int register_vmap_purge_notifier(struct notifier_block *nb);
290 : int unregister_vmap_purge_notifier(struct notifier_block *nb);
291 :
292 : #if defined(CONFIG_MMU) && defined(CONFIG_PRINTK)
293 : bool vmalloc_dump_obj(void *object);
294 : #else
295 : static inline bool vmalloc_dump_obj(void *object) { return false; }
296 : #endif
297 :
298 : #endif /* _LINUX_VMALLOC_H */
|
