Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * mm_init.c - Memory initialisation verification and debugging
4 : *
5 : * Copyright 2008 IBM Corporation, 2008
6 : * Author Mel Gorman <mel@csn.ul.ie>
7 : *
8 : */
9 : #include <linux/kernel.h>
10 : #include <linux/init.h>
11 : #include <linux/kobject.h>
12 : #include <linux/export.h>
13 : #include <linux/memory.h>
14 : #include <linux/notifier.h>
15 : #include <linux/sched.h>
16 : #include <linux/mman.h>
17 : #include <linux/memblock.h>
18 : #include <linux/page-isolation.h>
19 : #include <linux/padata.h>
20 : #include <linux/nmi.h>
21 : #include <linux/buffer_head.h>
22 : #include <linux/kmemleak.h>
23 : #include <linux/kfence.h>
24 : #include <linux/page_ext.h>
25 : #include <linux/pti.h>
26 : #include <linux/pgtable.h>
27 : #include <linux/swap.h>
28 : #include <linux/cma.h>
29 : #include "internal.h"
30 : #include "slab.h"
31 : #include "shuffle.h"
32 :
33 : #include <asm/setup.h>
34 :
35 : #ifdef CONFIG_DEBUG_MEMORY_INIT
36 : int __meminitdata mminit_loglevel;
37 :
38 : /* The zonelists are simply reported, validation is manual. */
39 1 : void __init mminit_verify_zonelist(void)
40 : {
41 : int nid;
42 :
43 1 : if (mminit_loglevel < MMINIT_VERIFY)
44 : return;
45 :
46 0 : for_each_online_node(nid) {
47 : pg_data_t *pgdat = NODE_DATA(nid);
48 : struct zone *zone;
49 : struct zoneref *z;
50 : struct zonelist *zonelist;
51 : int i, listid, zoneid;
52 :
53 : BUILD_BUG_ON(MAX_ZONELISTS > 2);
54 0 : for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) {
55 :
56 : /* Identify the zone and nodelist */
57 0 : zoneid = i % MAX_NR_ZONES;
58 0 : listid = i / MAX_NR_ZONES;
59 0 : zonelist = &pgdat->node_zonelists[listid];
60 0 : zone = &pgdat->node_zones[zoneid];
61 0 : if (!populated_zone(zone))
62 0 : continue;
63 :
64 : /* Print information about the zonelist */
65 0 : printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ",
66 : listid > 0 ? "thisnode" : "general", nid,
67 : zone->name);
68 :
69 : /* Iterate the zonelist */
70 0 : for_each_zone_zonelist(zone, z, zonelist, zoneid)
71 0 : pr_cont("%d:%s ", zone_to_nid(zone), zone->name);
72 0 : pr_cont("\n");
73 : }
74 : }
75 : }
76 :
77 1 : void __init mminit_verify_pageflags_layout(void)
78 : {
79 : int shift, width;
80 : unsigned long or_mask, add_mask;
81 :
82 1 : shift = 8 * sizeof(unsigned long);
83 1 : width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
84 : - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
85 1 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
86 : "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
87 : SECTIONS_WIDTH,
88 : NODES_WIDTH,
89 : ZONES_WIDTH,
90 : LAST_CPUPID_WIDTH,
91 : KASAN_TAG_WIDTH,
92 : LRU_GEN_WIDTH,
93 : LRU_REFS_WIDTH,
94 : NR_PAGEFLAGS);
95 1 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
96 : "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
97 : SECTIONS_SHIFT,
98 : NODES_SHIFT,
99 : ZONES_SHIFT,
100 : LAST_CPUPID_SHIFT,
101 : KASAN_TAG_WIDTH);
102 1 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",
103 : "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",
104 : (unsigned long)SECTIONS_PGSHIFT,
105 : (unsigned long)NODES_PGSHIFT,
106 : (unsigned long)ZONES_PGSHIFT,
107 : (unsigned long)LAST_CPUPID_PGSHIFT,
108 : (unsigned long)KASAN_TAG_PGSHIFT);
109 1 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",
110 : "Node/Zone ID: %lu -> %lu\n",
111 : (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),
112 : (unsigned long)ZONEID_PGOFF);
113 1 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage",
114 : "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",
115 : shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0);
116 : #ifdef NODE_NOT_IN_PAGE_FLAGS
117 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
118 : "Node not in page flags");
119 : #endif
120 : #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
121 : mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
122 : "Last cpupid not in page flags");
123 : #endif
124 :
125 : if (SECTIONS_WIDTH) {
126 : shift -= SECTIONS_WIDTH;
127 : BUG_ON(shift != SECTIONS_PGSHIFT);
128 : }
129 : if (NODES_WIDTH) {
130 : shift -= NODES_WIDTH;
131 : BUG_ON(shift != NODES_PGSHIFT);
132 : }
133 : if (ZONES_WIDTH) {
134 1 : shift -= ZONES_WIDTH;
135 : BUG_ON(shift != ZONES_PGSHIFT);
136 : }
137 :
138 : /* Check for bitmask overlaps */
139 1 : or_mask = (ZONES_MASK << ZONES_PGSHIFT) |
140 : (NODES_MASK << NODES_PGSHIFT) |
141 : (SECTIONS_MASK << SECTIONS_PGSHIFT);
142 1 : add_mask = (ZONES_MASK << ZONES_PGSHIFT) +
143 : (NODES_MASK << NODES_PGSHIFT) +
144 : (SECTIONS_MASK << SECTIONS_PGSHIFT);
145 : BUG_ON(or_mask != add_mask);
146 1 : }
147 :
148 0 : static __init int set_mminit_loglevel(char *str)
149 : {
150 0 : get_option(&str, &mminit_loglevel);
151 0 : return 0;
152 : }
153 : early_param("mminit_loglevel", set_mminit_loglevel);
154 : #endif /* CONFIG_DEBUG_MEMORY_INIT */
155 :
156 : struct kobject *mm_kobj;
157 : EXPORT_SYMBOL_GPL(mm_kobj);
158 :
159 : #ifdef CONFIG_SMP
160 : s32 vm_committed_as_batch = 32;
161 :
162 : void mm_compute_batch(int overcommit_policy)
163 : {
164 : u64 memsized_batch;
165 : s32 nr = num_present_cpus();
166 : s32 batch = max_t(s32, nr*2, 32);
167 : unsigned long ram_pages = totalram_pages();
168 :
169 : /*
170 : * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of
171 : * (total memory/#cpus), and lift it to 25% for other policies
172 : * to easy the possible lock contention for percpu_counter
173 : * vm_committed_as, while the max limit is INT_MAX
174 : */
175 : if (overcommit_policy == OVERCOMMIT_NEVER)
176 : memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX);
177 : else
178 : memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX);
179 :
180 : vm_committed_as_batch = max_t(s32, memsized_batch, batch);
181 : }
182 :
183 : static int __meminit mm_compute_batch_notifier(struct notifier_block *self,
184 : unsigned long action, void *arg)
185 : {
186 : switch (action) {
187 : case MEM_ONLINE:
188 : case MEM_OFFLINE:
189 : mm_compute_batch(sysctl_overcommit_memory);
190 : break;
191 : default:
192 : break;
193 : }
194 : return NOTIFY_OK;
195 : }
196 :
197 : static int __init mm_compute_batch_init(void)
198 : {
199 : mm_compute_batch(sysctl_overcommit_memory);
200 : hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI);
201 : return 0;
202 : }
203 :
204 : __initcall(mm_compute_batch_init);
205 :
206 : #endif
207 :
208 1 : static int __init mm_sysfs_init(void)
209 : {
210 1 : mm_kobj = kobject_create_and_add("mm", kernel_kobj);
211 1 : if (!mm_kobj)
212 : return -ENOMEM;
213 :
214 1 : return 0;
215 : }
216 : postcore_initcall(mm_sysfs_init);
217 :
218 : static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
219 : static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
220 : static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
221 :
222 : static unsigned long required_kernelcore __initdata;
223 : static unsigned long required_kernelcore_percent __initdata;
224 : static unsigned long required_movablecore __initdata;
225 : static unsigned long required_movablecore_percent __initdata;
226 :
227 : static unsigned long nr_kernel_pages __initdata;
228 : static unsigned long nr_all_pages __initdata;
229 : static unsigned long dma_reserve __initdata;
230 :
231 : static bool deferred_struct_pages __meminitdata;
232 :
233 : static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
234 :
235 0 : static int __init cmdline_parse_core(char *p, unsigned long *core,
236 : unsigned long *percent)
237 : {
238 : unsigned long long coremem;
239 : char *endptr;
240 :
241 0 : if (!p)
242 : return -EINVAL;
243 :
244 : /* Value may be a percentage of total memory, otherwise bytes */
245 0 : coremem = simple_strtoull(p, &endptr, 0);
246 0 : if (*endptr == '%') {
247 : /* Paranoid check for percent values greater than 100 */
248 0 : WARN_ON(coremem > 100);
249 :
250 0 : *percent = coremem;
251 : } else {
252 0 : coremem = memparse(p, &p);
253 : /* Paranoid check that UL is enough for the coremem value */
254 0 : WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
255 :
256 0 : *core = coremem >> PAGE_SHIFT;
257 0 : *percent = 0UL;
258 : }
259 : return 0;
260 : }
261 :
262 : bool mirrored_kernelcore __initdata_memblock;
263 :
264 : /*
265 : * kernelcore=size sets the amount of memory for use for allocations that
266 : * cannot be reclaimed or migrated.
267 : */
268 0 : static int __init cmdline_parse_kernelcore(char *p)
269 : {
270 : /* parse kernelcore=mirror */
271 0 : if (parse_option_str(p, "mirror")) {
272 0 : mirrored_kernelcore = true;
273 0 : return 0;
274 : }
275 :
276 0 : return cmdline_parse_core(p, &required_kernelcore,
277 : &required_kernelcore_percent);
278 : }
279 : early_param("kernelcore", cmdline_parse_kernelcore);
280 :
281 : /*
282 : * movablecore=size sets the amount of memory for use for allocations that
283 : * can be reclaimed or migrated.
284 : */
285 0 : static int __init cmdline_parse_movablecore(char *p)
286 : {
287 0 : return cmdline_parse_core(p, &required_movablecore,
288 : &required_movablecore_percent);
289 : }
290 : early_param("movablecore", cmdline_parse_movablecore);
291 :
292 : /*
293 : * early_calculate_totalpages()
294 : * Sum pages in active regions for movable zone.
295 : * Populate N_MEMORY for calculating usable_nodes.
296 : */
297 1 : static unsigned long __init early_calculate_totalpages(void)
298 : {
299 1 : unsigned long totalpages = 0;
300 : unsigned long start_pfn, end_pfn;
301 : int i, nid;
302 :
303 2 : for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
304 1 : unsigned long pages = end_pfn - start_pfn;
305 :
306 1 : totalpages += pages;
307 : if (pages)
308 : node_set_state(nid, N_MEMORY);
309 : }
310 1 : return totalpages;
311 : }
312 :
313 : /*
314 : * This finds a zone that can be used for ZONE_MOVABLE pages. The
315 : * assumption is made that zones within a node are ordered in monotonic
316 : * increasing memory addresses so that the "highest" populated zone is used
317 : */
318 1 : static void __init find_usable_zone_for_movable(void)
319 : {
320 : int zone_index;
321 2 : for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
322 2 : if (zone_index == ZONE_MOVABLE)
323 1 : continue;
324 :
325 2 : if (arch_zone_highest_possible_pfn[zone_index] >
326 1 : arch_zone_lowest_possible_pfn[zone_index])
327 : break;
328 : }
329 :
330 : VM_BUG_ON(zone_index == -1);
331 1 : movable_zone = zone_index;
332 1 : }
333 :
334 : /*
335 : * Find the PFN the Movable zone begins in each node. Kernel memory
336 : * is spread evenly between nodes as long as the nodes have enough
337 : * memory. When they don't, some nodes will have more kernelcore than
338 : * others
339 : */
340 1 : static void __init find_zone_movable_pfns_for_nodes(void)
341 : {
342 : int i, nid;
343 : unsigned long usable_startpfn;
344 : unsigned long kernelcore_node, kernelcore_remaining;
345 : /* save the state before borrow the nodemask */
346 1 : nodemask_t saved_node_state = node_states[N_MEMORY];
347 1 : unsigned long totalpages = early_calculate_totalpages();
348 1 : int usable_nodes = nodes_weight(node_states[N_MEMORY]);
349 : struct memblock_region *r;
350 :
351 : /* Need to find movable_zone earlier when movable_node is specified. */
352 1 : find_usable_zone_for_movable();
353 :
354 : /*
355 : * If movable_node is specified, ignore kernelcore and movablecore
356 : * options.
357 : */
358 : if (movable_node_is_enabled()) {
359 : for_each_mem_region(r) {
360 : if (!memblock_is_hotpluggable(r))
361 : continue;
362 :
363 : nid = memblock_get_region_node(r);
364 :
365 : usable_startpfn = PFN_DOWN(r->base);
366 : zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
367 : min(usable_startpfn, zone_movable_pfn[nid]) :
368 : usable_startpfn;
369 : }
370 :
371 : goto out2;
372 : }
373 :
374 : /*
375 : * If kernelcore=mirror is specified, ignore movablecore option
376 : */
377 1 : if (mirrored_kernelcore) {
378 0 : bool mem_below_4gb_not_mirrored = false;
379 :
380 0 : for_each_mem_region(r) {
381 0 : if (memblock_is_mirror(r))
382 0 : continue;
383 :
384 0 : nid = memblock_get_region_node(r);
385 :
386 0 : usable_startpfn = memblock_region_memory_base_pfn(r);
387 :
388 0 : if (usable_startpfn < PHYS_PFN(SZ_4G)) {
389 0 : mem_below_4gb_not_mirrored = true;
390 0 : continue;
391 : }
392 :
393 0 : zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
394 0 : min(usable_startpfn, zone_movable_pfn[nid]) :
395 : usable_startpfn;
396 : }
397 :
398 0 : if (mem_below_4gb_not_mirrored)
399 0 : pr_warn("This configuration results in unmirrored kernel memory.\n");
400 :
401 : goto out2;
402 : }
403 :
404 : /*
405 : * If kernelcore=nn% or movablecore=nn% was specified, calculate the
406 : * amount of necessary memory.
407 : */
408 1 : if (required_kernelcore_percent)
409 0 : required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
410 : 10000UL;
411 1 : if (required_movablecore_percent)
412 0 : required_movablecore = (totalpages * 100 * required_movablecore_percent) /
413 : 10000UL;
414 :
415 : /*
416 : * If movablecore= was specified, calculate what size of
417 : * kernelcore that corresponds so that memory usable for
418 : * any allocation type is evenly spread. If both kernelcore
419 : * and movablecore are specified, then the value of kernelcore
420 : * will be used for required_kernelcore if it's greater than
421 : * what movablecore would have allowed.
422 : */
423 1 : if (required_movablecore) {
424 : unsigned long corepages;
425 :
426 : /*
427 : * Round-up so that ZONE_MOVABLE is at least as large as what
428 : * was requested by the user
429 : */
430 : required_movablecore =
431 0 : roundup(required_movablecore, MAX_ORDER_NR_PAGES);
432 0 : required_movablecore = min(totalpages, required_movablecore);
433 0 : corepages = totalpages - required_movablecore;
434 :
435 0 : required_kernelcore = max(required_kernelcore, corepages);
436 : }
437 :
438 : /*
439 : * If kernelcore was not specified or kernelcore size is larger
440 : * than totalpages, there is no ZONE_MOVABLE.
441 : */
442 1 : if (!required_kernelcore || required_kernelcore >= totalpages)
443 : goto out;
444 :
445 : /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
446 0 : usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
447 :
448 : restart:
449 : /* Spread kernelcore memory as evenly as possible throughout nodes */
450 0 : kernelcore_node = required_kernelcore / usable_nodes;
451 0 : for_each_node_state(nid, N_MEMORY) {
452 : unsigned long start_pfn, end_pfn;
453 :
454 : /*
455 : * Recalculate kernelcore_node if the division per node
456 : * now exceeds what is necessary to satisfy the requested
457 : * amount of memory for the kernel
458 : */
459 0 : if (required_kernelcore < kernelcore_node)
460 0 : kernelcore_node = required_kernelcore / usable_nodes;
461 :
462 : /*
463 : * As the map is walked, we track how much memory is usable
464 : * by the kernel using kernelcore_remaining. When it is
465 : * 0, the rest of the node is usable by ZONE_MOVABLE
466 : */
467 0 : kernelcore_remaining = kernelcore_node;
468 :
469 : /* Go through each range of PFNs within this node */
470 0 : for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
471 : unsigned long size_pages;
472 :
473 0 : start_pfn = max(start_pfn, zone_movable_pfn[nid]);
474 0 : if (start_pfn >= end_pfn)
475 0 : continue;
476 :
477 : /* Account for what is only usable for kernelcore */
478 0 : if (start_pfn < usable_startpfn) {
479 : unsigned long kernel_pages;
480 0 : kernel_pages = min(end_pfn, usable_startpfn)
481 : - start_pfn;
482 :
483 0 : kernelcore_remaining -= min(kernel_pages,
484 : kernelcore_remaining);
485 0 : required_kernelcore -= min(kernel_pages,
486 : required_kernelcore);
487 :
488 : /* Continue if range is now fully accounted */
489 0 : if (end_pfn <= usable_startpfn) {
490 :
491 : /*
492 : * Push zone_movable_pfn to the end so
493 : * that if we have to rebalance
494 : * kernelcore across nodes, we will
495 : * not double account here
496 : */
497 0 : zone_movable_pfn[nid] = end_pfn;
498 0 : continue;
499 : }
500 0 : start_pfn = usable_startpfn;
501 : }
502 :
503 : /*
504 : * The usable PFN range for ZONE_MOVABLE is from
505 : * start_pfn->end_pfn. Calculate size_pages as the
506 : * number of pages used as kernelcore
507 : */
508 0 : size_pages = end_pfn - start_pfn;
509 0 : if (size_pages > kernelcore_remaining)
510 0 : size_pages = kernelcore_remaining;
511 0 : zone_movable_pfn[nid] = start_pfn + size_pages;
512 :
513 : /*
514 : * Some kernelcore has been met, update counts and
515 : * break if the kernelcore for this node has been
516 : * satisfied
517 : */
518 0 : required_kernelcore -= min(required_kernelcore,
519 : size_pages);
520 0 : kernelcore_remaining -= size_pages;
521 0 : if (!kernelcore_remaining)
522 : break;
523 : }
524 : }
525 :
526 : /*
527 : * If there is still required_kernelcore, we do another pass with one
528 : * less node in the count. This will push zone_movable_pfn[nid] further
529 : * along on the nodes that still have memory until kernelcore is
530 : * satisfied
531 : */
532 0 : usable_nodes--;
533 0 : if (usable_nodes && required_kernelcore > usable_nodes)
534 : goto restart;
535 :
536 : out2:
537 : /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
538 0 : for (nid = 0; nid < MAX_NUMNODES; nid++) {
539 : unsigned long start_pfn, end_pfn;
540 :
541 0 : zone_movable_pfn[nid] =
542 0 : roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
543 :
544 0 : get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
545 0 : if (zone_movable_pfn[nid] >= end_pfn)
546 0 : zone_movable_pfn[nid] = 0;
547 : }
548 :
549 : out:
550 : /* restore the node_state */
551 1 : node_states[N_MEMORY] = saved_node_state;
552 1 : }
553 :
554 265447 : static void __meminit __init_single_page(struct page *page, unsigned long pfn,
555 : unsigned long zone, int nid)
556 : {
557 265447 : mm_zero_struct_page(page);
558 530894 : set_page_links(page, zone, nid, pfn);
559 265447 : init_page_count(page);
560 265447 : page_mapcount_reset(page);
561 265447 : page_cpupid_reset_last(page);
562 265447 : page_kasan_tag_reset(page);
563 :
564 530894 : INIT_LIST_HEAD(&page->lru);
565 : #ifdef WANT_PAGE_VIRTUAL
566 : /* The shift won't overflow because ZONE_NORMAL is below 4G. */
567 : if (!is_highmem_idx(zone))
568 : set_page_address(page, __va(pfn << PAGE_SHIFT));
569 : #endif
570 265447 : }
571 :
572 : #ifdef CONFIG_NUMA
573 : /*
574 : * During memory init memblocks map pfns to nids. The search is expensive and
575 : * this caches recent lookups. The implementation of __early_pfn_to_nid
576 : * treats start/end as pfns.
577 : */
578 : struct mminit_pfnnid_cache {
579 : unsigned long last_start;
580 : unsigned long last_end;
581 : int last_nid;
582 : };
583 :
584 : static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
585 :
586 : /*
587 : * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
588 : */
589 : static int __meminit __early_pfn_to_nid(unsigned long pfn,
590 : struct mminit_pfnnid_cache *state)
591 : {
592 : unsigned long start_pfn, end_pfn;
593 : int nid;
594 :
595 : if (state->last_start <= pfn && pfn < state->last_end)
596 : return state->last_nid;
597 :
598 : nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
599 : if (nid != NUMA_NO_NODE) {
600 : state->last_start = start_pfn;
601 : state->last_end = end_pfn;
602 : state->last_nid = nid;
603 : }
604 :
605 : return nid;
606 : }
607 :
608 : int __meminit early_pfn_to_nid(unsigned long pfn)
609 : {
610 : static DEFINE_SPINLOCK(early_pfn_lock);
611 : int nid;
612 :
613 : spin_lock(&early_pfn_lock);
614 : nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
615 : if (nid < 0)
616 : nid = first_online_node;
617 : spin_unlock(&early_pfn_lock);
618 :
619 : return nid;
620 : }
621 :
622 : int hashdist = HASHDIST_DEFAULT;
623 :
624 : static int __init set_hashdist(char *str)
625 : {
626 : if (!str)
627 : return 0;
628 : hashdist = simple_strtoul(str, &str, 0);
629 : return 1;
630 : }
631 : __setup("hashdist=", set_hashdist);
632 :
633 : static inline void fixup_hashdist(void)
634 : {
635 : if (num_node_state(N_MEMORY) == 1)
636 : hashdist = 0;
637 : }
638 : #else
639 : static inline void fixup_hashdist(void) {}
640 : #endif /* CONFIG_NUMA */
641 :
642 : #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
643 : static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
644 : {
645 : pgdat->first_deferred_pfn = ULONG_MAX;
646 : }
647 :
648 : /* Returns true if the struct page for the pfn is initialised */
649 : static inline bool __meminit early_page_initialised(unsigned long pfn, int nid)
650 : {
651 : if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
652 : return false;
653 :
654 : return true;
655 : }
656 :
657 : /*
658 : * Returns true when the remaining initialisation should be deferred until
659 : * later in the boot cycle when it can be parallelised.
660 : */
661 : static bool __meminit
662 : defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
663 : {
664 : static unsigned long prev_end_pfn, nr_initialised;
665 :
666 : if (early_page_ext_enabled())
667 : return false;
668 : /*
669 : * prev_end_pfn static that contains the end of previous zone
670 : * No need to protect because called very early in boot before smp_init.
671 : */
672 : if (prev_end_pfn != end_pfn) {
673 : prev_end_pfn = end_pfn;
674 : nr_initialised = 0;
675 : }
676 :
677 : /* Always populate low zones for address-constrained allocations */
678 : if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
679 : return false;
680 :
681 : if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
682 : return true;
683 : /*
684 : * We start only with one section of pages, more pages are added as
685 : * needed until the rest of deferred pages are initialized.
686 : */
687 : nr_initialised++;
688 : if ((nr_initialised > PAGES_PER_SECTION) &&
689 : (pfn & (PAGES_PER_SECTION - 1)) == 0) {
690 : NODE_DATA(nid)->first_deferred_pfn = pfn;
691 : return true;
692 : }
693 : return false;
694 : }
695 :
696 : static void __meminit init_reserved_page(unsigned long pfn, int nid)
697 : {
698 : pg_data_t *pgdat;
699 : int zid;
700 :
701 : if (early_page_initialised(pfn, nid))
702 : return;
703 :
704 : pgdat = NODE_DATA(nid);
705 :
706 : for (zid = 0; zid < MAX_NR_ZONES; zid++) {
707 : struct zone *zone = &pgdat->node_zones[zid];
708 :
709 : if (zone_spans_pfn(zone, pfn))
710 : break;
711 : }
712 : __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
713 : }
714 : #else
715 : static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
716 :
717 : static inline bool early_page_initialised(unsigned long pfn, int nid)
718 : {
719 : return true;
720 : }
721 :
722 : static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
723 : {
724 : return false;
725 : }
726 :
727 : static inline void init_reserved_page(unsigned long pfn, int nid)
728 : {
729 : }
730 : #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
731 :
732 : /*
733 : * Initialised pages do not have PageReserved set. This function is
734 : * called for each range allocated by the bootmem allocator and
735 : * marks the pages PageReserved. The remaining valid pages are later
736 : * sent to the buddy page allocator.
737 : */
738 13 : void __meminit reserve_bootmem_region(phys_addr_t start,
739 : phys_addr_t end, int nid)
740 : {
741 13 : unsigned long start_pfn = PFN_DOWN(start);
742 13 : unsigned long end_pfn = PFN_UP(end);
743 :
744 10723 : for (; start_pfn < end_pfn; start_pfn++) {
745 10710 : if (pfn_valid(start_pfn)) {
746 10710 : struct page *page = pfn_to_page(start_pfn);
747 :
748 10710 : init_reserved_page(start_pfn, nid);
749 :
750 : /* Avoid false-positive PageTail() */
751 21420 : INIT_LIST_HEAD(&page->lru);
752 :
753 : /*
754 : * no need for atomic set_bit because the struct
755 : * page is not visible yet so nobody should
756 : * access it yet.
757 : */
758 : __SetPageReserved(page);
759 : }
760 : }
761 13 : }
762 :
763 : /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
764 : static bool __meminit
765 265447 : overlap_memmap_init(unsigned long zone, unsigned long *pfn)
766 : {
767 : static struct memblock_region *r;
768 :
769 265447 : if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
770 0 : if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
771 0 : for_each_mem_region(r) {
772 0 : if (*pfn < memblock_region_memory_end_pfn(r))
773 : break;
774 : }
775 : }
776 0 : if (*pfn >= memblock_region_memory_base_pfn(r) &&
777 0 : memblock_is_mirror(r)) {
778 0 : *pfn = memblock_region_memory_end_pfn(r);
779 0 : return true;
780 : }
781 : }
782 : return false;
783 : }
784 :
785 : /*
786 : * Only struct pages that correspond to ranges defined by memblock.memory
787 : * are zeroed and initialized by going through __init_single_page() during
788 : * memmap_init_zone_range().
789 : *
790 : * But, there could be struct pages that correspond to holes in
791 : * memblock.memory. This can happen because of the following reasons:
792 : * - physical memory bank size is not necessarily the exact multiple of the
793 : * arbitrary section size
794 : * - early reserved memory may not be listed in memblock.memory
795 : * - memory layouts defined with memmap= kernel parameter may not align
796 : * nicely with memmap sections
797 : *
798 : * Explicitly initialize those struct pages so that:
799 : * - PG_Reserved is set
800 : * - zone and node links point to zone and node that span the page if the
801 : * hole is in the middle of a zone
802 : * - zone and node links point to adjacent zone/node if the hole falls on
803 : * the zone boundary; the pages in such holes will be prepended to the
804 : * zone/node above the hole except for the trailing pages in the last
805 : * section that will be appended to the zone/node below.
806 : */
807 1 : static void __init init_unavailable_range(unsigned long spfn,
808 : unsigned long epfn,
809 : int zone, int node)
810 : {
811 : unsigned long pfn;
812 1 : u64 pgcnt = 0;
813 :
814 1 : for (pfn = spfn; pfn < epfn; pfn++) {
815 0 : if (!pfn_valid(pageblock_start_pfn(pfn))) {
816 0 : pfn = pageblock_end_pfn(pfn) - 1;
817 0 : continue;
818 : }
819 0 : __init_single_page(pfn_to_page(pfn), pfn, zone, node);
820 0 : __SetPageReserved(pfn_to_page(pfn));
821 0 : pgcnt++;
822 : }
823 :
824 1 : if (pgcnt)
825 0 : pr_info("On node %d, zone %s: %lld pages in unavailable ranges",
826 : node, zone_names[zone], pgcnt);
827 1 : }
828 :
829 : /*
830 : * Initially all pages are reserved - free ones are freed
831 : * up by memblock_free_all() once the early boot process is
832 : * done. Non-atomic initialization, single-pass.
833 : *
834 : * All aligned pageblocks are initialized to the specified migratetype
835 : * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
836 : * zone stats (e.g., nr_isolate_pageblock) are touched.
837 : */
838 1 : void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,
839 : unsigned long start_pfn, unsigned long zone_end_pfn,
840 : enum meminit_context context,
841 : struct vmem_altmap *altmap, int migratetype)
842 : {
843 1 : unsigned long pfn, end_pfn = start_pfn + size;
844 : struct page *page;
845 :
846 1 : if (highest_memmap_pfn < end_pfn - 1)
847 1 : highest_memmap_pfn = end_pfn - 1;
848 :
849 : #ifdef CONFIG_ZONE_DEVICE
850 : /*
851 : * Honor reservation requested by the driver for this ZONE_DEVICE
852 : * memory. We limit the total number of pages to initialize to just
853 : * those that might contain the memory mapping. We will defer the
854 : * ZONE_DEVICE page initialization until after we have released
855 : * the hotplug lock.
856 : */
857 : if (zone == ZONE_DEVICE) {
858 : if (!altmap)
859 : return;
860 :
861 : if (start_pfn == altmap->base_pfn)
862 : start_pfn += altmap->reserve;
863 : end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
864 : }
865 : #endif
866 :
867 265449 : for (pfn = start_pfn; pfn < end_pfn; ) {
868 : /*
869 : * There can be holes in boot-time mem_map[]s handed to this
870 : * function. They do not exist on hotplugged memory.
871 : */
872 265447 : if (context == MEMINIT_EARLY) {
873 265447 : if (overlap_memmap_init(zone, &pfn))
874 0 : continue;
875 : if (defer_init(nid, pfn, zone_end_pfn)) {
876 : deferred_struct_pages = true;
877 : break;
878 : }
879 : }
880 :
881 265447 : page = pfn_to_page(pfn);
882 265447 : __init_single_page(page, pfn, zone, nid);
883 265447 : if (context == MEMINIT_HOTPLUG)
884 : __SetPageReserved(page);
885 :
886 : /*
887 : * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
888 : * such that unmovable allocations won't be scattered all
889 : * over the place during system boot.
890 : */
891 265447 : if (pageblock_aligned(pfn)) {
892 260 : set_pageblock_migratetype(page, migratetype);
893 260 : cond_resched();
894 : }
895 265447 : pfn++;
896 : }
897 1 : }
898 :
899 1 : static void __init memmap_init_zone_range(struct zone *zone,
900 : unsigned long start_pfn,
901 : unsigned long end_pfn,
902 : unsigned long *hole_pfn)
903 : {
904 1 : unsigned long zone_start_pfn = zone->zone_start_pfn;
905 1 : unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
906 1 : int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
907 :
908 1 : start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
909 1 : end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
910 :
911 1 : if (start_pfn >= end_pfn)
912 : return;
913 :
914 1 : memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
915 : zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
916 :
917 1 : if (*hole_pfn < start_pfn)
918 0 : init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
919 :
920 1 : *hole_pfn = end_pfn;
921 : }
922 :
923 1 : static void __init memmap_init(void)
924 : {
925 : unsigned long start_pfn, end_pfn;
926 1 : unsigned long hole_pfn = 0;
927 1 : int i, j, zone_id = 0, nid;
928 :
929 2 : for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
930 : struct pglist_data *node = NODE_DATA(nid);
931 :
932 2 : for (j = 0; j < MAX_NR_ZONES; j++) {
933 2 : struct zone *zone = node->node_zones + j;
934 :
935 2 : if (!populated_zone(zone))
936 1 : continue;
937 :
938 1 : memmap_init_zone_range(zone, start_pfn, end_pfn,
939 : &hole_pfn);
940 1 : zone_id = j;
941 : }
942 : }
943 :
944 : #ifdef CONFIG_SPARSEMEM
945 : /*
946 : * Initialize the memory map for hole in the range [memory_end,
947 : * section_end].
948 : * Append the pages in this hole to the highest zone in the last
949 : * node.
950 : * The call to init_unavailable_range() is outside the ifdef to
951 : * silence the compiler warining about zone_id set but not used;
952 : * for FLATMEM it is a nop anyway
953 : */
954 : end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
955 : if (hole_pfn < end_pfn)
956 : #endif
957 1 : init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
958 1 : }
959 :
960 : #ifdef CONFIG_ZONE_DEVICE
961 : static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
962 : unsigned long zone_idx, int nid,
963 : struct dev_pagemap *pgmap)
964 : {
965 :
966 : __init_single_page(page, pfn, zone_idx, nid);
967 :
968 : /*
969 : * Mark page reserved as it will need to wait for onlining
970 : * phase for it to be fully associated with a zone.
971 : *
972 : * We can use the non-atomic __set_bit operation for setting
973 : * the flag as we are still initializing the pages.
974 : */
975 : __SetPageReserved(page);
976 :
977 : /*
978 : * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
979 : * and zone_device_data. It is a bug if a ZONE_DEVICE page is
980 : * ever freed or placed on a driver-private list.
981 : */
982 : page->pgmap = pgmap;
983 : page->zone_device_data = NULL;
984 :
985 : /*
986 : * Mark the block movable so that blocks are reserved for
987 : * movable at startup. This will force kernel allocations
988 : * to reserve their blocks rather than leaking throughout
989 : * the address space during boot when many long-lived
990 : * kernel allocations are made.
991 : *
992 : * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
993 : * because this is done early in section_activate()
994 : */
995 : if (pageblock_aligned(pfn)) {
996 : set_pageblock_migratetype(page, MIGRATE_MOVABLE);
997 : cond_resched();
998 : }
999 :
1000 : /*
1001 : * ZONE_DEVICE pages are released directly to the driver page allocator
1002 : * which will set the page count to 1 when allocating the page.
1003 : */
1004 : if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
1005 : pgmap->type == MEMORY_DEVICE_COHERENT)
1006 : set_page_count(page, 0);
1007 : }
1008 :
1009 : /*
1010 : * With compound page geometry and when struct pages are stored in ram most
1011 : * tail pages are reused. Consequently, the amount of unique struct pages to
1012 : * initialize is a lot smaller that the total amount of struct pages being
1013 : * mapped. This is a paired / mild layering violation with explicit knowledge
1014 : * of how the sparse_vmemmap internals handle compound pages in the lack
1015 : * of an altmap. See vmemmap_populate_compound_pages().
1016 : */
1017 : static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,
1018 : struct dev_pagemap *pgmap)
1019 : {
1020 : if (!vmemmap_can_optimize(altmap, pgmap))
1021 : return pgmap_vmemmap_nr(pgmap);
1022 :
1023 : return 2 * (PAGE_SIZE / sizeof(struct page));
1024 : }
1025 :
1026 : static void __ref memmap_init_compound(struct page *head,
1027 : unsigned long head_pfn,
1028 : unsigned long zone_idx, int nid,
1029 : struct dev_pagemap *pgmap,
1030 : unsigned long nr_pages)
1031 : {
1032 : unsigned long pfn, end_pfn = head_pfn + nr_pages;
1033 : unsigned int order = pgmap->vmemmap_shift;
1034 :
1035 : __SetPageHead(head);
1036 : for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {
1037 : struct page *page = pfn_to_page(pfn);
1038 :
1039 : __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
1040 : prep_compound_tail(head, pfn - head_pfn);
1041 : set_page_count(page, 0);
1042 :
1043 : /*
1044 : * The first tail page stores important compound page info.
1045 : * Call prep_compound_head() after the first tail page has
1046 : * been initialized, to not have the data overwritten.
1047 : */
1048 : if (pfn == head_pfn + 1)
1049 : prep_compound_head(head, order);
1050 : }
1051 : }
1052 :
1053 : void __ref memmap_init_zone_device(struct zone *zone,
1054 : unsigned long start_pfn,
1055 : unsigned long nr_pages,
1056 : struct dev_pagemap *pgmap)
1057 : {
1058 : unsigned long pfn, end_pfn = start_pfn + nr_pages;
1059 : struct pglist_data *pgdat = zone->zone_pgdat;
1060 : struct vmem_altmap *altmap = pgmap_altmap(pgmap);
1061 : unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);
1062 : unsigned long zone_idx = zone_idx(zone);
1063 : unsigned long start = jiffies;
1064 : int nid = pgdat->node_id;
1065 :
1066 : if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
1067 : return;
1068 :
1069 : /*
1070 : * The call to memmap_init should have already taken care
1071 : * of the pages reserved for the memmap, so we can just jump to
1072 : * the end of that region and start processing the device pages.
1073 : */
1074 : if (altmap) {
1075 : start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
1076 : nr_pages = end_pfn - start_pfn;
1077 : }
1078 :
1079 : for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
1080 : struct page *page = pfn_to_page(pfn);
1081 :
1082 : __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
1083 :
1084 : if (pfns_per_compound == 1)
1085 : continue;
1086 :
1087 : memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
1088 : compound_nr_pages(altmap, pgmap));
1089 : }
1090 :
1091 : pr_debug("%s initialised %lu pages in %ums\n", __func__,
1092 : nr_pages, jiffies_to_msecs(jiffies - start));
1093 : }
1094 : #endif
1095 :
1096 : /*
1097 : * The zone ranges provided by the architecture do not include ZONE_MOVABLE
1098 : * because it is sized independent of architecture. Unlike the other zones,
1099 : * the starting point for ZONE_MOVABLE is not fixed. It may be different
1100 : * in each node depending on the size of each node and how evenly kernelcore
1101 : * is distributed. This helper function adjusts the zone ranges
1102 : * provided by the architecture for a given node by using the end of the
1103 : * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
1104 : * zones within a node are in order of monotonic increases memory addresses
1105 : */
1106 2 : static void __init adjust_zone_range_for_zone_movable(int nid,
1107 : unsigned long zone_type,
1108 : unsigned long node_start_pfn,
1109 : unsigned long node_end_pfn,
1110 : unsigned long *zone_start_pfn,
1111 : unsigned long *zone_end_pfn)
1112 : {
1113 : /* Only adjust if ZONE_MOVABLE is on this node */
1114 2 : if (zone_movable_pfn[nid]) {
1115 : /* Size ZONE_MOVABLE */
1116 0 : if (zone_type == ZONE_MOVABLE) {
1117 0 : *zone_start_pfn = zone_movable_pfn[nid];
1118 0 : *zone_end_pfn = min(node_end_pfn,
1119 : arch_zone_highest_possible_pfn[movable_zone]);
1120 :
1121 : /* Adjust for ZONE_MOVABLE starting within this range */
1122 0 : } else if (!mirrored_kernelcore &&
1123 0 : *zone_start_pfn < zone_movable_pfn[nid] &&
1124 0 : *zone_end_pfn > zone_movable_pfn[nid]) {
1125 0 : *zone_end_pfn = zone_movable_pfn[nid];
1126 :
1127 : /* Check if this whole range is within ZONE_MOVABLE */
1128 0 : } else if (*zone_start_pfn >= zone_movable_pfn[nid])
1129 0 : *zone_start_pfn = *zone_end_pfn;
1130 : }
1131 2 : }
1132 :
1133 : /*
1134 : * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
1135 : * then all holes in the requested range will be accounted for.
1136 : */
1137 1 : unsigned long __init __absent_pages_in_range(int nid,
1138 : unsigned long range_start_pfn,
1139 : unsigned long range_end_pfn)
1140 : {
1141 1 : unsigned long nr_absent = range_end_pfn - range_start_pfn;
1142 : unsigned long start_pfn, end_pfn;
1143 : int i;
1144 :
1145 2 : for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
1146 1 : start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
1147 1 : end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
1148 1 : nr_absent -= end_pfn - start_pfn;
1149 : }
1150 1 : return nr_absent;
1151 : }
1152 :
1153 : /**
1154 : * absent_pages_in_range - Return number of page frames in holes within a range
1155 : * @start_pfn: The start PFN to start searching for holes
1156 : * @end_pfn: The end PFN to stop searching for holes
1157 : *
1158 : * Return: the number of pages frames in memory holes within a range.
1159 : */
1160 0 : unsigned long __init absent_pages_in_range(unsigned long start_pfn,
1161 : unsigned long end_pfn)
1162 : {
1163 0 : return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
1164 : }
1165 :
1166 : /* Return the number of page frames in holes in a zone on a node */
1167 2 : static unsigned long __init zone_absent_pages_in_node(int nid,
1168 : unsigned long zone_type,
1169 : unsigned long zone_start_pfn,
1170 : unsigned long zone_end_pfn)
1171 : {
1172 : unsigned long nr_absent;
1173 :
1174 : /* zone is empty, we don't have any absent pages */
1175 2 : if (zone_start_pfn == zone_end_pfn)
1176 : return 0;
1177 :
1178 1 : nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
1179 :
1180 : /*
1181 : * ZONE_MOVABLE handling.
1182 : * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
1183 : * and vice versa.
1184 : */
1185 1 : if (mirrored_kernelcore && zone_movable_pfn[nid]) {
1186 : unsigned long start_pfn, end_pfn;
1187 : struct memblock_region *r;
1188 :
1189 0 : for_each_mem_region(r) {
1190 0 : start_pfn = clamp(memblock_region_memory_base_pfn(r),
1191 : zone_start_pfn, zone_end_pfn);
1192 0 : end_pfn = clamp(memblock_region_memory_end_pfn(r),
1193 : zone_start_pfn, zone_end_pfn);
1194 :
1195 0 : if (zone_type == ZONE_MOVABLE &&
1196 0 : memblock_is_mirror(r))
1197 0 : nr_absent += end_pfn - start_pfn;
1198 :
1199 0 : if (zone_type == ZONE_NORMAL &&
1200 0 : !memblock_is_mirror(r))
1201 0 : nr_absent += end_pfn - start_pfn;
1202 : }
1203 : }
1204 :
1205 : return nr_absent;
1206 : }
1207 :
1208 : /*
1209 : * Return the number of pages a zone spans in a node, including holes
1210 : * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
1211 : */
1212 2 : static unsigned long __init zone_spanned_pages_in_node(int nid,
1213 : unsigned long zone_type,
1214 : unsigned long node_start_pfn,
1215 : unsigned long node_end_pfn,
1216 : unsigned long *zone_start_pfn,
1217 : unsigned long *zone_end_pfn)
1218 : {
1219 2 : unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
1220 2 : unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
1221 :
1222 : /* Get the start and end of the zone */
1223 2 : *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
1224 2 : *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
1225 2 : adjust_zone_range_for_zone_movable(nid, zone_type,
1226 : node_start_pfn, node_end_pfn,
1227 : zone_start_pfn, zone_end_pfn);
1228 :
1229 : /* Check that this node has pages within the zone's required range */
1230 2 : if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
1231 : return 0;
1232 :
1233 : /* Move the zone boundaries inside the node if necessary */
1234 2 : *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
1235 2 : *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
1236 :
1237 : /* Return the spanned pages */
1238 2 : return *zone_end_pfn - *zone_start_pfn;
1239 : }
1240 :
1241 0 : static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat)
1242 : {
1243 : struct zone *z;
1244 :
1245 0 : for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) {
1246 0 : z->zone_start_pfn = 0;
1247 0 : z->spanned_pages = 0;
1248 0 : z->present_pages = 0;
1249 : #if defined(CONFIG_MEMORY_HOTPLUG)
1250 : z->present_early_pages = 0;
1251 : #endif
1252 : }
1253 :
1254 0 : pgdat->node_spanned_pages = 0;
1255 0 : pgdat->node_present_pages = 0;
1256 : pr_debug("On node %d totalpages: 0\n", pgdat->node_id);
1257 0 : }
1258 :
1259 1 : static void __init calculate_node_totalpages(struct pglist_data *pgdat,
1260 : unsigned long node_start_pfn,
1261 : unsigned long node_end_pfn)
1262 : {
1263 1 : unsigned long realtotalpages = 0, totalpages = 0;
1264 : enum zone_type i;
1265 :
1266 3 : for (i = 0; i < MAX_NR_ZONES; i++) {
1267 2 : struct zone *zone = pgdat->node_zones + i;
1268 : unsigned long zone_start_pfn, zone_end_pfn;
1269 : unsigned long spanned, absent;
1270 : unsigned long real_size;
1271 :
1272 2 : spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
1273 : node_start_pfn,
1274 : node_end_pfn,
1275 : &zone_start_pfn,
1276 : &zone_end_pfn);
1277 2 : absent = zone_absent_pages_in_node(pgdat->node_id, i,
1278 : zone_start_pfn,
1279 : zone_end_pfn);
1280 :
1281 2 : real_size = spanned - absent;
1282 :
1283 2 : if (spanned)
1284 1 : zone->zone_start_pfn = zone_start_pfn;
1285 : else
1286 1 : zone->zone_start_pfn = 0;
1287 2 : zone->spanned_pages = spanned;
1288 2 : zone->present_pages = real_size;
1289 : #if defined(CONFIG_MEMORY_HOTPLUG)
1290 : zone->present_early_pages = real_size;
1291 : #endif
1292 :
1293 2 : totalpages += spanned;
1294 2 : realtotalpages += real_size;
1295 : }
1296 :
1297 1 : pgdat->node_spanned_pages = totalpages;
1298 1 : pgdat->node_present_pages = realtotalpages;
1299 : pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
1300 1 : }
1301 :
1302 : static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
1303 : unsigned long present_pages)
1304 : {
1305 2 : unsigned long pages = spanned_pages;
1306 :
1307 : /*
1308 : * Provide a more accurate estimation if there are holes within
1309 : * the zone and SPARSEMEM is in use. If there are holes within the
1310 : * zone, each populated memory region may cost us one or two extra
1311 : * memmap pages due to alignment because memmap pages for each
1312 : * populated regions may not be naturally aligned on page boundary.
1313 : * So the (present_pages >> 4) heuristic is a tradeoff for that.
1314 : */
1315 : if (spanned_pages > present_pages + (present_pages >> 4) &&
1316 : IS_ENABLED(CONFIG_SPARSEMEM))
1317 : pages = present_pages;
1318 :
1319 2 : return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
1320 : }
1321 :
1322 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1323 : static void pgdat_init_split_queue(struct pglist_data *pgdat)
1324 : {
1325 : struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
1326 :
1327 : spin_lock_init(&ds_queue->split_queue_lock);
1328 : INIT_LIST_HEAD(&ds_queue->split_queue);
1329 : ds_queue->split_queue_len = 0;
1330 : }
1331 : #else
1332 : static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
1333 : #endif
1334 :
1335 : #ifdef CONFIG_COMPACTION
1336 : static void pgdat_init_kcompactd(struct pglist_data *pgdat)
1337 : {
1338 1 : init_waitqueue_head(&pgdat->kcompactd_wait);
1339 : }
1340 : #else
1341 : static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
1342 : #endif
1343 :
1344 1 : static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
1345 : {
1346 : int i;
1347 :
1348 1 : pgdat_resize_init(pgdat);
1349 1 : pgdat_kswapd_lock_init(pgdat);
1350 :
1351 1 : pgdat_init_split_queue(pgdat);
1352 1 : pgdat_init_kcompactd(pgdat);
1353 :
1354 1 : init_waitqueue_head(&pgdat->kswapd_wait);
1355 1 : init_waitqueue_head(&pgdat->pfmemalloc_wait);
1356 :
1357 5 : for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
1358 4 : init_waitqueue_head(&pgdat->reclaim_wait[i]);
1359 :
1360 1 : pgdat_page_ext_init(pgdat);
1361 1 : lruvec_init(&pgdat->__lruvec);
1362 1 : }
1363 :
1364 2 : static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
1365 : unsigned long remaining_pages)
1366 : {
1367 4 : atomic_long_set(&zone->managed_pages, remaining_pages);
1368 2 : zone_set_nid(zone, nid);
1369 2 : zone->name = zone_names[idx];
1370 2 : zone->zone_pgdat = NODE_DATA(nid);
1371 2 : spin_lock_init(&zone->lock);
1372 2 : zone_seqlock_init(zone);
1373 2 : zone_pcp_init(zone);
1374 2 : }
1375 :
1376 1 : static void __meminit zone_init_free_lists(struct zone *zone)
1377 : {
1378 : unsigned int order, t;
1379 45 : for_each_migratetype_order(order, t) {
1380 88 : INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
1381 44 : zone->free_area[order].nr_free = 0;
1382 : }
1383 :
1384 : #ifdef CONFIG_UNACCEPTED_MEMORY
1385 : INIT_LIST_HEAD(&zone->unaccepted_pages);
1386 : #endif
1387 1 : }
1388 :
1389 1 : void __meminit init_currently_empty_zone(struct zone *zone,
1390 : unsigned long zone_start_pfn,
1391 : unsigned long size)
1392 : {
1393 1 : struct pglist_data *pgdat = zone->zone_pgdat;
1394 1 : int zone_idx = zone_idx(zone) + 1;
1395 :
1396 1 : if (zone_idx > pgdat->nr_zones)
1397 1 : pgdat->nr_zones = zone_idx;
1398 :
1399 1 : zone->zone_start_pfn = zone_start_pfn;
1400 :
1401 1 : mminit_dprintk(MMINIT_TRACE, "memmap_init",
1402 : "Initialising map node %d zone %lu pfns %lu -> %lu\n",
1403 : pgdat->node_id,
1404 : (unsigned long)zone_idx(zone),
1405 : zone_start_pfn, (zone_start_pfn + size));
1406 :
1407 1 : zone_init_free_lists(zone);
1408 1 : zone->initialized = 1;
1409 1 : }
1410 :
1411 : #ifndef CONFIG_SPARSEMEM
1412 : /*
1413 : * Calculate the size of the zone->blockflags rounded to an unsigned long
1414 : * Start by making sure zonesize is a multiple of pageblock_order by rounding
1415 : * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
1416 : * round what is now in bits to nearest long in bits, then return it in
1417 : * bytes.
1418 : */
1419 1 : static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
1420 : {
1421 : unsigned long usemapsize;
1422 :
1423 1 : zonesize += zone_start_pfn & (pageblock_nr_pages-1);
1424 1 : usemapsize = roundup(zonesize, pageblock_nr_pages);
1425 1 : usemapsize = usemapsize >> pageblock_order;
1426 1 : usemapsize *= NR_PAGEBLOCK_BITS;
1427 1 : usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
1428 :
1429 1 : return usemapsize / 8;
1430 : }
1431 :
1432 1 : static void __ref setup_usemap(struct zone *zone)
1433 : {
1434 1 : unsigned long usemapsize = usemap_size(zone->zone_start_pfn,
1435 : zone->spanned_pages);
1436 1 : zone->pageblock_flags = NULL;
1437 1 : if (usemapsize) {
1438 1 : zone->pageblock_flags =
1439 2 : memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
1440 : zone_to_nid(zone));
1441 1 : if (!zone->pageblock_flags)
1442 0 : panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
1443 : usemapsize, zone->name, zone_to_nid(zone));
1444 : }
1445 1 : }
1446 : #else
1447 : static inline void setup_usemap(struct zone *zone) {}
1448 : #endif /* CONFIG_SPARSEMEM */
1449 :
1450 : #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
1451 :
1452 : /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
1453 : void __init set_pageblock_order(void)
1454 : {
1455 : unsigned int order = MAX_ORDER;
1456 :
1457 : /* Check that pageblock_nr_pages has not already been setup */
1458 : if (pageblock_order)
1459 : return;
1460 :
1461 : /* Don't let pageblocks exceed the maximum allocation granularity. */
1462 : if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order)
1463 : order = HUGETLB_PAGE_ORDER;
1464 :
1465 : /*
1466 : * Assume the largest contiguous order of interest is a huge page.
1467 : * This value may be variable depending on boot parameters on IA64 and
1468 : * powerpc.
1469 : */
1470 : pageblock_order = order;
1471 : }
1472 : #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1473 :
1474 : /*
1475 : * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
1476 : * is unused as pageblock_order is set at compile-time. See
1477 : * include/linux/pageblock-flags.h for the values of pageblock_order based on
1478 : * the kernel config
1479 : */
1480 0 : void __init set_pageblock_order(void)
1481 : {
1482 0 : }
1483 :
1484 : #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1485 :
1486 : /*
1487 : * Set up the zone data structures
1488 : * - init pgdat internals
1489 : * - init all zones belonging to this node
1490 : *
1491 : * NOTE: this function is only called during memory hotplug
1492 : */
1493 : #ifdef CONFIG_MEMORY_HOTPLUG
1494 : void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
1495 : {
1496 : int nid = pgdat->node_id;
1497 : enum zone_type z;
1498 : int cpu;
1499 :
1500 : pgdat_init_internals(pgdat);
1501 :
1502 : if (pgdat->per_cpu_nodestats == &boot_nodestats)
1503 : pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);
1504 :
1505 : /*
1506 : * Reset the nr_zones, order and highest_zoneidx before reuse.
1507 : * Note that kswapd will init kswapd_highest_zoneidx properly
1508 : * when it starts in the near future.
1509 : */
1510 : pgdat->nr_zones = 0;
1511 : pgdat->kswapd_order = 0;
1512 : pgdat->kswapd_highest_zoneidx = 0;
1513 : pgdat->node_start_pfn = 0;
1514 : pgdat->node_present_pages = 0;
1515 :
1516 : for_each_online_cpu(cpu) {
1517 : struct per_cpu_nodestat *p;
1518 :
1519 : p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
1520 : memset(p, 0, sizeof(*p));
1521 : }
1522 :
1523 : /*
1524 : * When memory is hot-added, all the memory is in offline state. So
1525 : * clear all zones' present_pages and managed_pages because they will
1526 : * be updated in online_pages() and offline_pages().
1527 : */
1528 : for (z = 0; z < MAX_NR_ZONES; z++) {
1529 : struct zone *zone = pgdat->node_zones + z;
1530 :
1531 : zone->present_pages = 0;
1532 : zone_init_internals(zone, z, nid, 0);
1533 : }
1534 : }
1535 : #endif
1536 :
1537 : /*
1538 : * Set up the zone data structures:
1539 : * - mark all pages reserved
1540 : * - mark all memory queues empty
1541 : * - clear the memory bitmaps
1542 : *
1543 : * NOTE: pgdat should get zeroed by caller.
1544 : * NOTE: this function is only called during early init.
1545 : */
1546 1 : static void __init free_area_init_core(struct pglist_data *pgdat)
1547 : {
1548 : enum zone_type j;
1549 1 : int nid = pgdat->node_id;
1550 :
1551 1 : pgdat_init_internals(pgdat);
1552 1 : pgdat->per_cpu_nodestats = &boot_nodestats;
1553 :
1554 3 : for (j = 0; j < MAX_NR_ZONES; j++) {
1555 2 : struct zone *zone = pgdat->node_zones + j;
1556 : unsigned long size, freesize, memmap_pages;
1557 :
1558 2 : size = zone->spanned_pages;
1559 2 : freesize = zone->present_pages;
1560 :
1561 : /*
1562 : * Adjust freesize so that it accounts for how much memory
1563 : * is used by this zone for memmap. This affects the watermark
1564 : * and per-cpu initialisations
1565 : */
1566 4 : memmap_pages = calc_memmap_size(size, freesize);
1567 2 : if (!is_highmem_idx(j)) {
1568 2 : if (freesize >= memmap_pages) {
1569 2 : freesize -= memmap_pages;
1570 : if (memmap_pages)
1571 : pr_debug(" %s zone: %lu pages used for memmap\n",
1572 : zone_names[j], memmap_pages);
1573 : } else
1574 0 : pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n",
1575 : zone_names[j], memmap_pages, freesize);
1576 : }
1577 :
1578 : /* Account for reserved pages */
1579 2 : if (j == 0 && freesize > dma_reserve) {
1580 1 : freesize -= dma_reserve;
1581 : pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve);
1582 : }
1583 :
1584 2 : if (!is_highmem_idx(j))
1585 2 : nr_kernel_pages += freesize;
1586 : /* Charge for highmem memmap if there are enough kernel pages */
1587 : else if (nr_kernel_pages > memmap_pages * 2)
1588 : nr_kernel_pages -= memmap_pages;
1589 2 : nr_all_pages += freesize;
1590 :
1591 : /*
1592 : * Set an approximate value for lowmem here, it will be adjusted
1593 : * when the bootmem allocator frees pages into the buddy system.
1594 : * And all highmem pages will be managed by the buddy system.
1595 : */
1596 2 : zone_init_internals(zone, j, nid, freesize);
1597 :
1598 2 : if (!size)
1599 1 : continue;
1600 :
1601 1 : setup_usemap(zone);
1602 1 : init_currently_empty_zone(zone, zone->zone_start_pfn, size);
1603 : }
1604 1 : }
1605 :
1606 1 : void __init *memmap_alloc(phys_addr_t size, phys_addr_t align,
1607 : phys_addr_t min_addr, int nid, bool exact_nid)
1608 : {
1609 : void *ptr;
1610 :
1611 1 : if (exact_nid)
1612 0 : ptr = memblock_alloc_exact_nid_raw(size, align, min_addr,
1613 : MEMBLOCK_ALLOC_ACCESSIBLE,
1614 : nid);
1615 : else
1616 1 : ptr = memblock_alloc_try_nid_raw(size, align, min_addr,
1617 : MEMBLOCK_ALLOC_ACCESSIBLE,
1618 : nid);
1619 :
1620 : if (ptr && size > 0)
1621 : page_init_poison(ptr, size);
1622 :
1623 1 : return ptr;
1624 : }
1625 :
1626 : #ifdef CONFIG_FLATMEM
1627 1 : static void __init alloc_node_mem_map(struct pglist_data *pgdat)
1628 : {
1629 1 : unsigned long __maybe_unused start = 0;
1630 1 : unsigned long __maybe_unused offset = 0;
1631 :
1632 : /* Skip empty nodes */
1633 1 : if (!pgdat->node_spanned_pages)
1634 : return;
1635 :
1636 1 : start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
1637 1 : offset = pgdat->node_start_pfn - start;
1638 : /* ia64 gets its own node_mem_map, before this, without bootmem */
1639 1 : if (!pgdat->node_mem_map) {
1640 : unsigned long size, end;
1641 : struct page *map;
1642 :
1643 : /*
1644 : * The zone's endpoints aren't required to be MAX_ORDER
1645 : * aligned but the node_mem_map endpoints must be in order
1646 : * for the buddy allocator to function correctly.
1647 : */
1648 2 : end = pgdat_end_pfn(pgdat);
1649 1 : end = ALIGN(end, MAX_ORDER_NR_PAGES);
1650 1 : size = (end - start) * sizeof(struct page);
1651 1 : map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,
1652 : pgdat->node_id, false);
1653 1 : if (!map)
1654 0 : panic("Failed to allocate %ld bytes for node %d memory map\n",
1655 : size, pgdat->node_id);
1656 1 : pgdat->node_mem_map = map + offset;
1657 : }
1658 : pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
1659 : __func__, pgdat->node_id, (unsigned long)pgdat,
1660 : (unsigned long)pgdat->node_mem_map);
1661 : #ifndef CONFIG_NUMA
1662 : /*
1663 : * With no DISCONTIG, the global mem_map is just set as node 0's
1664 : */
1665 1 : if (pgdat == NODE_DATA(0)) {
1666 1 : mem_map = NODE_DATA(0)->node_mem_map;
1667 1 : if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
1668 0 : mem_map -= offset;
1669 : }
1670 : #endif
1671 : }
1672 : #else
1673 : static inline void alloc_node_mem_map(struct pglist_data *pgdat) { }
1674 : #endif /* CONFIG_FLATMEM */
1675 :
1676 : /**
1677 : * get_pfn_range_for_nid - Return the start and end page frames for a node
1678 : * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
1679 : * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
1680 : * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
1681 : *
1682 : * It returns the start and end page frame of a node based on information
1683 : * provided by memblock_set_node(). If called for a node
1684 : * with no available memory, a warning is printed and the start and end
1685 : * PFNs will be 0.
1686 : */
1687 1 : void __init get_pfn_range_for_nid(unsigned int nid,
1688 : unsigned long *start_pfn, unsigned long *end_pfn)
1689 : {
1690 : unsigned long this_start_pfn, this_end_pfn;
1691 : int i;
1692 :
1693 1 : *start_pfn = -1UL;
1694 1 : *end_pfn = 0;
1695 :
1696 2 : for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
1697 1 : *start_pfn = min(*start_pfn, this_start_pfn);
1698 1 : *end_pfn = max(*end_pfn, this_end_pfn);
1699 : }
1700 :
1701 1 : if (*start_pfn == -1UL)
1702 0 : *start_pfn = 0;
1703 1 : }
1704 :
1705 1 : static void __init free_area_init_node(int nid)
1706 : {
1707 1 : pg_data_t *pgdat = NODE_DATA(nid);
1708 1 : unsigned long start_pfn = 0;
1709 1 : unsigned long end_pfn = 0;
1710 :
1711 : /* pg_data_t should be reset to zero when it's allocated */
1712 1 : WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
1713 :
1714 1 : get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
1715 :
1716 1 : pgdat->node_id = nid;
1717 1 : pgdat->node_start_pfn = start_pfn;
1718 1 : pgdat->per_cpu_nodestats = NULL;
1719 :
1720 1 : if (start_pfn != end_pfn) {
1721 1 : pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
1722 : (u64)start_pfn << PAGE_SHIFT,
1723 : end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
1724 :
1725 1 : calculate_node_totalpages(pgdat, start_pfn, end_pfn);
1726 : } else {
1727 0 : pr_info("Initmem setup node %d as memoryless\n", nid);
1728 :
1729 0 : reset_memoryless_node_totalpages(pgdat);
1730 : }
1731 :
1732 1 : alloc_node_mem_map(pgdat);
1733 : pgdat_set_deferred_range(pgdat);
1734 :
1735 1 : free_area_init_core(pgdat);
1736 : lru_gen_init_pgdat(pgdat);
1737 1 : }
1738 :
1739 : /* Any regular or high memory on that node ? */
1740 : static void check_for_memory(pg_data_t *pgdat)
1741 : {
1742 : enum zone_type zone_type;
1743 :
1744 1 : for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
1745 1 : struct zone *zone = &pgdat->node_zones[zone_type];
1746 1 : if (populated_zone(zone)) {
1747 : if (IS_ENABLED(CONFIG_HIGHMEM))
1748 : node_set_state(pgdat->node_id, N_HIGH_MEMORY);
1749 : if (zone_type <= ZONE_NORMAL)
1750 : node_set_state(pgdat->node_id, N_NORMAL_MEMORY);
1751 : break;
1752 : }
1753 : }
1754 : }
1755 :
1756 : #if MAX_NUMNODES > 1
1757 : /*
1758 : * Figure out the number of possible node ids.
1759 : */
1760 : void __init setup_nr_node_ids(void)
1761 : {
1762 : unsigned int highest;
1763 :
1764 : highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
1765 : nr_node_ids = highest + 1;
1766 : }
1767 : #endif
1768 :
1769 : /*
1770 : * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
1771 : * such cases we allow max_zone_pfn sorted in the descending order
1772 : */
1773 : static bool arch_has_descending_max_zone_pfns(void)
1774 : {
1775 : return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40);
1776 : }
1777 :
1778 : /**
1779 : * free_area_init - Initialise all pg_data_t and zone data
1780 : * @max_zone_pfn: an array of max PFNs for each zone
1781 : *
1782 : * This will call free_area_init_node() for each active node in the system.
1783 : * Using the page ranges provided by memblock_set_node(), the size of each
1784 : * zone in each node and their holes is calculated. If the maximum PFN
1785 : * between two adjacent zones match, it is assumed that the zone is empty.
1786 : * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
1787 : * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
1788 : * starts where the previous one ended. For example, ZONE_DMA32 starts
1789 : * at arch_max_dma_pfn.
1790 : */
1791 1 : void __init free_area_init(unsigned long *max_zone_pfn)
1792 : {
1793 : unsigned long start_pfn, end_pfn;
1794 : int i, nid, zone;
1795 : bool descending;
1796 :
1797 : /* Record where the zone boundaries are */
1798 1 : memset(arch_zone_lowest_possible_pfn, 0,
1799 : sizeof(arch_zone_lowest_possible_pfn));
1800 1 : memset(arch_zone_highest_possible_pfn, 0,
1801 : sizeof(arch_zone_highest_possible_pfn));
1802 :
1803 1 : start_pfn = PHYS_PFN(memblock_start_of_DRAM());
1804 1 : descending = arch_has_descending_max_zone_pfns();
1805 :
1806 3 : for (i = 0; i < MAX_NR_ZONES; i++) {
1807 : if (descending)
1808 : zone = MAX_NR_ZONES - i - 1;
1809 : else
1810 2 : zone = i;
1811 :
1812 2 : if (zone == ZONE_MOVABLE)
1813 1 : continue;
1814 :
1815 1 : end_pfn = max(max_zone_pfn[zone], start_pfn);
1816 1 : arch_zone_lowest_possible_pfn[zone] = start_pfn;
1817 1 : arch_zone_highest_possible_pfn[zone] = end_pfn;
1818 :
1819 1 : start_pfn = end_pfn;
1820 : }
1821 :
1822 : /* Find the PFNs that ZONE_MOVABLE begins at in each node */
1823 1 : memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
1824 1 : find_zone_movable_pfns_for_nodes();
1825 :
1826 : /* Print out the zone ranges */
1827 1 : pr_info("Zone ranges:\n");
1828 3 : for (i = 0; i < MAX_NR_ZONES; i++) {
1829 2 : if (i == ZONE_MOVABLE)
1830 1 : continue;
1831 1 : pr_info(" %-8s ", zone_names[i]);
1832 2 : if (arch_zone_lowest_possible_pfn[i] ==
1833 1 : arch_zone_highest_possible_pfn[i])
1834 0 : pr_cont("empty\n");
1835 : else
1836 1 : pr_cont("[mem %#018Lx-%#018Lx]\n",
1837 : (u64)arch_zone_lowest_possible_pfn[i]
1838 : << PAGE_SHIFT,
1839 : ((u64)arch_zone_highest_possible_pfn[i]
1840 : << PAGE_SHIFT) - 1);
1841 : }
1842 :
1843 : /* Print out the PFNs ZONE_MOVABLE begins at in each node */
1844 1 : pr_info("Movable zone start for each node\n");
1845 2 : for (i = 0; i < MAX_NUMNODES; i++) {
1846 1 : if (zone_movable_pfn[i])
1847 0 : pr_info(" Node %d: %#018Lx\n", i,
1848 : (u64)zone_movable_pfn[i] << PAGE_SHIFT);
1849 : }
1850 :
1851 : /*
1852 : * Print out the early node map, and initialize the
1853 : * subsection-map relative to active online memory ranges to
1854 : * enable future "sub-section" extensions of the memory map.
1855 : */
1856 1 : pr_info("Early memory node ranges\n");
1857 2 : for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
1858 1 : pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
1859 : (u64)start_pfn << PAGE_SHIFT,
1860 : ((u64)end_pfn << PAGE_SHIFT) - 1);
1861 : subsection_map_init(start_pfn, end_pfn - start_pfn);
1862 : }
1863 :
1864 : /* Initialise every node */
1865 1 : mminit_verify_pageflags_layout();
1866 : setup_nr_node_ids();
1867 : set_pageblock_order();
1868 :
1869 2 : for_each_node(nid) {
1870 : pg_data_t *pgdat;
1871 :
1872 1 : if (!node_online(nid)) {
1873 : pr_info("Initializing node %d as memoryless\n", nid);
1874 :
1875 : /* Allocator not initialized yet */
1876 : pgdat = arch_alloc_nodedata(nid);
1877 : if (!pgdat)
1878 : panic("Cannot allocate %zuB for node %d.\n",
1879 : sizeof(*pgdat), nid);
1880 : arch_refresh_nodedata(nid, pgdat);
1881 : free_area_init_node(nid);
1882 :
1883 : /*
1884 : * We do not want to confuse userspace by sysfs
1885 : * files/directories for node without any memory
1886 : * attached to it, so this node is not marked as
1887 : * N_MEMORY and not marked online so that no sysfs
1888 : * hierarchy will be created via register_one_node for
1889 : * it. The pgdat will get fully initialized by
1890 : * hotadd_init_pgdat() when memory is hotplugged into
1891 : * this node.
1892 : */
1893 : continue;
1894 : }
1895 :
1896 1 : pgdat = NODE_DATA(nid);
1897 1 : free_area_init_node(nid);
1898 :
1899 : /* Any memory on that node */
1900 : if (pgdat->node_present_pages)
1901 : node_set_state(nid, N_MEMORY);
1902 1 : check_for_memory(pgdat);
1903 : }
1904 :
1905 1 : memmap_init();
1906 :
1907 : /* disable hash distribution for systems with a single node */
1908 : fixup_hashdist();
1909 1 : }
1910 :
1911 : /**
1912 : * node_map_pfn_alignment - determine the maximum internode alignment
1913 : *
1914 : * This function should be called after node map is populated and sorted.
1915 : * It calculates the maximum power of two alignment which can distinguish
1916 : * all the nodes.
1917 : *
1918 : * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
1919 : * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
1920 : * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
1921 : * shifted, 1GiB is enough and this function will indicate so.
1922 : *
1923 : * This is used to test whether pfn -> nid mapping of the chosen memory
1924 : * model has fine enough granularity to avoid incorrect mapping for the
1925 : * populated node map.
1926 : *
1927 : * Return: the determined alignment in pfn's. 0 if there is no alignment
1928 : * requirement (single node).
1929 : */
1930 0 : unsigned long __init node_map_pfn_alignment(void)
1931 : {
1932 0 : unsigned long accl_mask = 0, last_end = 0;
1933 : unsigned long start, end, mask;
1934 0 : int last_nid = NUMA_NO_NODE;
1935 : int i, nid;
1936 :
1937 0 : for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
1938 0 : if (!start || last_nid < 0 || last_nid == nid) {
1939 0 : last_nid = nid;
1940 0 : last_end = end;
1941 0 : continue;
1942 : }
1943 :
1944 : /*
1945 : * Start with a mask granular enough to pin-point to the
1946 : * start pfn and tick off bits one-by-one until it becomes
1947 : * too coarse to separate the current node from the last.
1948 : */
1949 0 : mask = ~((1 << __ffs(start)) - 1);
1950 0 : while (mask && last_end <= (start & (mask << 1)))
1951 : mask <<= 1;
1952 :
1953 : /* accumulate all internode masks */
1954 0 : accl_mask |= mask;
1955 : }
1956 :
1957 : /* convert mask to number of pages */
1958 0 : return ~accl_mask + 1;
1959 : }
1960 :
1961 : #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1962 : static void __init deferred_free_range(unsigned long pfn,
1963 : unsigned long nr_pages)
1964 : {
1965 : struct page *page;
1966 : unsigned long i;
1967 :
1968 : if (!nr_pages)
1969 : return;
1970 :
1971 : page = pfn_to_page(pfn);
1972 :
1973 : /* Free a large naturally-aligned chunk if possible */
1974 : if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {
1975 : for (i = 0; i < nr_pages; i += pageblock_nr_pages)
1976 : set_pageblock_migratetype(page + i, MIGRATE_MOVABLE);
1977 : __free_pages_core(page, MAX_ORDER);
1978 : return;
1979 : }
1980 :
1981 : /* Accept chunks smaller than MAX_ORDER upfront */
1982 : accept_memory(PFN_PHYS(pfn), PFN_PHYS(pfn + nr_pages));
1983 :
1984 : for (i = 0; i < nr_pages; i++, page++, pfn++) {
1985 : if (pageblock_aligned(pfn))
1986 : set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1987 : __free_pages_core(page, 0);
1988 : }
1989 : }
1990 :
1991 : /* Completion tracking for deferred_init_memmap() threads */
1992 : static atomic_t pgdat_init_n_undone __initdata;
1993 : static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1994 :
1995 : static inline void __init pgdat_init_report_one_done(void)
1996 : {
1997 : if (atomic_dec_and_test(&pgdat_init_n_undone))
1998 : complete(&pgdat_init_all_done_comp);
1999 : }
2000 :
2001 : /*
2002 : * Returns true if page needs to be initialized or freed to buddy allocator.
2003 : *
2004 : * We check if a current MAX_ORDER block is valid by only checking the validity
2005 : * of the head pfn.
2006 : */
2007 : static inline bool __init deferred_pfn_valid(unsigned long pfn)
2008 : {
2009 : if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn))
2010 : return false;
2011 : return true;
2012 : }
2013 :
2014 : /*
2015 : * Free pages to buddy allocator. Try to free aligned pages in
2016 : * MAX_ORDER_NR_PAGES sizes.
2017 : */
2018 : static void __init deferred_free_pages(unsigned long pfn,
2019 : unsigned long end_pfn)
2020 : {
2021 : unsigned long nr_free = 0;
2022 :
2023 : for (; pfn < end_pfn; pfn++) {
2024 : if (!deferred_pfn_valid(pfn)) {
2025 : deferred_free_range(pfn - nr_free, nr_free);
2026 : nr_free = 0;
2027 : } else if (IS_MAX_ORDER_ALIGNED(pfn)) {
2028 : deferred_free_range(pfn - nr_free, nr_free);
2029 : nr_free = 1;
2030 : } else {
2031 : nr_free++;
2032 : }
2033 : }
2034 : /* Free the last block of pages to allocator */
2035 : deferred_free_range(pfn - nr_free, nr_free);
2036 : }
2037 :
2038 : /*
2039 : * Initialize struct pages. We minimize pfn page lookups and scheduler checks
2040 : * by performing it only once every MAX_ORDER_NR_PAGES.
2041 : * Return number of pages initialized.
2042 : */
2043 : static unsigned long __init deferred_init_pages(struct zone *zone,
2044 : unsigned long pfn,
2045 : unsigned long end_pfn)
2046 : {
2047 : int nid = zone_to_nid(zone);
2048 : unsigned long nr_pages = 0;
2049 : int zid = zone_idx(zone);
2050 : struct page *page = NULL;
2051 :
2052 : for (; pfn < end_pfn; pfn++) {
2053 : if (!deferred_pfn_valid(pfn)) {
2054 : page = NULL;
2055 : continue;
2056 : } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) {
2057 : page = pfn_to_page(pfn);
2058 : } else {
2059 : page++;
2060 : }
2061 : __init_single_page(page, pfn, zid, nid);
2062 : nr_pages++;
2063 : }
2064 : return (nr_pages);
2065 : }
2066 :
2067 : /*
2068 : * This function is meant to pre-load the iterator for the zone init.
2069 : * Specifically it walks through the ranges until we are caught up to the
2070 : * first_init_pfn value and exits there. If we never encounter the value we
2071 : * return false indicating there are no valid ranges left.
2072 : */
2073 : static bool __init
2074 : deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
2075 : unsigned long *spfn, unsigned long *epfn,
2076 : unsigned long first_init_pfn)
2077 : {
2078 : u64 j;
2079 :
2080 : /*
2081 : * Start out by walking through the ranges in this zone that have
2082 : * already been initialized. We don't need to do anything with them
2083 : * so we just need to flush them out of the system.
2084 : */
2085 : for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
2086 : if (*epfn <= first_init_pfn)
2087 : continue;
2088 : if (*spfn < first_init_pfn)
2089 : *spfn = first_init_pfn;
2090 : *i = j;
2091 : return true;
2092 : }
2093 :
2094 : return false;
2095 : }
2096 :
2097 : /*
2098 : * Initialize and free pages. We do it in two loops: first we initialize
2099 : * struct page, then free to buddy allocator, because while we are
2100 : * freeing pages we can access pages that are ahead (computing buddy
2101 : * page in __free_one_page()).
2102 : *
2103 : * In order to try and keep some memory in the cache we have the loop
2104 : * broken along max page order boundaries. This way we will not cause
2105 : * any issues with the buddy page computation.
2106 : */
2107 : static unsigned long __init
2108 : deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
2109 : unsigned long *end_pfn)
2110 : {
2111 : unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
2112 : unsigned long spfn = *start_pfn, epfn = *end_pfn;
2113 : unsigned long nr_pages = 0;
2114 : u64 j = *i;
2115 :
2116 : /* First we loop through and initialize the page values */
2117 : for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
2118 : unsigned long t;
2119 :
2120 : if (mo_pfn <= *start_pfn)
2121 : break;
2122 :
2123 : t = min(mo_pfn, *end_pfn);
2124 : nr_pages += deferred_init_pages(zone, *start_pfn, t);
2125 :
2126 : if (mo_pfn < *end_pfn) {
2127 : *start_pfn = mo_pfn;
2128 : break;
2129 : }
2130 : }
2131 :
2132 : /* Reset values and now loop through freeing pages as needed */
2133 : swap(j, *i);
2134 :
2135 : for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
2136 : unsigned long t;
2137 :
2138 : if (mo_pfn <= spfn)
2139 : break;
2140 :
2141 : t = min(mo_pfn, epfn);
2142 : deferred_free_pages(spfn, t);
2143 :
2144 : if (mo_pfn <= epfn)
2145 : break;
2146 : }
2147 :
2148 : return nr_pages;
2149 : }
2150 :
2151 : static void __init
2152 : deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2153 : void *arg)
2154 : {
2155 : unsigned long spfn, epfn;
2156 : struct zone *zone = arg;
2157 : u64 i;
2158 :
2159 : deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
2160 :
2161 : /*
2162 : * Initialize and free pages in MAX_ORDER sized increments so that we
2163 : * can avoid introducing any issues with the buddy allocator.
2164 : */
2165 : while (spfn < end_pfn) {
2166 : deferred_init_maxorder(&i, zone, &spfn, &epfn);
2167 : cond_resched();
2168 : }
2169 : }
2170 :
2171 : /* An arch may override for more concurrency. */
2172 : __weak int __init
2173 : deferred_page_init_max_threads(const struct cpumask *node_cpumask)
2174 : {
2175 : return 1;
2176 : }
2177 :
2178 : /* Initialise remaining memory on a node */
2179 : static int __init deferred_init_memmap(void *data)
2180 : {
2181 : pg_data_t *pgdat = data;
2182 : const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2183 : unsigned long spfn = 0, epfn = 0;
2184 : unsigned long first_init_pfn, flags;
2185 : unsigned long start = jiffies;
2186 : struct zone *zone;
2187 : int zid, max_threads;
2188 : u64 i;
2189 :
2190 : /* Bind memory initialisation thread to a local node if possible */
2191 : if (!cpumask_empty(cpumask))
2192 : set_cpus_allowed_ptr(current, cpumask);
2193 :
2194 : pgdat_resize_lock(pgdat, &flags);
2195 : first_init_pfn = pgdat->first_deferred_pfn;
2196 : if (first_init_pfn == ULONG_MAX) {
2197 : pgdat_resize_unlock(pgdat, &flags);
2198 : pgdat_init_report_one_done();
2199 : return 0;
2200 : }
2201 :
2202 : /* Sanity check boundaries */
2203 : BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
2204 : BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
2205 : pgdat->first_deferred_pfn = ULONG_MAX;
2206 :
2207 : /*
2208 : * Once we unlock here, the zone cannot be grown anymore, thus if an
2209 : * interrupt thread must allocate this early in boot, zone must be
2210 : * pre-grown prior to start of deferred page initialization.
2211 : */
2212 : pgdat_resize_unlock(pgdat, &flags);
2213 :
2214 : /* Only the highest zone is deferred so find it */
2215 : for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2216 : zone = pgdat->node_zones + zid;
2217 : if (first_init_pfn < zone_end_pfn(zone))
2218 : break;
2219 : }
2220 :
2221 : /* If the zone is empty somebody else may have cleared out the zone */
2222 : if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2223 : first_init_pfn))
2224 : goto zone_empty;
2225 :
2226 : max_threads = deferred_page_init_max_threads(cpumask);
2227 :
2228 : while (spfn < epfn) {
2229 : unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
2230 : struct padata_mt_job job = {
2231 : .thread_fn = deferred_init_memmap_chunk,
2232 : .fn_arg = zone,
2233 : .start = spfn,
2234 : .size = epfn_align - spfn,
2235 : .align = PAGES_PER_SECTION,
2236 : .min_chunk = PAGES_PER_SECTION,
2237 : .max_threads = max_threads,
2238 : };
2239 :
2240 : padata_do_multithreaded(&job);
2241 : deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2242 : epfn_align);
2243 : }
2244 : zone_empty:
2245 : /* Sanity check that the next zone really is unpopulated */
2246 : WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
2247 :
2248 : pr_info("node %d deferred pages initialised in %ums\n",
2249 : pgdat->node_id, jiffies_to_msecs(jiffies - start));
2250 :
2251 : pgdat_init_report_one_done();
2252 : return 0;
2253 : }
2254 :
2255 : /*
2256 : * If this zone has deferred pages, try to grow it by initializing enough
2257 : * deferred pages to satisfy the allocation specified by order, rounded up to
2258 : * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
2259 : * of SECTION_SIZE bytes by initializing struct pages in increments of
2260 : * PAGES_PER_SECTION * sizeof(struct page) bytes.
2261 : *
2262 : * Return true when zone was grown, otherwise return false. We return true even
2263 : * when we grow less than requested, to let the caller decide if there are
2264 : * enough pages to satisfy the allocation.
2265 : *
2266 : * Note: We use noinline because this function is needed only during boot, and
2267 : * it is called from a __ref function _deferred_grow_zone. This way we are
2268 : * making sure that it is not inlined into permanent text section.
2269 : */
2270 : bool __init deferred_grow_zone(struct zone *zone, unsigned int order)
2271 : {
2272 : unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
2273 : pg_data_t *pgdat = zone->zone_pgdat;
2274 : unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
2275 : unsigned long spfn, epfn, flags;
2276 : unsigned long nr_pages = 0;
2277 : u64 i;
2278 :
2279 : /* Only the last zone may have deferred pages */
2280 : if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2281 : return false;
2282 :
2283 : pgdat_resize_lock(pgdat, &flags);
2284 :
2285 : /*
2286 : * If someone grew this zone while we were waiting for spinlock, return
2287 : * true, as there might be enough pages already.
2288 : */
2289 : if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2290 : pgdat_resize_unlock(pgdat, &flags);
2291 : return true;
2292 : }
2293 :
2294 : /* If the zone is empty somebody else may have cleared out the zone */
2295 : if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2296 : first_deferred_pfn)) {
2297 : pgdat->first_deferred_pfn = ULONG_MAX;
2298 : pgdat_resize_unlock(pgdat, &flags);
2299 : /* Retry only once. */
2300 : return first_deferred_pfn != ULONG_MAX;
2301 : }
2302 :
2303 : /*
2304 : * Initialize and free pages in MAX_ORDER sized increments so
2305 : * that we can avoid introducing any issues with the buddy
2306 : * allocator.
2307 : */
2308 : while (spfn < epfn) {
2309 : /* update our first deferred PFN for this section */
2310 : first_deferred_pfn = spfn;
2311 :
2312 : nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2313 : touch_nmi_watchdog();
2314 :
2315 : /* We should only stop along section boundaries */
2316 : if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2317 : continue;
2318 :
2319 : /* If our quota has been met we can stop here */
2320 : if (nr_pages >= nr_pages_needed)
2321 : break;
2322 : }
2323 :
2324 : pgdat->first_deferred_pfn = spfn;
2325 : pgdat_resize_unlock(pgdat, &flags);
2326 :
2327 : return nr_pages > 0;
2328 : }
2329 :
2330 : #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2331 :
2332 : #ifdef CONFIG_CMA
2333 : void __init init_cma_reserved_pageblock(struct page *page)
2334 : {
2335 : unsigned i = pageblock_nr_pages;
2336 : struct page *p = page;
2337 :
2338 : do {
2339 : __ClearPageReserved(p);
2340 : set_page_count(p, 0);
2341 : } while (++p, --i);
2342 :
2343 : set_pageblock_migratetype(page, MIGRATE_CMA);
2344 : set_page_refcounted(page);
2345 : __free_pages(page, pageblock_order);
2346 :
2347 : adjust_managed_page_count(page, pageblock_nr_pages);
2348 : page_zone(page)->cma_pages += pageblock_nr_pages;
2349 : }
2350 : #endif
2351 :
2352 1 : void set_zone_contiguous(struct zone *zone)
2353 : {
2354 1 : unsigned long block_start_pfn = zone->zone_start_pfn;
2355 : unsigned long block_end_pfn;
2356 :
2357 1 : block_end_pfn = pageblock_end_pfn(block_start_pfn);
2358 523 : for (; block_start_pfn < zone_end_pfn(zone);
2359 260 : block_start_pfn = block_end_pfn,
2360 260 : block_end_pfn += pageblock_nr_pages) {
2361 :
2362 260 : block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
2363 :
2364 260 : if (!__pageblock_pfn_to_page(block_start_pfn,
2365 : block_end_pfn, zone))
2366 : return;
2367 260 : cond_resched();
2368 : }
2369 :
2370 : /* We confirm that there is no hole */
2371 1 : zone->contiguous = true;
2372 : }
2373 :
2374 1 : void __init page_alloc_init_late(void)
2375 : {
2376 : struct zone *zone;
2377 : int nid;
2378 :
2379 : #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
2380 :
2381 : /* There will be num_node_state(N_MEMORY) threads */
2382 : atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
2383 : for_each_node_state(nid, N_MEMORY) {
2384 : kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2385 : }
2386 :
2387 : /* Block until all are initialised */
2388 : wait_for_completion(&pgdat_init_all_done_comp);
2389 :
2390 : /*
2391 : * We initialized the rest of the deferred pages. Permanently disable
2392 : * on-demand struct page initialization.
2393 : */
2394 : static_branch_disable(&deferred_pages);
2395 :
2396 : /* Reinit limits that are based on free pages after the kernel is up */
2397 : files_maxfiles_init();
2398 : #endif
2399 :
2400 1 : buffer_init();
2401 :
2402 : /* Discard memblock private memory */
2403 1 : memblock_discard();
2404 :
2405 1 : for_each_node_state(nid, N_MEMORY)
2406 : shuffle_free_memory(NODE_DATA(nid));
2407 :
2408 3 : for_each_populated_zone(zone)
2409 1 : set_zone_contiguous(zone);
2410 :
2411 : /* Initialize page ext after all struct pages are initialized. */
2412 : if (deferred_struct_pages)
2413 : page_ext_init();
2414 :
2415 1 : page_alloc_sysctl_init();
2416 1 : }
2417 :
2418 : #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
2419 : /*
2420 : * Returns the number of pages that arch has reserved but
2421 : * is not known to alloc_large_system_hash().
2422 : */
2423 : static unsigned long __init arch_reserved_kernel_pages(void)
2424 : {
2425 : return 0;
2426 : }
2427 : #endif
2428 :
2429 : /*
2430 : * Adaptive scale is meant to reduce sizes of hash tables on large memory
2431 : * machines. As memory size is increased the scale is also increased but at
2432 : * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
2433 : * quadruples the scale is increased by one, which means the size of hash table
2434 : * only doubles, instead of quadrupling as well.
2435 : * Because 32-bit systems cannot have large physical memory, where this scaling
2436 : * makes sense, it is disabled on such platforms.
2437 : */
2438 : #if __BITS_PER_LONG > 32
2439 : #define ADAPT_SCALE_BASE (64ul << 30)
2440 : #define ADAPT_SCALE_SHIFT 2
2441 : #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
2442 : #endif
2443 :
2444 : /*
2445 : * allocate a large system hash table from bootmem
2446 : * - it is assumed that the hash table must contain an exact power-of-2
2447 : * quantity of entries
2448 : * - limit is the number of hash buckets, not the total allocation size
2449 : */
2450 5 : void *__init alloc_large_system_hash(const char *tablename,
2451 : unsigned long bucketsize,
2452 : unsigned long numentries,
2453 : int scale,
2454 : int flags,
2455 : unsigned int *_hash_shift,
2456 : unsigned int *_hash_mask,
2457 : unsigned long low_limit,
2458 : unsigned long high_limit)
2459 : {
2460 5 : unsigned long long max = high_limit;
2461 : unsigned long log2qty, size;
2462 : void *table;
2463 : gfp_t gfp_flags;
2464 : bool virt;
2465 : bool huge;
2466 :
2467 : /* allow the kernel cmdline to have a say */
2468 5 : if (!numentries) {
2469 : /* round applicable memory size up to nearest megabyte */
2470 4 : numentries = nr_kernel_pages;
2471 4 : numentries -= arch_reserved_kernel_pages();
2472 :
2473 : /* It isn't necessary when PAGE_SIZE >= 1MB */
2474 : if (PAGE_SIZE < SZ_1M)
2475 4 : numentries = round_up(numentries, SZ_1M / PAGE_SIZE);
2476 :
2477 : #if __BITS_PER_LONG > 32
2478 4 : if (!high_limit) {
2479 : unsigned long adapt;
2480 :
2481 4 : for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
2482 0 : adapt <<= ADAPT_SCALE_SHIFT)
2483 0 : scale++;
2484 : }
2485 : #endif
2486 :
2487 : /* limit to 1 bucket per 2^scale bytes of low memory */
2488 4 : if (scale > PAGE_SHIFT)
2489 4 : numentries >>= (scale - PAGE_SHIFT);
2490 : else
2491 0 : numentries <<= (PAGE_SHIFT - scale);
2492 :
2493 : /* Make sure we've got at least a 0-order allocation.. */
2494 4 : if (unlikely(flags & HASH_SMALL)) {
2495 : /* Makes no sense without HASH_EARLY */
2496 0 : WARN_ON(!(flags & HASH_EARLY));
2497 0 : if (!(numentries >> *_hash_shift)) {
2498 0 : numentries = 1UL << *_hash_shift;
2499 0 : BUG_ON(!numentries);
2500 : }
2501 4 : } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
2502 0 : numentries = PAGE_SIZE / bucketsize;
2503 : }
2504 10 : numentries = roundup_pow_of_two(numentries);
2505 :
2506 : /* limit allocation size to 1/16 total memory by default */
2507 5 : if (max == 0) {
2508 4 : max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
2509 4 : do_div(max, bucketsize);
2510 : }
2511 5 : max = min(max, 0x80000000ULL);
2512 :
2513 5 : if (numentries < low_limit)
2514 0 : numentries = low_limit;
2515 5 : if (numentries > max)
2516 0 : numentries = max;
2517 :
2518 10 : log2qty = ilog2(numentries);
2519 :
2520 5 : gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
2521 : do {
2522 5 : virt = false;
2523 5 : size = bucketsize << log2qty;
2524 5 : if (flags & HASH_EARLY) {
2525 2 : if (flags & HASH_ZERO)
2526 2 : table = memblock_alloc(size, SMP_CACHE_BYTES);
2527 : else
2528 0 : table = memblock_alloc_raw(size,
2529 : SMP_CACHE_BYTES);
2530 3 : } else if (get_order(size) > MAX_ORDER || hashdist) {
2531 0 : table = vmalloc_huge(size, gfp_flags);
2532 0 : virt = true;
2533 : if (table)
2534 : huge = is_vm_area_hugepages(table);
2535 : } else {
2536 : /*
2537 : * If bucketsize is not a power-of-two, we may free
2538 : * some pages at the end of hash table which
2539 : * alloc_pages_exact() automatically does
2540 : */
2541 3 : table = alloc_pages_exact(size, gfp_flags);
2542 3 : kmemleak_alloc(table, size, 1, gfp_flags);
2543 : }
2544 5 : } while (!table && size > PAGE_SIZE && --log2qty);
2545 :
2546 5 : if (!table)
2547 0 : panic("Failed to allocate %s hash table\n", tablename);
2548 :
2549 10 : pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
2550 : tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
2551 : virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
2552 :
2553 5 : if (_hash_shift)
2554 5 : *_hash_shift = log2qty;
2555 5 : if (_hash_mask)
2556 3 : *_hash_mask = (1 << log2qty) - 1;
2557 :
2558 5 : return table;
2559 : }
2560 :
2561 : /**
2562 : * set_dma_reserve - set the specified number of pages reserved in the first zone
2563 : * @new_dma_reserve: The number of pages to mark reserved
2564 : *
2565 : * The per-cpu batchsize and zone watermarks are determined by managed_pages.
2566 : * In the DMA zone, a significant percentage may be consumed by kernel image
2567 : * and other unfreeable allocations which can skew the watermarks badly. This
2568 : * function may optionally be used to account for unfreeable pages in the
2569 : * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
2570 : * smaller per-cpu batchsize.
2571 : */
2572 0 : void __init set_dma_reserve(unsigned long new_dma_reserve)
2573 : {
2574 0 : dma_reserve = new_dma_reserve;
2575 0 : }
2576 :
2577 259 : void __init memblock_free_pages(struct page *page, unsigned long pfn,
2578 : unsigned int order)
2579 : {
2580 :
2581 : if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) {
2582 : int nid = early_pfn_to_nid(pfn);
2583 :
2584 : if (!early_page_initialised(pfn, nid))
2585 : return;
2586 : }
2587 :
2588 259 : if (!kmsan_memblock_free_pages(page, order)) {
2589 : /* KMSAN will take care of these pages. */
2590 : return;
2591 : }
2592 259 : __free_pages_core(page, order);
2593 : }
2594 :
2595 : DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
2596 : EXPORT_SYMBOL(init_on_alloc);
2597 :
2598 : DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
2599 : EXPORT_SYMBOL(init_on_free);
2600 :
2601 : static bool _init_on_alloc_enabled_early __read_mostly
2602 : = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
2603 0 : static int __init early_init_on_alloc(char *buf)
2604 : {
2605 :
2606 0 : return kstrtobool(buf, &_init_on_alloc_enabled_early);
2607 : }
2608 : early_param("init_on_alloc", early_init_on_alloc);
2609 :
2610 : static bool _init_on_free_enabled_early __read_mostly
2611 : = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
2612 0 : static int __init early_init_on_free(char *buf)
2613 : {
2614 0 : return kstrtobool(buf, &_init_on_free_enabled_early);
2615 : }
2616 : early_param("init_on_free", early_init_on_free);
2617 :
2618 : DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
2619 :
2620 : /*
2621 : * Enable static keys related to various memory debugging and hardening options.
2622 : * Some override others, and depend on early params that are evaluated in the
2623 : * order of appearance. So we need to first gather the full picture of what was
2624 : * enabled, and then make decisions.
2625 : */
2626 1 : static void __init mem_debugging_and_hardening_init(void)
2627 : {
2628 1 : bool page_poisoning_requested = false;
2629 1 : bool want_check_pages = false;
2630 :
2631 : #ifdef CONFIG_PAGE_POISONING
2632 : /*
2633 : * Page poisoning is debug page alloc for some arches. If
2634 : * either of those options are enabled, enable poisoning.
2635 : */
2636 : if (page_poisoning_enabled() ||
2637 : (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
2638 : debug_pagealloc_enabled())) {
2639 : static_branch_enable(&_page_poisoning_enabled);
2640 : page_poisoning_requested = true;
2641 : want_check_pages = true;
2642 : }
2643 : #endif
2644 :
2645 1 : if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
2646 : page_poisoning_requested) {
2647 : pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
2648 : "will take precedence over init_on_alloc and init_on_free\n");
2649 : _init_on_alloc_enabled_early = false;
2650 : _init_on_free_enabled_early = false;
2651 : }
2652 :
2653 1 : if (_init_on_alloc_enabled_early) {
2654 0 : want_check_pages = true;
2655 0 : static_branch_enable(&init_on_alloc);
2656 : } else {
2657 1 : static_branch_disable(&init_on_alloc);
2658 : }
2659 :
2660 1 : if (_init_on_free_enabled_early) {
2661 0 : want_check_pages = true;
2662 0 : static_branch_enable(&init_on_free);
2663 : } else {
2664 1 : static_branch_disable(&init_on_free);
2665 : }
2666 :
2667 : if (IS_ENABLED(CONFIG_KMSAN) &&
2668 : (_init_on_alloc_enabled_early || _init_on_free_enabled_early))
2669 : pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");
2670 :
2671 : #ifdef CONFIG_DEBUG_PAGEALLOC
2672 : if (debug_pagealloc_enabled()) {
2673 : want_check_pages = true;
2674 : static_branch_enable(&_debug_pagealloc_enabled);
2675 :
2676 : if (debug_guardpage_minorder())
2677 : static_branch_enable(&_debug_guardpage_enabled);
2678 : }
2679 : #endif
2680 :
2681 : /*
2682 : * Any page debugging or hardening option also enables sanity checking
2683 : * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's
2684 : * enabled already.
2685 : */
2686 1 : if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages)
2687 0 : static_branch_enable(&check_pages_enabled);
2688 1 : }
2689 :
2690 : /* Report memory auto-initialization states for this boot. */
2691 1 : static void __init report_meminit(void)
2692 : {
2693 : const char *stack;
2694 :
2695 : if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN))
2696 : stack = "all(pattern)";
2697 : else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO))
2698 : stack = "all(zero)";
2699 : else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL))
2700 : stack = "byref_all(zero)";
2701 : else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF))
2702 : stack = "byref(zero)";
2703 : else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER))
2704 : stack = "__user(zero)";
2705 : else
2706 1 : stack = "off";
2707 :
2708 2 : pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",
2709 : stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",
2710 : want_init_on_free() ? "on" : "off");
2711 1 : if (want_init_on_free())
2712 0 : pr_info("mem auto-init: clearing system memory may take some time...\n");
2713 1 : }
2714 :
2715 1 : static void __init mem_init_print_info(void)
2716 : {
2717 : unsigned long physpages, codesize, datasize, rosize, bss_size;
2718 : unsigned long init_code_size, init_data_size;
2719 :
2720 1 : physpages = get_num_physpages();
2721 1 : codesize = _etext - _stext;
2722 1 : datasize = _edata - _sdata;
2723 1 : rosize = __end_rodata - __start_rodata;
2724 1 : bss_size = __bss_stop - __bss_start;
2725 1 : init_data_size = __init_end - __init_begin;
2726 1 : init_code_size = _einittext - _sinittext;
2727 :
2728 : /*
2729 : * Detect special cases and adjust section sizes accordingly:
2730 : * 1) .init.* may be embedded into .data sections
2731 : * 2) .init.text.* may be out of [__init_begin, __init_end],
2732 : * please refer to arch/tile/kernel/vmlinux.lds.S.
2733 : * 3) .rodata.* may be embedded into .text or .data sections.
2734 : */
2735 : #define adj_init_size(start, end, size, pos, adj) \
2736 : do { \
2737 : if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
2738 : size -= adj; \
2739 : } while (0)
2740 :
2741 1 : adj_init_size(__init_begin, __init_end, init_data_size,
2742 : _sinittext, init_code_size);
2743 1 : adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
2744 1 : adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
2745 1 : adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
2746 1 : adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
2747 :
2748 : #undef adj_init_size
2749 :
2750 3 : pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
2751 : #ifdef CONFIG_HIGHMEM
2752 : ", %luK highmem"
2753 : #endif
2754 : ")\n",
2755 : K(nr_free_pages()), K(physpages),
2756 : codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,
2757 : (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,
2758 : K(physpages - totalram_pages() - totalcma_pages),
2759 : K(totalcma_pages)
2760 : #ifdef CONFIG_HIGHMEM
2761 : , K(totalhigh_pages())
2762 : #endif
2763 : );
2764 1 : }
2765 :
2766 : /*
2767 : * Set up kernel memory allocators
2768 : */
2769 1 : void __init mm_core_init(void)
2770 : {
2771 : /* Initializations relying on SMP setup */
2772 1 : build_all_zonelists(NULL);
2773 1 : page_alloc_init_cpuhp();
2774 :
2775 : /*
2776 : * page_ext requires contiguous pages,
2777 : * bigger than MAX_ORDER unless SPARSEMEM.
2778 : */
2779 : page_ext_init_flatmem();
2780 1 : mem_debugging_and_hardening_init();
2781 : kfence_alloc_pool();
2782 1 : report_meminit();
2783 : kmsan_init_shadow();
2784 1 : stack_depot_early_init();
2785 1 : mem_init();
2786 1 : mem_init_print_info();
2787 1 : kmem_cache_init();
2788 : /*
2789 : * page_owner must be initialized after buddy is ready, and also after
2790 : * slab is ready so that stack_depot_init() works properly
2791 : */
2792 : page_ext_init_flatmem_late();
2793 : kmemleak_init();
2794 : ptlock_cache_init();
2795 1 : pgtable_cache_init();
2796 : debug_objects_mem_init();
2797 1 : vmalloc_init();
2798 : /* If no deferred init page_ext now, as vmap is fully initialized */
2799 : if (!deferred_struct_pages)
2800 : page_ext_init();
2801 : /* Should be run before the first non-init thread is created */
2802 : init_espfix_bsp();
2803 : /* Should be run after espfix64 is set up. */
2804 : pti_init();
2805 : kmsan_init_runtime();
2806 1 : mm_cache_init();
2807 1 : }
|
