Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /*
       3             :  * mm_init.c - Memory initialisation verification and debugging
       4             :  *
       5             :  * Copyright 2008 IBM Corporation, 2008
       6             :  * Author Mel Gorman <mel@csn.ul.ie>
       7             :  *
       8             :  */
       9             : #include <linux/kernel.h>
      10             : #include <linux/init.h>
      11             : #include <linux/kobject.h>
      12             : #include <linux/export.h>
      13             : #include <linux/memory.h>
      14             : #include <linux/notifier.h>
      15             : #include <linux/sched.h>
      16             : #include <linux/mman.h>
      17             : #include "internal.h"
      18             : 
      19             : #ifdef CONFIG_DEBUG_MEMORY_INIT
      20             : int __meminitdata mminit_loglevel;
      21             : 
      22             : /* The zonelists are simply reported, validation is manual. */
      23           1 : void __init mminit_verify_zonelist(void)
      24             : {
      25             :         int nid;
      26             : 
      27           1 :         if (mminit_loglevel < MMINIT_VERIFY)
      28             :                 return;
      29             : 
      30           0 :         for_each_online_node(nid) {
      31             :                 pg_data_t *pgdat = NODE_DATA(nid);
      32             :                 struct zone *zone;
      33             :                 struct zoneref *z;
      34             :                 struct zonelist *zonelist;
      35             :                 int i, listid, zoneid;
      36             : 
      37             :                 BUILD_BUG_ON(MAX_ZONELISTS > 2);
      38           0 :                 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) {
      39             : 
      40             :                         /* Identify the zone and nodelist */
      41           0 :                         zoneid = i % MAX_NR_ZONES;
      42           0 :                         listid = i / MAX_NR_ZONES;
      43           0 :                         zonelist = &pgdat->node_zonelists[listid];
      44           0 :                         zone = &pgdat->node_zones[zoneid];
      45           0 :                         if (!populated_zone(zone))
      46           0 :                                 continue;
      47             : 
      48             :                         /* Print information about the zonelist */
      49           0 :                         printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ",
      50             :                                 listid > 0 ? "thisnode" : "general", nid,
      51             :                                 zone->name);
      52             : 
      53             :                         /* Iterate the zonelist */
      54           0 :                         for_each_zone_zonelist(zone, z, zonelist, zoneid)
      55           0 :                                 pr_cont("%d:%s ", zone_to_nid(zone), zone->name);
      56           0 :                         pr_cont("\n");
      57             :                 }
      58             :         }
      59             : }
      60             : 
      61           1 : void __init mminit_verify_pageflags_layout(void)
      62             : {
      63             :         int shift, width;
      64             :         unsigned long or_mask, add_mask;
      65             : 
      66           1 :         shift = 8 * sizeof(unsigned long);
      67           1 :         width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
      68             :                 - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
      69           1 :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
      70             :                 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
      71             :                 SECTIONS_WIDTH,
      72             :                 NODES_WIDTH,
      73             :                 ZONES_WIDTH,
      74             :                 LAST_CPUPID_WIDTH,
      75             :                 KASAN_TAG_WIDTH,
      76             :                 LRU_GEN_WIDTH,
      77             :                 LRU_REFS_WIDTH,
      78             :                 NR_PAGEFLAGS);
      79           1 :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
      80             :                 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
      81             :                 SECTIONS_SHIFT,
      82             :                 NODES_SHIFT,
      83             :                 ZONES_SHIFT,
      84             :                 LAST_CPUPID_SHIFT,
      85             :                 KASAN_TAG_WIDTH);
      86           1 :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",
      87             :                 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",
      88             :                 (unsigned long)SECTIONS_PGSHIFT,
      89             :                 (unsigned long)NODES_PGSHIFT,
      90             :                 (unsigned long)ZONES_PGSHIFT,
      91             :                 (unsigned long)LAST_CPUPID_PGSHIFT,
      92             :                 (unsigned long)KASAN_TAG_PGSHIFT);
      93           1 :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",
      94             :                 "Node/Zone ID: %lu -> %lu\n",
      95             :                 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),
      96             :                 (unsigned long)ZONEID_PGOFF);
      97           1 :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage",
      98             :                 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",
      99             :                 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0);
     100             : #ifdef NODE_NOT_IN_PAGE_FLAGS
     101             :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
     102             :                 "Node not in page flags");
     103             : #endif
     104             : #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
     105             :         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
     106             :                 "Last cpupid not in page flags");
     107             : #endif
     108             : 
     109             :         if (SECTIONS_WIDTH) {
     110             :                 shift -= SECTIONS_WIDTH;
     111             :                 BUG_ON(shift != SECTIONS_PGSHIFT);
     112             :         }
     113             :         if (NODES_WIDTH) {
     114             :                 shift -= NODES_WIDTH;
     115             :                 BUG_ON(shift != NODES_PGSHIFT);
     116             :         }
     117             :         if (ZONES_WIDTH) {
     118           1 :                 shift -= ZONES_WIDTH;
     119             :                 BUG_ON(shift != ZONES_PGSHIFT);
     120             :         }
     121             : 
     122             :         /* Check for bitmask overlaps */
     123           1 :         or_mask = (ZONES_MASK << ZONES_PGSHIFT) |
     124             :                         (NODES_MASK << NODES_PGSHIFT) |
     125             :                         (SECTIONS_MASK << SECTIONS_PGSHIFT);
     126           1 :         add_mask = (ZONES_MASK << ZONES_PGSHIFT) +
     127             :                         (NODES_MASK << NODES_PGSHIFT) +
     128             :                         (SECTIONS_MASK << SECTIONS_PGSHIFT);
     129             :         BUG_ON(or_mask != add_mask);
     130           1 : }
     131             : 
     132           0 : static __init int set_mminit_loglevel(char *str)
     133             : {
     134           0 :         get_option(&str, &mminit_loglevel);
     135           0 :         return 0;
     136             : }
     137             : early_param("mminit_loglevel", set_mminit_loglevel);
     138             : #endif /* CONFIG_DEBUG_MEMORY_INIT */
     139             : 
     140             : struct kobject *mm_kobj;
     141             : EXPORT_SYMBOL_GPL(mm_kobj);
     142             : 
     143             : #ifdef CONFIG_SMP
     144             : s32 vm_committed_as_batch = 32;
     145             : 
     146             : void mm_compute_batch(int overcommit_policy)
     147             : {
     148             :         u64 memsized_batch;
     149             :         s32 nr = num_present_cpus();
     150             :         s32 batch = max_t(s32, nr*2, 32);
     151             :         unsigned long ram_pages = totalram_pages();
     152             : 
     153             :         /*
     154             :          * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of
     155             :          * (total memory/#cpus), and lift it to 25% for other policies
     156             :          * to easy the possible lock contention for percpu_counter
     157             :          * vm_committed_as, while the max limit is INT_MAX
     158             :          */
     159             :         if (overcommit_policy == OVERCOMMIT_NEVER)
     160             :                 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX);
     161             :         else
     162             :                 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX);
     163             : 
     164             :         vm_committed_as_batch = max_t(s32, memsized_batch, batch);
     165             : }
     166             : 
     167             : static int __meminit mm_compute_batch_notifier(struct notifier_block *self,
     168             :                                         unsigned long action, void *arg)
     169             : {
     170             :         switch (action) {
     171             :         case MEM_ONLINE:
     172             :         case MEM_OFFLINE:
     173             :                 mm_compute_batch(sysctl_overcommit_memory);
     174             :                 break;
     175             :         default:
     176             :                 break;
     177             :         }
     178             :         return NOTIFY_OK;
     179             : }
     180             : 
     181             : static int __init mm_compute_batch_init(void)
     182             : {
     183             :         mm_compute_batch(sysctl_overcommit_memory);
     184             :         hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI);
     185             :         return 0;
     186             : }
     187             : 
     188             : __initcall(mm_compute_batch_init);
     189             : 
     190             : #endif
     191             : 
     192           1 : static int __init mm_sysfs_init(void)
     193             : {
     194           1 :         mm_kobj = kobject_create_and_add("mm", kernel_kobj);
     195           1 :         if (!mm_kobj)
     196             :                 return -ENOMEM;
     197             : 
     198           1 :         return 0;
     199             : }
     200             : postcore_initcall(mm_sysfs_init);