Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */ 2 : #ifndef _MM_PERCPU_INTERNAL_H 3 : #define _MM_PERCPU_INTERNAL_H 4 : 5 : #include <linux/types.h> 6 : #include <linux/percpu.h> 7 : #include <linux/memcontrol.h> 8 : 9 : /* 10 : * pcpu_block_md is the metadata block struct. 11 : * Each chunk's bitmap is split into a number of full blocks. 12 : * All units are in terms of bits. 13 : * 14 : * The scan hint is the largest known contiguous area before the contig hint. 15 : * It is not necessarily the actual largest contig hint though. There is an 16 : * invariant that the scan_hint_start > contig_hint_start iff 17 : * scan_hint == contig_hint. This is necessary because when scanning forward, 18 : * we don't know if a new contig hint would be better than the current one. 19 : */ 20 : struct pcpu_block_md { 21 : int scan_hint; /* scan hint for block */ 22 : int scan_hint_start; /* block relative starting 23 : position of the scan hint */ 24 : int contig_hint; /* contig hint for block */ 25 : int contig_hint_start; /* block relative starting 26 : position of the contig hint */ 27 : int left_free; /* size of free space along 28 : the left side of the block */ 29 : int right_free; /* size of free space along 30 : the right side of the block */ 31 : int first_free; /* block position of first free */ 32 : int nr_bits; /* total bits responsible for */ 33 : }; 34 : 35 : struct pcpu_chunk { 36 : #ifdef CONFIG_PERCPU_STATS 37 : int nr_alloc; /* # of allocations */ 38 : size_t max_alloc_size; /* largest allocation size */ 39 : #endif 40 : 41 : struct list_head list; /* linked to pcpu_slot lists */ 42 : int free_bytes; /* free bytes in the chunk */ 43 : struct pcpu_block_md chunk_md; 44 : unsigned long *bound_map; /* boundary map */ 45 : 46 : /* 47 : * base_addr is the base address of this chunk. 48 : * To reduce false sharing, current layout is optimized to make sure 49 : * base_addr locate in the different cacheline with free_bytes and 50 : * chunk_md. 51 : */ 52 : void *base_addr ____cacheline_aligned_in_smp; 53 : 54 : unsigned long *alloc_map; /* allocation map */ 55 : struct pcpu_block_md *md_blocks; /* metadata blocks */ 56 : 57 : void *data; /* chunk data */ 58 : bool immutable; /* no [de]population allowed */ 59 : bool isolated; /* isolated from active chunk 60 : slots */ 61 : int start_offset; /* the overlap with the previous 62 : region to have a page aligned 63 : base_addr */ 64 : int end_offset; /* additional area required to 65 : have the region end page 66 : aligned */ 67 : #ifdef CONFIG_MEMCG_KMEM 68 : struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 69 : #endif 70 : 71 : int nr_pages; /* # of pages served by this chunk */ 72 : int nr_populated; /* # of populated pages */ 73 : int nr_empty_pop_pages; /* # of empty populated pages */ 74 : unsigned long populated[]; /* populated bitmap */ 75 : }; 76 : 77 : extern spinlock_t pcpu_lock; 78 : 79 : extern struct list_head *pcpu_chunk_lists; 80 : extern int pcpu_nr_slots; 81 : extern int pcpu_sidelined_slot; 82 : extern int pcpu_to_depopulate_slot; 83 : extern int pcpu_nr_empty_pop_pages; 84 : 85 : extern struct pcpu_chunk *pcpu_first_chunk; 86 : extern struct pcpu_chunk *pcpu_reserved_chunk; 87 : 88 : /** 89 : * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 90 : * @chunk: chunk of interest 91 : * 92 : * This conversion is from the number of physical pages that the chunk 93 : * serves to the number of bitmap blocks used. 94 : */ 95 : static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 96 : { 97 2078 : return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 98 : } 99 : 100 : /** 101 : * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 102 : * @pages: number of physical pages 103 : * 104 : * This conversion is from physical pages to the number of bits 105 : * required in the bitmap. 106 : */ 107 : static inline int pcpu_nr_pages_to_map_bits(int pages) 108 : { 109 1314 : return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 110 : } 111 : 112 : /** 113 : * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 114 : * @chunk: chunk of interest 115 : * 116 : * This conversion is from the number of physical pages that the chunk 117 : * serves to the number of bits in the bitmap. 118 : */ 119 : static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 120 : { 121 2628 : return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 122 : } 123 : 124 : /** 125 : * pcpu_obj_full_size - helper to calculate size of each accounted object 126 : * @size: size of area to allocate in bytes 127 : * 128 : * For each accounted object there is an extra space which is used to store 129 : * obj_cgroup membership if kmemcg is not disabled. Charge it too. 130 : */ 131 : static inline size_t pcpu_obj_full_size(size_t size) 132 : { 133 242 : size_t extra_size = 0; 134 : 135 : #ifdef CONFIG_MEMCG_KMEM 136 : if (!mem_cgroup_kmem_disabled()) 137 : extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *); 138 : #endif 139 : 140 : return size * num_possible_cpus() + extra_size; 141 : } 142 : 143 : #ifdef CONFIG_PERCPU_STATS 144 : 145 : #include <linux/spinlock.h> 146 : 147 : struct percpu_stats { 148 : u64 nr_alloc; /* lifetime # of allocations */ 149 : u64 nr_dealloc; /* lifetime # of deallocations */ 150 : u64 nr_cur_alloc; /* current # of allocations */ 151 : u64 nr_max_alloc; /* max # of live allocations */ 152 : u32 nr_chunks; /* current # of live chunks */ 153 : u32 nr_max_chunks; /* max # of live chunks */ 154 : size_t min_alloc_size; /* min allocation size */ 155 : size_t max_alloc_size; /* max allocation size */ 156 : }; 157 : 158 : extern struct percpu_stats pcpu_stats; 159 : extern struct pcpu_alloc_info pcpu_stats_ai; 160 : 161 : /* 162 : * For debug purposes. We don't care about the flexible array. 163 : */ 164 : static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 165 : { 166 : memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 167 : 168 : /* initialize min_alloc_size to unit_size */ 169 : pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 170 : } 171 : 172 : /* 173 : * pcpu_stats_area_alloc - increment area allocation stats 174 : * @chunk: the location of the area being allocated 175 : * @size: size of area to allocate in bytes 176 : * 177 : * CONTEXT: 178 : * pcpu_lock. 179 : */ 180 : static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 181 : { 182 : lockdep_assert_held(&pcpu_lock); 183 : 184 : pcpu_stats.nr_alloc++; 185 : pcpu_stats.nr_cur_alloc++; 186 : pcpu_stats.nr_max_alloc = 187 : max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 188 : pcpu_stats.min_alloc_size = 189 : min(pcpu_stats.min_alloc_size, size); 190 : pcpu_stats.max_alloc_size = 191 : max(pcpu_stats.max_alloc_size, size); 192 : 193 : chunk->nr_alloc++; 194 : chunk->max_alloc_size = max(chunk->max_alloc_size, size); 195 : } 196 : 197 : /* 198 : * pcpu_stats_area_dealloc - decrement allocation stats 199 : * @chunk: the location of the area being deallocated 200 : * 201 : * CONTEXT: 202 : * pcpu_lock. 203 : */ 204 : static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 205 : { 206 : lockdep_assert_held(&pcpu_lock); 207 : 208 : pcpu_stats.nr_dealloc++; 209 : pcpu_stats.nr_cur_alloc--; 210 : 211 : chunk->nr_alloc--; 212 : } 213 : 214 : /* 215 : * pcpu_stats_chunk_alloc - increment chunk stats 216 : */ 217 : static inline void pcpu_stats_chunk_alloc(void) 218 : { 219 : unsigned long flags; 220 : spin_lock_irqsave(&pcpu_lock, flags); 221 : 222 : pcpu_stats.nr_chunks++; 223 : pcpu_stats.nr_max_chunks = 224 : max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 225 : 226 : spin_unlock_irqrestore(&pcpu_lock, flags); 227 : } 228 : 229 : /* 230 : * pcpu_stats_chunk_dealloc - decrement chunk stats 231 : */ 232 : static inline void pcpu_stats_chunk_dealloc(void) 233 : { 234 : unsigned long flags; 235 : spin_lock_irqsave(&pcpu_lock, flags); 236 : 237 : pcpu_stats.nr_chunks--; 238 : 239 : spin_unlock_irqrestore(&pcpu_lock, flags); 240 : } 241 : 242 : #else 243 : 244 : static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 245 : { 246 : } 247 : 248 : static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 249 : { 250 : } 251 : 252 : static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 253 : { 254 : } 255 : 256 : static inline void pcpu_stats_chunk_alloc(void) 257 : { 258 : } 259 : 260 : static inline void pcpu_stats_chunk_dealloc(void) 261 : { 262 : } 263 : 264 : #endif /* !CONFIG_PERCPU_STATS */ 265 : 266 : #endif
