Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Manage cache of swap slots to be used for and returned from
       4             :  * swap.
       5             :  *
       6             :  * Copyright(c) 2016 Intel Corporation.
       7             :  *
       8             :  * Author: Tim Chen <tim.c.chen@linux.intel.com>
       9             :  *
      10             :  * We allocate the swap slots from the global pool and put
      11             :  * it into local per cpu caches.  This has the advantage
      12             :  * of no needing to acquire the swap_info lock every time
      13             :  * we need a new slot.
      14             :  *
      15             :  * There is also opportunity to simply return the slot
      16             :  * to local caches without needing to acquire swap_info
      17             :  * lock.  We do not reuse the returned slots directly but
      18             :  * move them back to the global pool in a batch.  This
      19             :  * allows the slots to coalesce and reduce fragmentation.
      20             :  *
      21             :  * The swap entry allocated is marked with SWAP_HAS_CACHE
      22             :  * flag in map_count that prevents it from being allocated
      23             :  * again from the global pool.
      24             :  *
      25             :  * The swap slots cache is protected by a mutex instead of
      26             :  * a spin lock as when we search for slots with scan_swap_map,
      27             :  * we can possibly sleep.
      28             :  */
      29             : 
      30             : #include <linux/swap_slots.h>
      31             : #include <linux/cpu.h>
      32             : #include <linux/cpumask.h>
      33             : #include <linux/slab.h>
      34             : #include <linux/vmalloc.h>
      35             : #include <linux/mutex.h>
      36             : #include <linux/mm.h>
      37             : 
      38             : static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
      39             : static bool     swap_slot_cache_active;
      40             : bool    swap_slot_cache_enabled;
      41             : static bool     swap_slot_cache_initialized;
      42             : static DEFINE_MUTEX(swap_slots_cache_mutex);
      43             : /* Serialize swap slots cache enable/disable operations */
      44             : static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
      45             : 
      46             : static void __drain_swap_slots_cache(unsigned int type);
      47             : 
      48             : #define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
      49             : #define SLOTS_CACHE 0x1
      50             : #define SLOTS_CACHE_RET 0x2
      51             : 
      52           0 : static void deactivate_swap_slots_cache(void)
      53             : {
      54           0 :         mutex_lock(&swap_slots_cache_mutex);
      55           0 :         swap_slot_cache_active = false;
      56           0 :         __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
      57           0 :         mutex_unlock(&swap_slots_cache_mutex);
      58           0 : }
      59             : 
      60             : static void reactivate_swap_slots_cache(void)
      61             : {
      62           0 :         mutex_lock(&swap_slots_cache_mutex);
      63           0 :         swap_slot_cache_active = true;
      64           0 :         mutex_unlock(&swap_slots_cache_mutex);
      65             : }
      66             : 
      67             : /* Must not be called with cpu hot plug lock */
      68           0 : void disable_swap_slots_cache_lock(void)
      69             : {
      70           0 :         mutex_lock(&swap_slots_cache_enable_mutex);
      71           0 :         swap_slot_cache_enabled = false;
      72           0 :         if (swap_slot_cache_initialized) {
      73             :                 /* serialize with cpu hotplug operations */
      74             :                 cpus_read_lock();
      75             :                 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
      76             :                 cpus_read_unlock();
      77             :         }
      78           0 : }
      79             : 
      80             : static void __reenable_swap_slots_cache(void)
      81             : {
      82           0 :         swap_slot_cache_enabled = has_usable_swap();
      83             : }
      84             : 
      85           0 : void reenable_swap_slots_cache_unlock(void)
      86             : {
      87             :         __reenable_swap_slots_cache();
      88           0 :         mutex_unlock(&swap_slots_cache_enable_mutex);
      89           0 : }
      90             : 
      91           0 : static bool check_cache_active(void)
      92             : {
      93             :         long pages;
      94             : 
      95           0 :         if (!swap_slot_cache_enabled)
      96             :                 return false;
      97             : 
      98           0 :         pages = get_nr_swap_pages();
      99           0 :         if (!swap_slot_cache_active) {
     100           0 :                 if (pages > num_online_cpus() *
     101             :                     THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
     102             :                         reactivate_swap_slots_cache();
     103             :                 goto out;
     104             :         }
     105             : 
     106             :         /* if global pool of slot caches too low, deactivate cache */
     107           0 :         if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
     108           0 :                 deactivate_swap_slots_cache();
     109             : out:
     110           0 :         return swap_slot_cache_active;
     111             : }
     112             : 
     113           0 : static int alloc_swap_slot_cache(unsigned int cpu)
     114             : {
     115             :         struct swap_slots_cache *cache;
     116             :         swp_entry_t *slots, *slots_ret;
     117             : 
     118             :         /*
     119             :          * Do allocation outside swap_slots_cache_mutex
     120             :          * as kvzalloc could trigger reclaim and folio_alloc_swap,
     121             :          * which can lock swap_slots_cache_mutex.
     122             :          */
     123           0 :         slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
     124             :                          GFP_KERNEL);
     125           0 :         if (!slots)
     126             :                 return -ENOMEM;
     127             : 
     128           0 :         slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
     129             :                              GFP_KERNEL);
     130           0 :         if (!slots_ret) {
     131           0 :                 kvfree(slots);
     132           0 :                 return -ENOMEM;
     133             :         }
     134             : 
     135           0 :         mutex_lock(&swap_slots_cache_mutex);
     136           0 :         cache = &per_cpu(swp_slots, cpu);
     137           0 :         if (cache->slots || cache->slots_ret) {
     138             :                 /* cache already allocated */
     139           0 :                 mutex_unlock(&swap_slots_cache_mutex);
     140             : 
     141           0 :                 kvfree(slots);
     142           0 :                 kvfree(slots_ret);
     143             : 
     144           0 :                 return 0;
     145             :         }
     146             : 
     147           0 :         if (!cache->lock_initialized) {
     148           0 :                 mutex_init(&cache->alloc_lock);
     149           0 :                 spin_lock_init(&cache->free_lock);
     150           0 :                 cache->lock_initialized = true;
     151             :         }
     152           0 :         cache->nr = 0;
     153           0 :         cache->cur = 0;
     154           0 :         cache->n_ret = 0;
     155             :         /*
     156             :          * We initialized alloc_lock and free_lock earlier.  We use
     157             :          * !cache->slots or !cache->slots_ret to know if it is safe to acquire
     158             :          * the corresponding lock and use the cache.  Memory barrier below
     159             :          * ensures the assumption.
     160             :          */
     161           0 :         mb();
     162           0 :         cache->slots = slots;
     163           0 :         cache->slots_ret = slots_ret;
     164           0 :         mutex_unlock(&swap_slots_cache_mutex);
     165           0 :         return 0;
     166             : }
     167             : 
     168           0 : static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
     169             :                                   bool free_slots)
     170             : {
     171             :         struct swap_slots_cache *cache;
     172           0 :         swp_entry_t *slots = NULL;
     173             : 
     174           0 :         cache = &per_cpu(swp_slots, cpu);
     175           0 :         if ((type & SLOTS_CACHE) && cache->slots) {
     176           0 :                 mutex_lock(&cache->alloc_lock);
     177           0 :                 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
     178           0 :                 cache->cur = 0;
     179           0 :                 cache->nr = 0;
     180           0 :                 if (free_slots && cache->slots) {
     181           0 :                         kvfree(cache->slots);
     182           0 :                         cache->slots = NULL;
     183             :                 }
     184           0 :                 mutex_unlock(&cache->alloc_lock);
     185             :         }
     186           0 :         if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
     187           0 :                 spin_lock_irq(&cache->free_lock);
     188           0 :                 swapcache_free_entries(cache->slots_ret, cache->n_ret);
     189           0 :                 cache->n_ret = 0;
     190           0 :                 if (free_slots && cache->slots_ret) {
     191           0 :                         slots = cache->slots_ret;
     192           0 :                         cache->slots_ret = NULL;
     193             :                 }
     194           0 :                 spin_unlock_irq(&cache->free_lock);
     195           0 :                 kvfree(slots);
     196             :         }
     197           0 : }
     198             : 
     199             : static void __drain_swap_slots_cache(unsigned int type)
     200             : {
     201             :         unsigned int cpu;
     202             : 
     203             :         /*
     204             :          * This function is called during
     205             :          *      1) swapoff, when we have to make sure no
     206             :          *         left over slots are in cache when we remove
     207             :          *         a swap device;
     208             :          *      2) disabling of swap slot cache, when we run low
     209             :          *         on swap slots when allocating memory and need
     210             :          *         to return swap slots to global pool.
     211             :          *
     212             :          * We cannot acquire cpu hot plug lock here as
     213             :          * this function can be invoked in the cpu
     214             :          * hot plug path:
     215             :          * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
     216             :          *   -> memory allocation -> direct reclaim -> folio_alloc_swap
     217             :          *   -> drain_swap_slots_cache
     218             :          *
     219             :          * Hence the loop over current online cpu below could miss cpu that
     220             :          * is being brought online but not yet marked as online.
     221             :          * That is okay as we do not schedule and run anything on a
     222             :          * cpu before it has been marked online. Hence, we will not
     223             :          * fill any swap slots in slots cache of such cpu.
     224             :          * There are no slots on such cpu that need to be drained.
     225             :          */
     226           0 :         for_each_online_cpu(cpu)
     227           0 :                 drain_slots_cache_cpu(cpu, type, false);
     228             : }
     229             : 
     230           0 : static int free_slot_cache(unsigned int cpu)
     231             : {
     232           0 :         mutex_lock(&swap_slots_cache_mutex);
     233           0 :         drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
     234           0 :         mutex_unlock(&swap_slots_cache_mutex);
     235           0 :         return 0;
     236             : }
     237             : 
     238           0 : void enable_swap_slots_cache(void)
     239             : {
     240           0 :         mutex_lock(&swap_slots_cache_enable_mutex);
     241           0 :         if (!swap_slot_cache_initialized) {
     242             :                 int ret;
     243             : 
     244           0 :                 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
     245             :                                         alloc_swap_slot_cache, free_slot_cache);
     246           0 :                 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
     247             :                                        "without swap slots cache.\n", __func__))
     248             :                         goto out_unlock;
     249             : 
     250           0 :                 swap_slot_cache_initialized = true;
     251             :         }
     252             : 
     253             :         __reenable_swap_slots_cache();
     254             : out_unlock:
     255           0 :         mutex_unlock(&swap_slots_cache_enable_mutex);
     256           0 : }
     257             : 
     258             : /* called with swap slot cache's alloc lock held */
     259           0 : static int refill_swap_slots_cache(struct swap_slots_cache *cache)
     260             : {
     261           0 :         if (!use_swap_slot_cache)
     262             :                 return 0;
     263             : 
     264           0 :         cache->cur = 0;
     265             :         if (swap_slot_cache_active)
     266           0 :                 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
     267             :                                            cache->slots, 1);
     268             : 
     269           0 :         return cache->nr;
     270             : }
     271             : 
     272           0 : void free_swap_slot(swp_entry_t entry)
     273             : {
     274             :         struct swap_slots_cache *cache;
     275             : 
     276           0 :         cache = raw_cpu_ptr(&swp_slots);
     277           0 :         if (likely(use_swap_slot_cache && cache->slots_ret)) {
     278           0 :                 spin_lock_irq(&cache->free_lock);
     279             :                 /* Swap slots cache may be deactivated before acquiring lock */
     280           0 :                 if (!use_swap_slot_cache || !cache->slots_ret) {
     281           0 :                         spin_unlock_irq(&cache->free_lock);
     282             :                         goto direct_free;
     283             :                 }
     284           0 :                 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
     285             :                         /*
     286             :                          * Return slots to global pool.
     287             :                          * The current swap_map value is SWAP_HAS_CACHE.
     288             :                          * Set it to 0 to indicate it is available for
     289             :                          * allocation in global pool
     290             :                          */
     291           0 :                         swapcache_free_entries(cache->slots_ret, cache->n_ret);
     292           0 :                         cache->n_ret = 0;
     293             :                 }
     294           0 :                 cache->slots_ret[cache->n_ret++] = entry;
     295           0 :                 spin_unlock_irq(&cache->free_lock);
     296             :         } else {
     297             : direct_free:
     298           0 :                 swapcache_free_entries(&entry, 1);
     299             :         }
     300           0 : }
     301             : 
     302           0 : swp_entry_t folio_alloc_swap(struct folio *folio)
     303             : {
     304             :         swp_entry_t entry;
     305             :         struct swap_slots_cache *cache;
     306             : 
     307           0 :         entry.val = 0;
     308             : 
     309           0 :         if (folio_test_large(folio)) {
     310             :                 if (IS_ENABLED(CONFIG_THP_SWAP) && arch_thp_swp_supported())
     311             :                         get_swap_pages(1, &entry, folio_nr_pages(folio));
     312             :                 goto out;
     313             :         }
     314             : 
     315             :         /*
     316             :          * Preemption is allowed here, because we may sleep
     317             :          * in refill_swap_slots_cache().  But it is safe, because
     318             :          * accesses to the per-CPU data structure are protected by the
     319             :          * mutex cache->alloc_lock.
     320             :          *
     321             :          * The alloc path here does not touch cache->slots_ret
     322             :          * so cache->free_lock is not taken.
     323             :          */
     324           0 :         cache = raw_cpu_ptr(&swp_slots);
     325             : 
     326           0 :         if (likely(check_cache_active() && cache->slots)) {
     327           0 :                 mutex_lock(&cache->alloc_lock);
     328           0 :                 if (cache->slots) {
     329             : repeat:
     330           0 :                         if (cache->nr) {
     331           0 :                                 entry = cache->slots[cache->cur];
     332           0 :                                 cache->slots[cache->cur++].val = 0;
     333           0 :                                 cache->nr--;
     334           0 :                         } else if (refill_swap_slots_cache(cache)) {
     335             :                                 goto repeat;
     336             :                         }
     337             :                 }
     338           0 :                 mutex_unlock(&cache->alloc_lock);
     339           0 :                 if (entry.val)
     340             :                         goto out;
     341             :         }
     342             : 
     343           0 :         get_swap_pages(1, &entry, 1);
     344             : out:
     345           0 :         if (mem_cgroup_try_charge_swap(folio, entry)) {
     346             :                 put_swap_folio(folio, entry);
     347             :                 entry.val = 0;
     348             :         }
     349           0 :         return entry;
     350             : }