LCOV - code coverage report
Current view: top level - include/linux - pagemap.h (source / functions) Hit Total Coverage
Test: coverage.info Lines: 5 111 4.5 %
Date: 2023-04-06 08:38:28 Functions: 0 11 0.0 %

          Line data    Source code
       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _LINUX_PAGEMAP_H
       3             : #define _LINUX_PAGEMAP_H
       4             : 
       5             : /*
       6             :  * Copyright 1995 Linus Torvalds
       7             :  */
       8             : #include <linux/mm.h>
       9             : #include <linux/fs.h>
      10             : #include <linux/list.h>
      11             : #include <linux/highmem.h>
      12             : #include <linux/compiler.h>
      13             : #include <linux/uaccess.h>
      14             : #include <linux/gfp.h>
      15             : #include <linux/bitops.h>
      16             : #include <linux/hardirq.h> /* for in_interrupt() */
      17             : #include <linux/hugetlb_inline.h>
      18             : 
      19             : struct folio_batch;
      20             : 
      21             : unsigned long invalidate_mapping_pages(struct address_space *mapping,
      22             :                                         pgoff_t start, pgoff_t end);
      23             : 
      24             : static inline void invalidate_remote_inode(struct inode *inode)
      25             : {
      26             :         if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
      27             :             S_ISLNK(inode->i_mode))
      28             :                 invalidate_mapping_pages(inode->i_mapping, 0, -1);
      29             : }
      30             : int invalidate_inode_pages2(struct address_space *mapping);
      31             : int invalidate_inode_pages2_range(struct address_space *mapping,
      32             :                 pgoff_t start, pgoff_t end);
      33             : int write_inode_now(struct inode *, int sync);
      34             : int filemap_fdatawrite(struct address_space *);
      35             : int filemap_flush(struct address_space *);
      36             : int filemap_fdatawait_keep_errors(struct address_space *mapping);
      37             : int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
      38             : int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
      39             :                 loff_t start_byte, loff_t end_byte);
      40             : 
      41             : static inline int filemap_fdatawait(struct address_space *mapping)
      42             : {
      43           0 :         return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
      44             : }
      45             : 
      46             : bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
      47             : int filemap_write_and_wait_range(struct address_space *mapping,
      48             :                 loff_t lstart, loff_t lend);
      49             : int __filemap_fdatawrite_range(struct address_space *mapping,
      50             :                 loff_t start, loff_t end, int sync_mode);
      51             : int filemap_fdatawrite_range(struct address_space *mapping,
      52             :                 loff_t start, loff_t end);
      53             : int filemap_check_errors(struct address_space *mapping);
      54             : void __filemap_set_wb_err(struct address_space *mapping, int err);
      55             : int filemap_fdatawrite_wbc(struct address_space *mapping,
      56             :                            struct writeback_control *wbc);
      57             : 
      58             : static inline int filemap_write_and_wait(struct address_space *mapping)
      59             : {
      60           0 :         return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
      61             : }
      62             : 
      63             : /**
      64             :  * filemap_set_wb_err - set a writeback error on an address_space
      65             :  * @mapping: mapping in which to set writeback error
      66             :  * @err: error to be set in mapping
      67             :  *
      68             :  * When writeback fails in some way, we must record that error so that
      69             :  * userspace can be informed when fsync and the like are called.  We endeavor
      70             :  * to report errors on any file that was open at the time of the error.  Some
      71             :  * internal callers also need to know when writeback errors have occurred.
      72             :  *
      73             :  * When a writeback error occurs, most filesystems will want to call
      74             :  * filemap_set_wb_err to record the error in the mapping so that it will be
      75             :  * automatically reported whenever fsync is called on the file.
      76             :  */
      77             : static inline void filemap_set_wb_err(struct address_space *mapping, int err)
      78             : {
      79             :         /* Fastpath for common case of no error */
      80             :         if (unlikely(err))
      81             :                 __filemap_set_wb_err(mapping, err);
      82             : }
      83             : 
      84             : /**
      85             :  * filemap_check_wb_err - has an error occurred since the mark was sampled?
      86             :  * @mapping: mapping to check for writeback errors
      87             :  * @since: previously-sampled errseq_t
      88             :  *
      89             :  * Grab the errseq_t value from the mapping, and see if it has changed "since"
      90             :  * the given value was sampled.
      91             :  *
      92             :  * If it has then report the latest error set, otherwise return 0.
      93             :  */
      94             : static inline int filemap_check_wb_err(struct address_space *mapping,
      95             :                                         errseq_t since)
      96             : {
      97             :         return errseq_check(&mapping->wb_err, since);
      98             : }
      99             : 
     100             : /**
     101             :  * filemap_sample_wb_err - sample the current errseq_t to test for later errors
     102             :  * @mapping: mapping to be sampled
     103             :  *
     104             :  * Writeback errors are always reported relative to a particular sample point
     105             :  * in the past. This function provides those sample points.
     106             :  */
     107             : static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
     108             : {
     109           0 :         return errseq_sample(&mapping->wb_err);
     110             : }
     111             : 
     112             : /**
     113             :  * file_sample_sb_err - sample the current errseq_t to test for later errors
     114             :  * @file: file pointer to be sampled
     115             :  *
     116             :  * Grab the most current superblock-level errseq_t value for the given
     117             :  * struct file.
     118             :  */
     119             : static inline errseq_t file_sample_sb_err(struct file *file)
     120             : {
     121           0 :         return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
     122             : }
     123             : 
     124             : /*
     125             :  * Flush file data before changing attributes.  Caller must hold any locks
     126             :  * required to prevent further writes to this file until we're done setting
     127             :  * flags.
     128             :  */
     129           0 : static inline int inode_drain_writes(struct inode *inode)
     130             : {
     131           0 :         inode_dio_wait(inode);
     132           0 :         return filemap_write_and_wait(inode->i_mapping);
     133             : }
     134             : 
     135             : static inline bool mapping_empty(struct address_space *mapping)
     136             : {
     137          68 :         return xa_empty(&mapping->i_pages);
     138             : }
     139             : 
     140             : /*
     141             :  * mapping_shrinkable - test if page cache state allows inode reclaim
     142             :  * @mapping: the page cache mapping
     143             :  *
     144             :  * This checks the mapping's cache state for the pupose of inode
     145             :  * reclaim and LRU management.
     146             :  *
     147             :  * The caller is expected to hold the i_lock, but is not required to
     148             :  * hold the i_pages lock, which usually protects cache state. That's
     149             :  * because the i_lock and the list_lru lock that protect the inode and
     150             :  * its LRU state don't nest inside the irq-safe i_pages lock.
     151             :  *
     152             :  * Cache deletions are performed under the i_lock, which ensures that
     153             :  * when an inode goes empty, it will reliably get queued on the LRU.
     154             :  *
     155             :  * Cache additions do not acquire the i_lock and may race with this
     156             :  * check, in which case we'll report the inode as shrinkable when it
     157             :  * has cache pages. This is okay: the shrinker also checks the
     158             :  * refcount and the referenced bit, which will be elevated or set in
     159             :  * the process of adding new cache pages to an inode.
     160             :  */
     161             : static inline bool mapping_shrinkable(struct address_space *mapping)
     162             : {
     163             :         void *head;
     164             : 
     165             :         /*
     166             :          * On highmem systems, there could be lowmem pressure from the
     167             :          * inodes before there is highmem pressure from the page
     168             :          * cache. Make inodes shrinkable regardless of cache state.
     169             :          */
     170             :         if (IS_ENABLED(CONFIG_HIGHMEM))
     171             :                 return true;
     172             : 
     173             :         /* Cache completely empty? Shrink away. */
     174           0 :         head = rcu_access_pointer(mapping->i_pages.xa_head);
     175           0 :         if (!head)
     176             :                 return true;
     177             : 
     178             :         /*
     179             :          * The xarray stores single offset-0 entries directly in the
     180             :          * head pointer, which allows non-resident page cache entries
     181             :          * to escape the shadow shrinker's list of xarray nodes. The
     182             :          * inode shrinker needs to pick them up under memory pressure.
     183             :          */
     184           0 :         if (!xa_is_node(head) && xa_is_value(head))
     185             :                 return true;
     186             : 
     187             :         return false;
     188             : }
     189             : 
     190             : /*
     191             :  * Bits in mapping->flags.
     192             :  */
     193             : enum mapping_flags {
     194             :         AS_EIO          = 0,    /* IO error on async write */
     195             :         AS_ENOSPC       = 1,    /* ENOSPC on async write */
     196             :         AS_MM_ALL_LOCKS = 2,    /* under mm_take_all_locks() */
     197             :         AS_UNEVICTABLE  = 3,    /* e.g., ramdisk, SHM_LOCK */
     198             :         AS_EXITING      = 4,    /* final truncate in progress */
     199             :         /* writeback related tags are not used */
     200             :         AS_NO_WRITEBACK_TAGS = 5,
     201             :         AS_LARGE_FOLIO_SUPPORT = 6,
     202             : };
     203             : 
     204             : /**
     205             :  * mapping_set_error - record a writeback error in the address_space
     206             :  * @mapping: the mapping in which an error should be set
     207             :  * @error: the error to set in the mapping
     208             :  *
     209             :  * When writeback fails in some way, we must record that error so that
     210             :  * userspace can be informed when fsync and the like are called.  We endeavor
     211             :  * to report errors on any file that was open at the time of the error.  Some
     212             :  * internal callers also need to know when writeback errors have occurred.
     213             :  *
     214             :  * When a writeback error occurs, most filesystems will want to call
     215             :  * mapping_set_error to record the error in the mapping so that it can be
     216             :  * reported when the application calls fsync(2).
     217             :  */
     218           0 : static inline void mapping_set_error(struct address_space *mapping, int error)
     219             : {
     220           0 :         if (likely(!error))
     221             :                 return;
     222             : 
     223             :         /* Record in wb_err for checkers using errseq_t based tracking */
     224           0 :         __filemap_set_wb_err(mapping, error);
     225             : 
     226             :         /* Record it in superblock */
     227           0 :         if (mapping->host)
     228           0 :                 errseq_set(&mapping->host->i_sb->s_wb_err, error);
     229             : 
     230             :         /* Record it in flags for now, for legacy callers */
     231           0 :         if (error == -ENOSPC)
     232           0 :                 set_bit(AS_ENOSPC, &mapping->flags);
     233             :         else
     234           0 :                 set_bit(AS_EIO, &mapping->flags);
     235             : }
     236             : 
     237             : static inline void mapping_set_unevictable(struct address_space *mapping)
     238             : {
     239           8 :         set_bit(AS_UNEVICTABLE, &mapping->flags);
     240             : }
     241             : 
     242             : static inline void mapping_clear_unevictable(struct address_space *mapping)
     243             : {
     244           0 :         clear_bit(AS_UNEVICTABLE, &mapping->flags);
     245             : }
     246             : 
     247             : static inline bool mapping_unevictable(struct address_space *mapping)
     248             : {
     249           0 :         return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);
     250             : }
     251             : 
     252             : static inline void mapping_set_exiting(struct address_space *mapping)
     253             : {
     254          68 :         set_bit(AS_EXITING, &mapping->flags);
     255             : }
     256             : 
     257             : static inline int mapping_exiting(struct address_space *mapping)
     258             : {
     259           0 :         return test_bit(AS_EXITING, &mapping->flags);
     260             : }
     261             : 
     262             : static inline void mapping_set_no_writeback_tags(struct address_space *mapping)
     263             : {
     264           0 :         set_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);
     265             : }
     266             : 
     267             : static inline int mapping_use_writeback_tags(struct address_space *mapping)
     268             : {
     269           0 :         return !test_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);
     270             : }
     271             : 
     272             : static inline gfp_t mapping_gfp_mask(struct address_space * mapping)
     273             : {
     274             :         return mapping->gfp_mask;
     275             : }
     276             : 
     277             : /* Restricts the given gfp_mask to what the mapping allows. */
     278             : static inline gfp_t mapping_gfp_constraint(struct address_space *mapping,
     279             :                 gfp_t gfp_mask)
     280             : {
     281           0 :         return mapping_gfp_mask(mapping) & gfp_mask;
     282             : }
     283             : 
     284             : /*
     285             :  * This is non-atomic.  Only to be used before the mapping is activated.
     286             :  * Probably needs a barrier...
     287             :  */
     288             : static inline void mapping_set_gfp_mask(struct address_space *m, gfp_t mask)
     289             : {
     290          55 :         m->gfp_mask = mask;
     291             : }
     292             : 
     293             : /**
     294             :  * mapping_set_large_folios() - Indicate the file supports large folios.
     295             :  * @mapping: The file.
     296             :  *
     297             :  * The filesystem should call this function in its inode constructor to
     298             :  * indicate that the VFS can use large folios to cache the contents of
     299             :  * the file.
     300             :  *
     301             :  * Context: This should not be called while the inode is active as it
     302             :  * is non-atomic.
     303             :  */
     304             : static inline void mapping_set_large_folios(struct address_space *mapping)
     305             : {
     306           2 :         __set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
     307             : }
     308             : 
     309             : /*
     310             :  * Large folio support currently depends on THP.  These dependencies are
     311             :  * being worked on but are not yet fixed.
     312             :  */
     313             : static inline bool mapping_large_folio_support(struct address_space *mapping)
     314             : {
     315             :         return IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
     316             :                 test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
     317             : }
     318             : 
     319             : static inline int filemap_nr_thps(struct address_space *mapping)
     320             : {
     321             : #ifdef CONFIG_READ_ONLY_THP_FOR_FS
     322             :         return atomic_read(&mapping->nr_thps);
     323             : #else
     324             :         return 0;
     325             : #endif
     326             : }
     327             : 
     328             : static inline void filemap_nr_thps_inc(struct address_space *mapping)
     329             : {
     330             : #ifdef CONFIG_READ_ONLY_THP_FOR_FS
     331             :         if (!mapping_large_folio_support(mapping))
     332             :                 atomic_inc(&mapping->nr_thps);
     333             : #else
     334             :         WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
     335             : #endif
     336             : }
     337             : 
     338             : static inline void filemap_nr_thps_dec(struct address_space *mapping)
     339             : {
     340             : #ifdef CONFIG_READ_ONLY_THP_FOR_FS
     341             :         if (!mapping_large_folio_support(mapping))
     342             :                 atomic_dec(&mapping->nr_thps);
     343             : #else
     344             :         WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
     345             : #endif
     346             : }
     347             : 
     348             : struct address_space *page_mapping(struct page *);
     349             : struct address_space *folio_mapping(struct folio *);
     350             : struct address_space *swapcache_mapping(struct folio *);
     351             : 
     352             : /**
     353             :  * folio_file_mapping - Find the mapping this folio belongs to.
     354             :  * @folio: The folio.
     355             :  *
     356             :  * For folios which are in the page cache, return the mapping that this
     357             :  * page belongs to.  Folios in the swap cache return the mapping of the
     358             :  * swap file or swap device where the data is stored.  This is different
     359             :  * from the mapping returned by folio_mapping().  The only reason to
     360             :  * use it is if, like NFS, you return 0 from ->activate_swapfile.
     361             :  *
     362             :  * Do not call this for folios which aren't in the page cache or swap cache.
     363             :  */
     364             : static inline struct address_space *folio_file_mapping(struct folio *folio)
     365             : {
     366             :         if (unlikely(folio_test_swapcache(folio)))
     367             :                 return swapcache_mapping(folio);
     368             : 
     369             :         return folio->mapping;
     370             : }
     371             : 
     372             : static inline struct address_space *page_file_mapping(struct page *page)
     373             : {
     374             :         return folio_file_mapping(page_folio(page));
     375             : }
     376             : 
     377             : /*
     378             :  * For file cache pages, return the address_space, otherwise return NULL
     379             :  */
     380             : static inline struct address_space *page_mapping_file(struct page *page)
     381             : {
     382             :         struct folio *folio = page_folio(page);
     383             : 
     384             :         if (unlikely(folio_test_swapcache(folio)))
     385             :                 return NULL;
     386             :         return folio_mapping(folio);
     387             : }
     388             : 
     389             : /**
     390             :  * folio_inode - Get the host inode for this folio.
     391             :  * @folio: The folio.
     392             :  *
     393             :  * For folios which are in the page cache, return the inode that this folio
     394             :  * belongs to.
     395             :  *
     396             :  * Do not call this for folios which aren't in the page cache.
     397             :  */
     398             : static inline struct inode *folio_inode(struct folio *folio)
     399             : {
     400           0 :         return folio->mapping->host;
     401             : }
     402             : 
     403             : /**
     404             :  * folio_attach_private - Attach private data to a folio.
     405             :  * @folio: Folio to attach data to.
     406             :  * @data: Data to attach to folio.
     407             :  *
     408             :  * Attaching private data to a folio increments the page's reference count.
     409             :  * The data must be detached before the folio will be freed.
     410             :  */
     411             : static inline void folio_attach_private(struct folio *folio, void *data)
     412             : {
     413           0 :         folio_get(folio);
     414           0 :         folio->private = data;
     415           0 :         folio_set_private(folio);
     416             : }
     417             : 
     418             : /**
     419             :  * folio_change_private - Change private data on a folio.
     420             :  * @folio: Folio to change the data on.
     421             :  * @data: Data to set on the folio.
     422             :  *
     423             :  * Change the private data attached to a folio and return the old
     424             :  * data.  The page must previously have had data attached and the data
     425             :  * must be detached before the folio will be freed.
     426             :  *
     427             :  * Return: Data that was previously attached to the folio.
     428             :  */
     429             : static inline void *folio_change_private(struct folio *folio, void *data)
     430             : {
     431             :         void *old = folio_get_private(folio);
     432             : 
     433             :         folio->private = data;
     434             :         return old;
     435             : }
     436             : 
     437             : /**
     438             :  * folio_detach_private - Detach private data from a folio.
     439             :  * @folio: Folio to detach data from.
     440             :  *
     441             :  * Removes the data that was previously attached to the folio and decrements
     442             :  * the refcount on the page.
     443             :  *
     444             :  * Return: Data that was attached to the folio.
     445             :  */
     446           0 : static inline void *folio_detach_private(struct folio *folio)
     447             : {
     448           0 :         void *data = folio_get_private(folio);
     449             : 
     450           0 :         if (!folio_test_private(folio))
     451             :                 return NULL;
     452           0 :         folio_clear_private(folio);
     453           0 :         folio->private = NULL;
     454             :         folio_put(folio);
     455             : 
     456             :         return data;
     457             : }
     458             : 
     459             : static inline void attach_page_private(struct page *page, void *data)
     460             : {
     461           0 :         folio_attach_private(page_folio(page), data);
     462             : }
     463             : 
     464             : static inline void *detach_page_private(struct page *page)
     465             : {
     466             :         return folio_detach_private(page_folio(page));
     467             : }
     468             : 
     469             : #ifdef CONFIG_NUMA
     470             : struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order);
     471             : #else
     472             : static inline struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order)
     473             : {
     474           0 :         return folio_alloc(gfp, order);
     475             : }
     476             : #endif
     477             : 
     478             : static inline struct page *__page_cache_alloc(gfp_t gfp)
     479             : {
     480             :         return &filemap_alloc_folio(gfp, 0)->page;
     481             : }
     482             : 
     483             : static inline struct page *page_cache_alloc(struct address_space *x)
     484             : {
     485             :         return __page_cache_alloc(mapping_gfp_mask(x));
     486             : }
     487             : 
     488             : static inline gfp_t readahead_gfp_mask(struct address_space *x)
     489             : {
     490           0 :         return mapping_gfp_mask(x) | __GFP_NORETRY | __GFP_NOWARN;
     491             : }
     492             : 
     493             : typedef int filler_t(struct file *, struct folio *);
     494             : 
     495             : pgoff_t page_cache_next_miss(struct address_space *mapping,
     496             :                              pgoff_t index, unsigned long max_scan);
     497             : pgoff_t page_cache_prev_miss(struct address_space *mapping,
     498             :                              pgoff_t index, unsigned long max_scan);
     499             : 
     500             : #define FGP_ACCESSED            0x00000001
     501             : #define FGP_LOCK                0x00000002
     502             : #define FGP_CREAT               0x00000004
     503             : #define FGP_WRITE               0x00000008
     504             : #define FGP_NOFS                0x00000010
     505             : #define FGP_NOWAIT              0x00000020
     506             : #define FGP_FOR_MMAP            0x00000040
     507             : #define FGP_ENTRY               0x00000080
     508             : #define FGP_STABLE              0x00000100
     509             : 
     510             : struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
     511             :                 int fgp_flags, gfp_t gfp);
     512             : struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
     513             :                 int fgp_flags, gfp_t gfp);
     514             : 
     515             : /**
     516             :  * filemap_get_folio - Find and get a folio.
     517             :  * @mapping: The address_space to search.
     518             :  * @index: The page index.
     519             :  *
     520             :  * Looks up the page cache entry at @mapping & @index.  If a folio is
     521             :  * present, it is returned with an increased refcount.
     522             :  *
     523             :  * Otherwise, %NULL is returned.
     524             :  */
     525             : static inline struct folio *filemap_get_folio(struct address_space *mapping,
     526             :                                         pgoff_t index)
     527             : {
     528           0 :         return __filemap_get_folio(mapping, index, 0, 0);
     529             : }
     530             : 
     531             : /**
     532             :  * filemap_lock_folio - Find and lock a folio.
     533             :  * @mapping: The address_space to search.
     534             :  * @index: The page index.
     535             :  *
     536             :  * Looks up the page cache entry at @mapping & @index.  If a folio is
     537             :  * present, it is returned locked with an increased refcount.
     538             :  *
     539             :  * Context: May sleep.
     540             :  * Return: A folio or %NULL if there is no folio in the cache for this
     541             :  * index.  Will not return a shadow, swap or DAX entry.
     542             :  */
     543             : static inline struct folio *filemap_lock_folio(struct address_space *mapping,
     544             :                                         pgoff_t index)
     545             : {
     546             :         return __filemap_get_folio(mapping, index, FGP_LOCK, 0);
     547             : }
     548             : 
     549             : /**
     550             :  * filemap_grab_folio - grab a folio from the page cache
     551             :  * @mapping: The address space to search
     552             :  * @index: The page index
     553             :  *
     554             :  * Looks up the page cache entry at @mapping & @index. If no folio is found,
     555             :  * a new folio is created. The folio is locked, marked as accessed, and
     556             :  * returned.
     557             :  *
     558             :  * Return: A found or created folio. NULL if no folio is found and failed to
     559             :  * create a folio.
     560             :  */
     561             : static inline struct folio *filemap_grab_folio(struct address_space *mapping,
     562             :                                         pgoff_t index)
     563             : {
     564             :         return __filemap_get_folio(mapping, index,
     565             :                         FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
     566             :                         mapping_gfp_mask(mapping));
     567             : }
     568             : 
     569             : /**
     570             :  * find_get_page - find and get a page reference
     571             :  * @mapping: the address_space to search
     572             :  * @offset: the page index
     573             :  *
     574             :  * Looks up the page cache slot at @mapping & @offset.  If there is a
     575             :  * page cache page, it is returned with an increased refcount.
     576             :  *
     577             :  * Otherwise, %NULL is returned.
     578             :  */
     579             : static inline struct page *find_get_page(struct address_space *mapping,
     580             :                                         pgoff_t offset)
     581             : {
     582           0 :         return pagecache_get_page(mapping, offset, 0, 0);
     583             : }
     584             : 
     585             : static inline struct page *find_get_page_flags(struct address_space *mapping,
     586             :                                         pgoff_t offset, int fgp_flags)
     587             : {
     588           0 :         return pagecache_get_page(mapping, offset, fgp_flags, 0);
     589             : }
     590             : 
     591             : /**
     592             :  * find_lock_page - locate, pin and lock a pagecache page
     593             :  * @mapping: the address_space to search
     594             :  * @index: the page index
     595             :  *
     596             :  * Looks up the page cache entry at @mapping & @index.  If there is a
     597             :  * page cache page, it is returned locked and with an increased
     598             :  * refcount.
     599             :  *
     600             :  * Context: May sleep.
     601             :  * Return: A struct page or %NULL if there is no page in the cache for this
     602             :  * index.
     603             :  */
     604             : static inline struct page *find_lock_page(struct address_space *mapping,
     605             :                                         pgoff_t index)
     606             : {
     607           0 :         return pagecache_get_page(mapping, index, FGP_LOCK, 0);
     608             : }
     609             : 
     610             : /**
     611             :  * find_or_create_page - locate or add a pagecache page
     612             :  * @mapping: the page's address_space
     613             :  * @index: the page's index into the mapping
     614             :  * @gfp_mask: page allocation mode
     615             :  *
     616             :  * Looks up the page cache slot at @mapping & @offset.  If there is a
     617             :  * page cache page, it is returned locked and with an increased
     618             :  * refcount.
     619             :  *
     620             :  * If the page is not present, a new page is allocated using @gfp_mask
     621             :  * and added to the page cache and the VM's LRU list.  The page is
     622             :  * returned locked and with an increased refcount.
     623             :  *
     624             :  * On memory exhaustion, %NULL is returned.
     625             :  *
     626             :  * find_or_create_page() may sleep, even if @gfp_flags specifies an
     627             :  * atomic allocation!
     628             :  */
     629             : static inline struct page *find_or_create_page(struct address_space *mapping,
     630             :                                         pgoff_t index, gfp_t gfp_mask)
     631             : {
     632           0 :         return pagecache_get_page(mapping, index,
     633             :                                         FGP_LOCK|FGP_ACCESSED|FGP_CREAT,
     634             :                                         gfp_mask);
     635             : }
     636             : 
     637             : /**
     638             :  * grab_cache_page_nowait - returns locked page at given index in given cache
     639             :  * @mapping: target address_space
     640             :  * @index: the page index
     641             :  *
     642             :  * Same as grab_cache_page(), but do not wait if the page is unavailable.
     643             :  * This is intended for speculative data generators, where the data can
     644             :  * be regenerated if the page couldn't be grabbed.  This routine should
     645             :  * be safe to call while holding the lock for another page.
     646             :  *
     647             :  * Clear __GFP_FS when allocating the page to avoid recursion into the fs
     648             :  * and deadlock against the caller's locked page.
     649             :  */
     650             : static inline struct page *grab_cache_page_nowait(struct address_space *mapping,
     651             :                                 pgoff_t index)
     652             : {
     653             :         return pagecache_get_page(mapping, index,
     654             :                         FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
     655             :                         mapping_gfp_mask(mapping));
     656             : }
     657             : 
     658             : #define swapcache_index(folio)  __page_file_index(&(folio)->page)
     659             : 
     660             : /**
     661             :  * folio_index - File index of a folio.
     662             :  * @folio: The folio.
     663             :  *
     664             :  * For a folio which is either in the page cache or the swap cache,
     665             :  * return its index within the address_space it belongs to.  If you know
     666             :  * the page is definitely in the page cache, you can look at the folio's
     667             :  * index directly.
     668             :  *
     669             :  * Return: The index (offset in units of pages) of a folio in its file.
     670             :  */
     671           0 : static inline pgoff_t folio_index(struct folio *folio)
     672             : {
     673           0 :         if (unlikely(folio_test_swapcache(folio)))
     674           0 :                 return swapcache_index(folio);
     675           0 :         return folio->index;
     676             : }
     677             : 
     678             : /**
     679             :  * folio_next_index - Get the index of the next folio.
     680             :  * @folio: The current folio.
     681             :  *
     682             :  * Return: The index of the folio which follows this folio in the file.
     683             :  */
     684             : static inline pgoff_t folio_next_index(struct folio *folio)
     685             : {
     686             :         return folio->index + folio_nr_pages(folio);
     687             : }
     688             : 
     689             : /**
     690             :  * folio_file_page - The page for a particular index.
     691             :  * @folio: The folio which contains this index.
     692             :  * @index: The index we want to look up.
     693             :  *
     694             :  * Sometimes after looking up a folio in the page cache, we need to
     695             :  * obtain the specific page for an index (eg a page fault).
     696             :  *
     697             :  * Return: The page containing the file data for this index.
     698             :  */
     699             : static inline struct page *folio_file_page(struct folio *folio, pgoff_t index)
     700             : {
     701             :         /* HugeTLBfs indexes the page cache in units of hpage_size */
     702           0 :         if (folio_test_hugetlb(folio))
     703             :                 return &folio->page;
     704           0 :         return folio_page(folio, index & (folio_nr_pages(folio) - 1));
     705             : }
     706             : 
     707             : /**
     708             :  * folio_contains - Does this folio contain this index?
     709             :  * @folio: The folio.
     710             :  * @index: The page index within the file.
     711             :  *
     712             :  * Context: The caller should have the page locked in order to prevent
     713             :  * (eg) shmem from moving the page between the page cache and swap cache
     714             :  * and changing its index in the middle of the operation.
     715             :  * Return: true or false.
     716             :  */
     717             : static inline bool folio_contains(struct folio *folio, pgoff_t index)
     718             : {
     719             :         /* HugeTLBfs indexes the page cache in units of hpage_size */
     720             :         if (folio_test_hugetlb(folio))
     721             :                 return folio->index == index;
     722             :         return index - folio_index(folio) < folio_nr_pages(folio);
     723             : }
     724             : 
     725             : /*
     726             :  * Given the page we found in the page cache, return the page corresponding
     727             :  * to this index in the file
     728             :  */
     729             : static inline struct page *find_subpage(struct page *head, pgoff_t index)
     730             : {
     731             :         /* HugeTLBfs wants the head page regardless */
     732           0 :         if (PageHuge(head))
     733             :                 return head;
     734             : 
     735           0 :         return head + (index & (thp_nr_pages(head) - 1));
     736             : }
     737             : 
     738             : unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
     739             :                 pgoff_t end, struct folio_batch *fbatch);
     740             : unsigned filemap_get_folios_contig(struct address_space *mapping,
     741             :                 pgoff_t *start, pgoff_t end, struct folio_batch *fbatch);
     742             : unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,
     743             :                 pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch);
     744             : 
     745             : struct page *grab_cache_page_write_begin(struct address_space *mapping,
     746             :                         pgoff_t index);
     747             : 
     748             : /*
     749             :  * Returns locked page at given index in given cache, creating it if needed.
     750             :  */
     751             : static inline struct page *grab_cache_page(struct address_space *mapping,
     752             :                                                                 pgoff_t index)
     753             : {
     754           0 :         return find_or_create_page(mapping, index, mapping_gfp_mask(mapping));
     755             : }
     756             : 
     757             : struct folio *read_cache_folio(struct address_space *, pgoff_t index,
     758             :                 filler_t *filler, struct file *file);
     759             : struct folio *mapping_read_folio_gfp(struct address_space *, pgoff_t index,
     760             :                 gfp_t flags);
     761             : struct page *read_cache_page(struct address_space *, pgoff_t index,
     762             :                 filler_t *filler, struct file *file);
     763             : extern struct page * read_cache_page_gfp(struct address_space *mapping,
     764             :                                 pgoff_t index, gfp_t gfp_mask);
     765             : 
     766             : static inline struct page *read_mapping_page(struct address_space *mapping,
     767             :                                 pgoff_t index, struct file *file)
     768             : {
     769           0 :         return read_cache_page(mapping, index, NULL, file);
     770             : }
     771             : 
     772             : static inline struct folio *read_mapping_folio(struct address_space *mapping,
     773             :                                 pgoff_t index, struct file *file)
     774             : {
     775           0 :         return read_cache_folio(mapping, index, NULL, file);
     776             : }
     777             : 
     778             : /*
     779             :  * Get index of the page within radix-tree (but not for hugetlb pages).
     780             :  * (TODO: remove once hugetlb pages will have ->index in PAGE_SIZE)
     781             :  */
     782             : static inline pgoff_t page_to_index(struct page *page)
     783             : {
     784             :         struct page *head;
     785             : 
     786           0 :         if (likely(!PageTransTail(page)))
     787             :                 return page->index;
     788             : 
     789             :         head = compound_head(page);
     790             :         /*
     791             :          *  We don't initialize ->index for tail pages: calculate based on
     792             :          *  head page
     793             :          */
     794             :         return head->index + page - head;
     795             : }
     796             : 
     797             : extern pgoff_t hugetlb_basepage_index(struct page *page);
     798             : 
     799             : /*
     800             :  * Get the offset in PAGE_SIZE (even for hugetlb pages).
     801             :  * (TODO: hugetlb pages should have ->index in PAGE_SIZE)
     802             :  */
     803             : static inline pgoff_t page_to_pgoff(struct page *page)
     804             : {
     805           0 :         if (unlikely(PageHuge(page)))
     806             :                 return hugetlb_basepage_index(page);
     807           0 :         return page_to_index(page);
     808             : }
     809             : 
     810             : /*
     811             :  * Return byte-offset into filesystem object for page.
     812             :  */
     813             : static inline loff_t page_offset(struct page *page)
     814             : {
     815           0 :         return ((loff_t)page->index) << PAGE_SHIFT;
     816             : }
     817             : 
     818             : static inline loff_t page_file_offset(struct page *page)
     819             : {
     820           0 :         return ((loff_t)page_index(page)) << PAGE_SHIFT;
     821             : }
     822             : 
     823             : /**
     824             :  * folio_pos - Returns the byte position of this folio in its file.
     825             :  * @folio: The folio.
     826             :  */
     827             : static inline loff_t folio_pos(struct folio *folio)
     828             : {
     829           0 :         return page_offset(&folio->page);
     830             : }
     831             : 
     832             : /**
     833             :  * folio_file_pos - Returns the byte position of this folio in its file.
     834             :  * @folio: The folio.
     835             :  *
     836             :  * This differs from folio_pos() for folios which belong to a swap file.
     837             :  * NFS is the only filesystem today which needs to use folio_file_pos().
     838             :  */
     839             : static inline loff_t folio_file_pos(struct folio *folio)
     840             : {
     841             :         return page_file_offset(&folio->page);
     842             : }
     843             : 
     844             : /*
     845             :  * Get the offset in PAGE_SIZE (even for hugetlb folios).
     846             :  * (TODO: hugetlb folios should have ->index in PAGE_SIZE)
     847             :  */
     848             : static inline pgoff_t folio_pgoff(struct folio *folio)
     849             : {
     850           0 :         if (unlikely(folio_test_hugetlb(folio)))
     851             :                 return hugetlb_basepage_index(&folio->page);
     852             :         return folio->index;
     853             : }
     854             : 
     855             : extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
     856             :                                      unsigned long address);
     857             : 
     858             : static inline pgoff_t linear_page_index(struct vm_area_struct *vma,
     859             :                                         unsigned long address)
     860             : {
     861             :         pgoff_t pgoff;
     862           0 :         if (unlikely(is_vm_hugetlb_page(vma)))
     863             :                 return linear_hugepage_index(vma, address);
     864           0 :         pgoff = (address - vma->vm_start) >> PAGE_SHIFT;
     865           0 :         pgoff += vma->vm_pgoff;
     866             :         return pgoff;
     867             : }
     868             : 
     869             : struct wait_page_key {
     870             :         struct folio *folio;
     871             :         int bit_nr;
     872             :         int page_match;
     873             : };
     874             : 
     875             : struct wait_page_queue {
     876             :         struct folio *folio;
     877             :         int bit_nr;
     878             :         wait_queue_entry_t wait;
     879             : };
     880             : 
     881             : static inline bool wake_page_match(struct wait_page_queue *wait_page,
     882             :                                   struct wait_page_key *key)
     883             : {
     884           0 :         if (wait_page->folio != key->folio)
     885             :                return false;
     886           0 :         key->page_match = 1;
     887             : 
     888           0 :         if (wait_page->bit_nr != key->bit_nr)
     889             :                 return false;
     890             : 
     891             :         return true;
     892             : }
     893             : 
     894             : void __folio_lock(struct folio *folio);
     895             : int __folio_lock_killable(struct folio *folio);
     896             : bool __folio_lock_or_retry(struct folio *folio, struct mm_struct *mm,
     897             :                                 unsigned int flags);
     898             : void unlock_page(struct page *page);
     899             : void folio_unlock(struct folio *folio);
     900             : 
     901             : /**
     902             :  * folio_trylock() - Attempt to lock a folio.
     903             :  * @folio: The folio to attempt to lock.
     904             :  *
     905             :  * Sometimes it is undesirable to wait for a folio to be unlocked (eg
     906             :  * when the locks are being taken in the wrong order, or if making
     907             :  * progress through a batch of folios is more important than processing
     908             :  * them in order).  Usually folio_lock() is the correct function to call.
     909             :  *
     910             :  * Context: Any context.
     911             :  * Return: Whether the lock was successfully acquired.
     912             :  */
     913             : static inline bool folio_trylock(struct folio *folio)
     914             : {
     915           0 :         return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0)));
     916             : }
     917             : 
     918             : /*
     919             :  * Return true if the page was successfully locked
     920             :  */
     921             : static inline int trylock_page(struct page *page)
     922             : {
     923           0 :         return folio_trylock(page_folio(page));
     924             : }
     925             : 
     926             : /**
     927             :  * folio_lock() - Lock this folio.
     928             :  * @folio: The folio to lock.
     929             :  *
     930             :  * The folio lock protects against many things, probably more than it
     931             :  * should.  It is primarily held while a folio is being brought uptodate,
     932             :  * either from its backing file or from swap.  It is also held while a
     933             :  * folio is being truncated from its address_space, so holding the lock
     934             :  * is sufficient to keep folio->mapping stable.
     935             :  *
     936             :  * The folio lock is also held while write() is modifying the page to
     937             :  * provide POSIX atomicity guarantees (as long as the write does not
     938             :  * cross a page boundary).  Other modifications to the data in the folio
     939             :  * do not hold the folio lock and can race with writes, eg DMA and stores
     940             :  * to mapped pages.
     941             :  *
     942             :  * Context: May sleep.  If you need to acquire the locks of two or
     943             :  * more folios, they must be in order of ascending index, if they are
     944             :  * in the same address_space.  If they are in different address_spaces,
     945             :  * acquire the lock of the folio which belongs to the address_space which
     946             :  * has the lowest address in memory first.
     947             :  */
     948           0 : static inline void folio_lock(struct folio *folio)
     949             : {
     950             :         might_sleep();
     951           0 :         if (!folio_trylock(folio))
     952           0 :                 __folio_lock(folio);
     953           0 : }
     954             : 
     955             : /**
     956             :  * lock_page() - Lock the folio containing this page.
     957             :  * @page: The page to lock.
     958             :  *
     959             :  * See folio_lock() for a description of what the lock protects.
     960             :  * This is a legacy function and new code should probably use folio_lock()
     961             :  * instead.
     962             :  *
     963             :  * Context: May sleep.  Pages in the same folio share a lock, so do not
     964             :  * attempt to lock two pages which share a folio.
     965             :  */
     966           0 : static inline void lock_page(struct page *page)
     967             : {
     968             :         struct folio *folio;
     969             :         might_sleep();
     970             : 
     971           0 :         folio = page_folio(page);
     972           0 :         if (!folio_trylock(folio))
     973           0 :                 __folio_lock(folio);
     974           0 : }
     975             : 
     976             : /**
     977             :  * folio_lock_killable() - Lock this folio, interruptible by a fatal signal.
     978             :  * @folio: The folio to lock.
     979             :  *
     980             :  * Attempts to lock the folio, like folio_lock(), except that the sleep
     981             :  * to acquire the lock is interruptible by a fatal signal.
     982             :  *
     983             :  * Context: May sleep; see folio_lock().
     984             :  * Return: 0 if the lock was acquired; -EINTR if a fatal signal was received.
     985             :  */
     986             : static inline int folio_lock_killable(struct folio *folio)
     987             : {
     988             :         might_sleep();
     989             :         if (!folio_trylock(folio))
     990             :                 return __folio_lock_killable(folio);
     991             :         return 0;
     992             : }
     993             : 
     994             : /*
     995             :  * folio_lock_or_retry - Lock the folio, unless this would block and the
     996             :  * caller indicated that it can handle a retry.
     997             :  *
     998             :  * Return value and mmap_lock implications depend on flags; see
     999             :  * __folio_lock_or_retry().
    1000             :  */
    1001           0 : static inline bool folio_lock_or_retry(struct folio *folio,
    1002             :                 struct mm_struct *mm, unsigned int flags)
    1003             : {
    1004             :         might_sleep();
    1005           0 :         return folio_trylock(folio) || __folio_lock_or_retry(folio, mm, flags);
    1006             : }
    1007             : 
    1008             : /*
    1009             :  * This is exported only for folio_wait_locked/folio_wait_writeback, etc.,
    1010             :  * and should not be used directly.
    1011             :  */
    1012             : void folio_wait_bit(struct folio *folio, int bit_nr);
    1013             : int folio_wait_bit_killable(struct folio *folio, int bit_nr);
    1014             : 
    1015             : /* 
    1016             :  * Wait for a folio to be unlocked.
    1017             :  *
    1018             :  * This must be called with the caller "holding" the folio,
    1019             :  * ie with increased folio reference count so that the folio won't
    1020             :  * go away during the wait.
    1021             :  */
    1022             : static inline void folio_wait_locked(struct folio *folio)
    1023             : {
    1024           0 :         if (folio_test_locked(folio))
    1025             :                 folio_wait_bit(folio, PG_locked);
    1026             : }
    1027             : 
    1028           0 : static inline int folio_wait_locked_killable(struct folio *folio)
    1029             : {
    1030           0 :         if (!folio_test_locked(folio))
    1031             :                 return 0;
    1032           0 :         return folio_wait_bit_killable(folio, PG_locked);
    1033             : }
    1034             : 
    1035             : static inline void wait_on_page_locked(struct page *page)
    1036             : {
    1037             :         folio_wait_locked(page_folio(page));
    1038             : }
    1039             : 
    1040             : static inline int wait_on_page_locked_killable(struct page *page)
    1041             : {
    1042             :         return folio_wait_locked_killable(page_folio(page));
    1043             : }
    1044             : 
    1045             : void wait_on_page_writeback(struct page *page);
    1046             : void folio_wait_writeback(struct folio *folio);
    1047             : int folio_wait_writeback_killable(struct folio *folio);
    1048             : void end_page_writeback(struct page *page);
    1049             : void folio_end_writeback(struct folio *folio);
    1050             : void wait_for_stable_page(struct page *page);
    1051             : void folio_wait_stable(struct folio *folio);
    1052             : void __folio_mark_dirty(struct folio *folio, struct address_space *, int warn);
    1053             : static inline void __set_page_dirty(struct page *page,
    1054             :                 struct address_space *mapping, int warn)
    1055             : {
    1056             :         __folio_mark_dirty(page_folio(page), mapping, warn);
    1057             : }
    1058             : void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb);
    1059             : void __folio_cancel_dirty(struct folio *folio);
    1060             : static inline void folio_cancel_dirty(struct folio *folio)
    1061             : {
    1062             :         /* Avoid atomic ops, locking, etc. when not actually needed. */
    1063           0 :         if (folio_test_dirty(folio))
    1064           0 :                 __folio_cancel_dirty(folio);
    1065             : }
    1066             : bool folio_clear_dirty_for_io(struct folio *folio);
    1067             : bool clear_page_dirty_for_io(struct page *page);
    1068             : void folio_invalidate(struct folio *folio, size_t offset, size_t length);
    1069             : int __must_check folio_write_one(struct folio *folio);
    1070             : static inline int __must_check write_one_page(struct page *page)
    1071             : {
    1072             :         return folio_write_one(page_folio(page));
    1073             : }
    1074             : 
    1075             : int __set_page_dirty_nobuffers(struct page *page);
    1076             : bool noop_dirty_folio(struct address_space *mapping, struct folio *folio);
    1077             : 
    1078             : #ifdef CONFIG_MIGRATION
    1079             : int filemap_migrate_folio(struct address_space *mapping, struct folio *dst,
    1080             :                 struct folio *src, enum migrate_mode mode);
    1081             : #else
    1082             : #define filemap_migrate_folio NULL
    1083             : #endif
    1084             : void page_endio(struct page *page, bool is_write, int err);
    1085             : 
    1086             : void folio_end_private_2(struct folio *folio);
    1087             : void folio_wait_private_2(struct folio *folio);
    1088             : int folio_wait_private_2_killable(struct folio *folio);
    1089             : 
    1090             : /*
    1091             :  * Add an arbitrary waiter to a page's wait queue
    1092             :  */
    1093             : void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter);
    1094             : 
    1095             : /*
    1096             :  * Fault in userspace address range.
    1097             :  */
    1098             : size_t fault_in_writeable(char __user *uaddr, size_t size);
    1099             : size_t fault_in_subpage_writeable(char __user *uaddr, size_t size);
    1100             : size_t fault_in_safe_writeable(const char __user *uaddr, size_t size);
    1101             : size_t fault_in_readable(const char __user *uaddr, size_t size);
    1102             : 
    1103             : int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
    1104             :                 pgoff_t index, gfp_t gfp);
    1105             : int filemap_add_folio(struct address_space *mapping, struct folio *folio,
    1106             :                 pgoff_t index, gfp_t gfp);
    1107             : void filemap_remove_folio(struct folio *folio);
    1108             : void __filemap_remove_folio(struct folio *folio, void *shadow);
    1109             : void replace_page_cache_folio(struct folio *old, struct folio *new);
    1110             : void delete_from_page_cache_batch(struct address_space *mapping,
    1111             :                                   struct folio_batch *fbatch);
    1112             : bool filemap_release_folio(struct folio *folio, gfp_t gfp);
    1113             : loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end,
    1114             :                 int whence);
    1115             : 
    1116             : /* Must be non-static for BPF error injection */
    1117             : int __filemap_add_folio(struct address_space *mapping, struct folio *folio,
    1118             :                 pgoff_t index, gfp_t gfp, void **shadowp);
    1119             : 
    1120             : bool filemap_range_has_writeback(struct address_space *mapping,
    1121             :                                  loff_t start_byte, loff_t end_byte);
    1122             : 
    1123             : /**
    1124             :  * filemap_range_needs_writeback - check if range potentially needs writeback
    1125             :  * @mapping:           address space within which to check
    1126             :  * @start_byte:        offset in bytes where the range starts
    1127             :  * @end_byte:          offset in bytes where the range ends (inclusive)
    1128             :  *
    1129             :  * Find at least one page in the range supplied, usually used to check if
    1130             :  * direct writing in this range will trigger a writeback. Used by O_DIRECT
    1131             :  * read/write with IOCB_NOWAIT, to see if the caller needs to do
    1132             :  * filemap_write_and_wait_range() before proceeding.
    1133             :  *
    1134             :  * Return: %true if the caller should do filemap_write_and_wait_range() before
    1135             :  * doing O_DIRECT to a page in this range, %false otherwise.
    1136             :  */
    1137           0 : static inline bool filemap_range_needs_writeback(struct address_space *mapping,
    1138             :                                                  loff_t start_byte,
    1139             :                                                  loff_t end_byte)
    1140             : {
    1141           0 :         if (!mapping->nrpages)
    1142             :                 return false;
    1143           0 :         if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
    1144           0 :             !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
    1145             :                 return false;
    1146           0 :         return filemap_range_has_writeback(mapping, start_byte, end_byte);
    1147             : }
    1148             : 
    1149             : /**
    1150             :  * struct readahead_control - Describes a readahead request.
    1151             :  *
    1152             :  * A readahead request is for consecutive pages.  Filesystems which
    1153             :  * implement the ->readahead method should call readahead_page() or
    1154             :  * readahead_page_batch() in a loop and attempt to start I/O against
    1155             :  * each page in the request.
    1156             :  *
    1157             :  * Most of the fields in this struct are private and should be accessed
    1158             :  * by the functions below.
    1159             :  *
    1160             :  * @file: The file, used primarily by network filesystems for authentication.
    1161             :  *        May be NULL if invoked internally by the filesystem.
    1162             :  * @mapping: Readahead this filesystem object.
    1163             :  * @ra: File readahead state.  May be NULL.
    1164             :  */
    1165             : struct readahead_control {
    1166             :         struct file *file;
    1167             :         struct address_space *mapping;
    1168             :         struct file_ra_state *ra;
    1169             : /* private: use the readahead_* accessors instead */
    1170             :         pgoff_t _index;
    1171             :         unsigned int _nr_pages;
    1172             :         unsigned int _batch_count;
    1173             :         bool _workingset;
    1174             :         unsigned long _pflags;
    1175             : };
    1176             : 
    1177             : #define DEFINE_READAHEAD(ractl, f, r, m, i)                             \
    1178             :         struct readahead_control ractl = {                              \
    1179             :                 .file = f,                                              \
    1180             :                 .mapping = m,                                           \
    1181             :                 .ra = r,                                                \
    1182             :                 ._index = i,                                            \
    1183             :         }
    1184             : 
    1185             : #define VM_READAHEAD_PAGES      (SZ_128K / PAGE_SIZE)
    1186             : 
    1187             : void page_cache_ra_unbounded(struct readahead_control *,
    1188             :                 unsigned long nr_to_read, unsigned long lookahead_count);
    1189             : void page_cache_sync_ra(struct readahead_control *, unsigned long req_count);
    1190             : void page_cache_async_ra(struct readahead_control *, struct folio *,
    1191             :                 unsigned long req_count);
    1192             : void readahead_expand(struct readahead_control *ractl,
    1193             :                       loff_t new_start, size_t new_len);
    1194             : 
    1195             : /**
    1196             :  * page_cache_sync_readahead - generic file readahead
    1197             :  * @mapping: address_space which holds the pagecache and I/O vectors
    1198             :  * @ra: file_ra_state which holds the readahead state
    1199             :  * @file: Used by the filesystem for authentication.
    1200             :  * @index: Index of first page to be read.
    1201             :  * @req_count: Total number of pages being read by the caller.
    1202             :  *
    1203             :  * page_cache_sync_readahead() should be called when a cache miss happened:
    1204             :  * it will submit the read.  The readahead logic may decide to piggyback more
    1205             :  * pages onto the read request if access patterns suggest it will improve
    1206             :  * performance.
    1207             :  */
    1208             : static inline
    1209             : void page_cache_sync_readahead(struct address_space *mapping,
    1210             :                 struct file_ra_state *ra, struct file *file, pgoff_t index,
    1211             :                 unsigned long req_count)
    1212             : {
    1213           0 :         DEFINE_READAHEAD(ractl, file, ra, mapping, index);
    1214           0 :         page_cache_sync_ra(&ractl, req_count);
    1215             : }
    1216             : 
    1217             : /**
    1218             :  * page_cache_async_readahead - file readahead for marked pages
    1219             :  * @mapping: address_space which holds the pagecache and I/O vectors
    1220             :  * @ra: file_ra_state which holds the readahead state
    1221             :  * @file: Used by the filesystem for authentication.
    1222             :  * @folio: The folio at @index which triggered the readahead call.
    1223             :  * @index: Index of first page to be read.
    1224             :  * @req_count: Total number of pages being read by the caller.
    1225             :  *
    1226             :  * page_cache_async_readahead() should be called when a page is used which
    1227             :  * is marked as PageReadahead; this is a marker to suggest that the application
    1228             :  * has used up enough of the readahead window that we should start pulling in
    1229             :  * more pages.
    1230             :  */
    1231             : static inline
    1232             : void page_cache_async_readahead(struct address_space *mapping,
    1233             :                 struct file_ra_state *ra, struct file *file,
    1234             :                 struct folio *folio, pgoff_t index, unsigned long req_count)
    1235             : {
    1236             :         DEFINE_READAHEAD(ractl, file, ra, mapping, index);
    1237             :         page_cache_async_ra(&ractl, folio, req_count);
    1238             : }
    1239             : 
    1240           0 : static inline struct folio *__readahead_folio(struct readahead_control *ractl)
    1241             : {
    1242             :         struct folio *folio;
    1243             : 
    1244           0 :         BUG_ON(ractl->_batch_count > ractl->_nr_pages);
    1245           0 :         ractl->_nr_pages -= ractl->_batch_count;
    1246           0 :         ractl->_index += ractl->_batch_count;
    1247             : 
    1248           0 :         if (!ractl->_nr_pages) {
    1249           0 :                 ractl->_batch_count = 0;
    1250           0 :                 return NULL;
    1251             :         }
    1252             : 
    1253           0 :         folio = xa_load(&ractl->mapping->i_pages, ractl->_index);
    1254             :         VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
    1255           0 :         ractl->_batch_count = folio_nr_pages(folio);
    1256             : 
    1257           0 :         return folio;
    1258             : }
    1259             : 
    1260             : /**
    1261             :  * readahead_page - Get the next page to read.
    1262             :  * @ractl: The current readahead request.
    1263             :  *
    1264             :  * Context: The page is locked and has an elevated refcount.  The caller
    1265             :  * should decreases the refcount once the page has been submitted for I/O
    1266             :  * and unlock the page once all I/O to that page has completed.
    1267             :  * Return: A pointer to the next page, or %NULL if we are done.
    1268             :  */
    1269             : static inline struct page *readahead_page(struct readahead_control *ractl)
    1270             : {
    1271             :         struct folio *folio = __readahead_folio(ractl);
    1272             : 
    1273             :         return &folio->page;
    1274             : }
    1275             : 
    1276             : /**
    1277             :  * readahead_folio - Get the next folio to read.
    1278             :  * @ractl: The current readahead request.
    1279             :  *
    1280             :  * Context: The folio is locked.  The caller should unlock the folio once
    1281             :  * all I/O to that folio has completed.
    1282             :  * Return: A pointer to the next folio, or %NULL if we are done.
    1283             :  */
    1284           0 : static inline struct folio *readahead_folio(struct readahead_control *ractl)
    1285             : {
    1286           0 :         struct folio *folio = __readahead_folio(ractl);
    1287             : 
    1288           0 :         if (folio)
    1289             :                 folio_put(folio);
    1290           0 :         return folio;
    1291             : }
    1292             : 
    1293             : static inline unsigned int __readahead_batch(struct readahead_control *rac,
    1294             :                 struct page **array, unsigned int array_sz)
    1295             : {
    1296             :         unsigned int i = 0;
    1297             :         XA_STATE(xas, &rac->mapping->i_pages, 0);
    1298             :         struct page *page;
    1299             : 
    1300             :         BUG_ON(rac->_batch_count > rac->_nr_pages);
    1301             :         rac->_nr_pages -= rac->_batch_count;
    1302             :         rac->_index += rac->_batch_count;
    1303             :         rac->_batch_count = 0;
    1304             : 
    1305             :         xas_set(&xas, rac->_index);
    1306             :         rcu_read_lock();
    1307             :         xas_for_each(&xas, page, rac->_index + rac->_nr_pages - 1) {
    1308             :                 if (xas_retry(&xas, page))
    1309             :                         continue;
    1310             :                 VM_BUG_ON_PAGE(!PageLocked(page), page);
    1311             :                 VM_BUG_ON_PAGE(PageTail(page), page);
    1312             :                 array[i++] = page;
    1313             :                 rac->_batch_count += thp_nr_pages(page);
    1314             :                 if (i == array_sz)
    1315             :                         break;
    1316             :         }
    1317             :         rcu_read_unlock();
    1318             : 
    1319             :         return i;
    1320             : }
    1321             : 
    1322             : /**
    1323             :  * readahead_page_batch - Get a batch of pages to read.
    1324             :  * @rac: The current readahead request.
    1325             :  * @array: An array of pointers to struct page.
    1326             :  *
    1327             :  * Context: The pages are locked and have an elevated refcount.  The caller
    1328             :  * should decreases the refcount once the page has been submitted for I/O
    1329             :  * and unlock the page once all I/O to that page has completed.
    1330             :  * Return: The number of pages placed in the array.  0 indicates the request
    1331             :  * is complete.
    1332             :  */
    1333             : #define readahead_page_batch(rac, array)                                \
    1334             :         __readahead_batch(rac, array, ARRAY_SIZE(array))
    1335             : 
    1336             : /**
    1337             :  * readahead_pos - The byte offset into the file of this readahead request.
    1338             :  * @rac: The readahead request.
    1339             :  */
    1340             : static inline loff_t readahead_pos(struct readahead_control *rac)
    1341             : {
    1342           0 :         return (loff_t)rac->_index * PAGE_SIZE;
    1343             : }
    1344             : 
    1345             : /**
    1346             :  * readahead_length - The number of bytes in this readahead request.
    1347             :  * @rac: The readahead request.
    1348             :  */
    1349             : static inline size_t readahead_length(struct readahead_control *rac)
    1350             : {
    1351             :         return rac->_nr_pages * PAGE_SIZE;
    1352             : }
    1353             : 
    1354             : /**
    1355             :  * readahead_index - The index of the first page in this readahead request.
    1356             :  * @rac: The readahead request.
    1357             :  */
    1358             : static inline pgoff_t readahead_index(struct readahead_control *rac)
    1359             : {
    1360             :         return rac->_index;
    1361             : }
    1362             : 
    1363             : /**
    1364             :  * readahead_count - The number of pages in this readahead request.
    1365             :  * @rac: The readahead request.
    1366             :  */
    1367             : static inline unsigned int readahead_count(struct readahead_control *rac)
    1368             : {
    1369             :         return rac->_nr_pages;
    1370             : }
    1371             : 
    1372             : /**
    1373             :  * readahead_batch_length - The number of bytes in the current batch.
    1374             :  * @rac: The readahead request.
    1375             :  */
    1376             : static inline size_t readahead_batch_length(struct readahead_control *rac)
    1377             : {
    1378             :         return rac->_batch_count * PAGE_SIZE;
    1379             : }
    1380             : 
    1381             : static inline unsigned long dir_pages(struct inode *inode)
    1382             : {
    1383             :         return (unsigned long)(inode->i_size + PAGE_SIZE - 1) >>
    1384             :                                PAGE_SHIFT;
    1385             : }
    1386             : 
    1387             : /**
    1388             :  * folio_mkwrite_check_truncate - check if folio was truncated
    1389             :  * @folio: the folio to check
    1390             :  * @inode: the inode to check the folio against
    1391             :  *
    1392             :  * Return: the number of bytes in the folio up to EOF,
    1393             :  * or -EFAULT if the folio was truncated.
    1394             :  */
    1395             : static inline ssize_t folio_mkwrite_check_truncate(struct folio *folio,
    1396             :                                               struct inode *inode)
    1397             : {
    1398             :         loff_t size = i_size_read(inode);
    1399             :         pgoff_t index = size >> PAGE_SHIFT;
    1400             :         size_t offset = offset_in_folio(folio, size);
    1401             : 
    1402             :         if (!folio->mapping)
    1403             :                 return -EFAULT;
    1404             : 
    1405             :         /* folio is wholly inside EOF */
    1406             :         if (folio_next_index(folio) - 1 < index)
    1407             :                 return folio_size(folio);
    1408             :         /* folio is wholly past EOF */
    1409             :         if (folio->index > index || !offset)
    1410             :                 return -EFAULT;
    1411             :         /* folio is partially inside EOF */
    1412             :         return offset;
    1413             : }
    1414             : 
    1415             : /**
    1416             :  * page_mkwrite_check_truncate - check if page was truncated
    1417             :  * @page: the page to check
    1418             :  * @inode: the inode to check the page against
    1419             :  *
    1420             :  * Returns the number of bytes in the page up to EOF,
    1421             :  * or -EFAULT if the page was truncated.
    1422             :  */
    1423             : static inline int page_mkwrite_check_truncate(struct page *page,
    1424             :                                               struct inode *inode)
    1425             : {
    1426             :         loff_t size = i_size_read(inode);
    1427             :         pgoff_t index = size >> PAGE_SHIFT;
    1428             :         int offset = offset_in_page(size);
    1429             : 
    1430             :         if (page->mapping != inode->i_mapping)
    1431             :                 return -EFAULT;
    1432             : 
    1433             :         /* page is wholly inside EOF */
    1434             :         if (page->index < index)
    1435             :                 return PAGE_SIZE;
    1436             :         /* page is wholly past EOF */
    1437             :         if (page->index > index || !offset)
    1438             :                 return -EFAULT;
    1439             :         /* page is partially inside EOF */
    1440             :         return offset;
    1441             : }
    1442             : 
    1443             : /**
    1444             :  * i_blocks_per_folio - How many blocks fit in this folio.
    1445             :  * @inode: The inode which contains the blocks.
    1446             :  * @folio: The folio.
    1447             :  *
    1448             :  * If the block size is larger than the size of this folio, return zero.
    1449             :  *
    1450             :  * Context: The caller should hold a refcount on the folio to prevent it
    1451             :  * from being split.
    1452             :  * Return: The number of filesystem blocks covered by this folio.
    1453             :  */
    1454             : static inline
    1455             : unsigned int i_blocks_per_folio(struct inode *inode, struct folio *folio)
    1456             : {
    1457             :         return folio_size(folio) >> inode->i_blkbits;
    1458             : }
    1459             : 
    1460             : static inline
    1461             : unsigned int i_blocks_per_page(struct inode *inode, struct page *page)
    1462             : {
    1463             :         return i_blocks_per_folio(inode, page_folio(page));
    1464             : }
    1465             : #endif /* _LINUX_PAGEMAP_H */

Generated by: LCOV version 1.14