Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Implement the default iomap interfaces
       4             :  *
       5             :  * (C) Copyright 2004 Linus Torvalds
       6             :  */
       7             : #include <linux/pci.h>
       8             : #include <linux/io.h>
       9             : #include <linux/kmsan-checks.h>
      10             : 
      11             : #include <linux/export.h>
      12             : 
      13             : /*
      14             :  * Read/write from/to an (offsettable) iomem cookie. It might be a PIO
      15             :  * access or a MMIO access, these functions don't care. The info is
      16             :  * encoded in the hardware mapping set up by the mapping functions
      17             :  * (or the cookie itself, depending on implementation and hw).
      18             :  *
      19             :  * The generic routines don't assume any hardware mappings, and just
      20             :  * encode the PIO/MMIO as part of the cookie. They coldly assume that
      21             :  * the MMIO IO mappings are not in the low address range.
      22             :  *
      23             :  * Architectures for which this is not true can't use this generic
      24             :  * implementation and should do their own copy.
      25             :  */
      26             : 
      27             : #ifndef HAVE_ARCH_PIO_SIZE
      28             : /*
      29             :  * We encode the physical PIO addresses (0-0xffff) into the
      30             :  * pointer by offsetting them with a constant (0x10000) and
      31             :  * assuming that all the low addresses are always PIO. That means
      32             :  * we can do some sanity checks on the low bits, and don't
      33             :  * need to just take things for granted.
      34             :  */
      35             : #define PIO_OFFSET      0x10000UL
      36             : #define PIO_MASK        0x0ffffUL
      37             : #define PIO_RESERVED    0x40000UL
      38             : #endif
      39             : 
      40           0 : static void bad_io_access(unsigned long port, const char *access)
      41             : {
      42             :         static int count = 10;
      43           0 :         if (count) {
      44           0 :                 count--;
      45           0 :                 WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n", port, access);
      46             :         }
      47           0 : }
      48             : 
      49             : /*
      50             :  * Ugly macros are a way of life.
      51             :  */
      52             : #define IO_COND(addr, is_pio, is_mmio) do {                     \
      53             :         unsigned long port = (unsigned long __force)addr;       \
      54             :         if (port >= PIO_RESERVED) {                          \
      55             :                 is_mmio;                                        \
      56             :         } else if (port > PIO_OFFSET) {                              \
      57             :                 port &= PIO_MASK;                           \
      58             :                 is_pio;                                         \
      59             :         } else                                                  \
      60             :                 bad_io_access(port, #is_pio );                  \
      61             : } while (0)
      62             : 
      63             : #ifndef pio_read16be
      64             : #define pio_read16be(port) swab16(inw(port))
      65             : #define pio_read32be(port) swab32(inl(port))
      66             : #endif
      67             : 
      68             : #ifndef mmio_read16be
      69             : #define mmio_read16be(addr) swab16(readw(addr))
      70             : #define mmio_read32be(addr) swab32(readl(addr))
      71             : #define mmio_read64be(addr) swab64(readq(addr))
      72             : #endif
      73             : 
      74             : /*
      75             :  * Here and below, we apply __no_kmsan_checks to functions reading data from
      76             :  * hardware, to ensure that KMSAN marks their return values as initialized.
      77             :  */
      78             : __no_kmsan_checks
      79           0 : unsigned int ioread8(const void __iomem *addr)
      80             : {
      81           0 :         IO_COND(addr, return inb(port), return readb(addr));
      82           0 :         return 0xff;
      83             : }
      84             : __no_kmsan_checks
      85           0 : unsigned int ioread16(const void __iomem *addr)
      86             : {
      87           0 :         IO_COND(addr, return inw(port), return readw(addr));
      88           0 :         return 0xffff;
      89             : }
      90             : __no_kmsan_checks
      91           0 : unsigned int ioread16be(const void __iomem *addr)
      92             : {
      93           0 :         IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr));
      94           0 :         return 0xffff;
      95             : }
      96             : __no_kmsan_checks
      97           0 : unsigned int ioread32(const void __iomem *addr)
      98             : {
      99           0 :         IO_COND(addr, return inl(port), return readl(addr));
     100           0 :         return 0xffffffff;
     101             : }
     102             : __no_kmsan_checks
     103           0 : unsigned int ioread32be(const void __iomem *addr)
     104             : {
     105           0 :         IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr));
     106           0 :         return 0xffffffff;
     107             : }
     108             : EXPORT_SYMBOL(ioread8);
     109             : EXPORT_SYMBOL(ioread16);
     110             : EXPORT_SYMBOL(ioread16be);
     111             : EXPORT_SYMBOL(ioread32);
     112             : EXPORT_SYMBOL(ioread32be);
     113             : 
     114             : #ifdef readq
     115           0 : static u64 pio_read64_lo_hi(unsigned long port)
     116             : {
     117             :         u64 lo, hi;
     118             : 
     119           0 :         lo = inl(port);
     120           0 :         hi = inl(port + sizeof(u32));
     121             : 
     122           0 :         return lo | (hi << 32);
     123             : }
     124             : 
     125           0 : static u64 pio_read64_hi_lo(unsigned long port)
     126             : {
     127             :         u64 lo, hi;
     128             : 
     129           0 :         hi = inl(port + sizeof(u32));
     130           0 :         lo = inl(port);
     131             : 
     132           0 :         return lo | (hi << 32);
     133             : }
     134             : 
     135           0 : static u64 pio_read64be_lo_hi(unsigned long port)
     136             : {
     137             :         u64 lo, hi;
     138             : 
     139           0 :         lo = pio_read32be(port + sizeof(u32));
     140           0 :         hi = pio_read32be(port);
     141             : 
     142           0 :         return lo | (hi << 32);
     143             : }
     144             : 
     145           0 : static u64 pio_read64be_hi_lo(unsigned long port)
     146             : {
     147             :         u64 lo, hi;
     148             : 
     149           0 :         hi = pio_read32be(port);
     150           0 :         lo = pio_read32be(port + sizeof(u32));
     151             : 
     152           0 :         return lo | (hi << 32);
     153             : }
     154             : 
     155             : __no_kmsan_checks
     156           0 : u64 ioread64_lo_hi(const void __iomem *addr)
     157             : {
     158           0 :         IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr));
     159           0 :         return 0xffffffffffffffffULL;
     160             : }
     161             : 
     162             : __no_kmsan_checks
     163           0 : u64 ioread64_hi_lo(const void __iomem *addr)
     164             : {
     165           0 :         IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr));
     166           0 :         return 0xffffffffffffffffULL;
     167             : }
     168             : 
     169             : __no_kmsan_checks
     170           0 : u64 ioread64be_lo_hi(const void __iomem *addr)
     171             : {
     172           0 :         IO_COND(addr, return pio_read64be_lo_hi(port),
     173             :                 return mmio_read64be(addr));
     174           0 :         return 0xffffffffffffffffULL;
     175             : }
     176             : 
     177             : __no_kmsan_checks
     178           0 : u64 ioread64be_hi_lo(const void __iomem *addr)
     179             : {
     180           0 :         IO_COND(addr, return pio_read64be_hi_lo(port),
     181             :                 return mmio_read64be(addr));
     182           0 :         return 0xffffffffffffffffULL;
     183             : }
     184             : 
     185             : EXPORT_SYMBOL(ioread64_lo_hi);
     186             : EXPORT_SYMBOL(ioread64_hi_lo);
     187             : EXPORT_SYMBOL(ioread64be_lo_hi);
     188             : EXPORT_SYMBOL(ioread64be_hi_lo);
     189             : 
     190             : #endif /* readq */
     191             : 
     192             : #ifndef pio_write16be
     193             : #define pio_write16be(val,port) outw(swab16(val),port)
     194             : #define pio_write32be(val,port) outl(swab32(val),port)
     195             : #endif
     196             : 
     197             : #ifndef mmio_write16be
     198             : #define mmio_write16be(val,port) writew(swab16(val),port)
     199             : #define mmio_write32be(val,port) writel(swab32(val),port)
     200             : #define mmio_write64be(val,port) writeq(swab64(val),port)
     201             : #endif
     202             : 
     203           0 : void iowrite8(u8 val, void __iomem *addr)
     204             : {
     205             :         /* Make sure uninitialized memory isn't copied to devices. */
     206           0 :         kmsan_check_memory(&val, sizeof(val));
     207           0 :         IO_COND(addr, outb(val,port), writeb(val, addr));
     208           0 : }
     209           0 : void iowrite16(u16 val, void __iomem *addr)
     210             : {
     211             :         /* Make sure uninitialized memory isn't copied to devices. */
     212           0 :         kmsan_check_memory(&val, sizeof(val));
     213           0 :         IO_COND(addr, outw(val,port), writew(val, addr));
     214           0 : }
     215           0 : void iowrite16be(u16 val, void __iomem *addr)
     216             : {
     217             :         /* Make sure uninitialized memory isn't copied to devices. */
     218           0 :         kmsan_check_memory(&val, sizeof(val));
     219           0 :         IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr));
     220           0 : }
     221           0 : void iowrite32(u32 val, void __iomem *addr)
     222             : {
     223             :         /* Make sure uninitialized memory isn't copied to devices. */
     224           0 :         kmsan_check_memory(&val, sizeof(val));
     225           0 :         IO_COND(addr, outl(val,port), writel(val, addr));
     226           0 : }
     227           0 : void iowrite32be(u32 val, void __iomem *addr)
     228             : {
     229             :         /* Make sure uninitialized memory isn't copied to devices. */
     230           0 :         kmsan_check_memory(&val, sizeof(val));
     231           0 :         IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr));
     232           0 : }
     233             : EXPORT_SYMBOL(iowrite8);
     234             : EXPORT_SYMBOL(iowrite16);
     235             : EXPORT_SYMBOL(iowrite16be);
     236             : EXPORT_SYMBOL(iowrite32);
     237             : EXPORT_SYMBOL(iowrite32be);
     238             : 
     239             : #ifdef writeq
     240           0 : static void pio_write64_lo_hi(u64 val, unsigned long port)
     241             : {
     242           0 :         outl(val, port);
     243           0 :         outl(val >> 32, port + sizeof(u32));
     244           0 : }
     245             : 
     246           0 : static void pio_write64_hi_lo(u64 val, unsigned long port)
     247             : {
     248           0 :         outl(val >> 32, port + sizeof(u32));
     249           0 :         outl(val, port);
     250           0 : }
     251             : 
     252           0 : static void pio_write64be_lo_hi(u64 val, unsigned long port)
     253             : {
     254           0 :         pio_write32be(val, port + sizeof(u32));
     255           0 :         pio_write32be(val >> 32, port);
     256           0 : }
     257             : 
     258           0 : static void pio_write64be_hi_lo(u64 val, unsigned long port)
     259             : {
     260           0 :         pio_write32be(val >> 32, port);
     261           0 :         pio_write32be(val, port + sizeof(u32));
     262           0 : }
     263             : 
     264           0 : void iowrite64_lo_hi(u64 val, void __iomem *addr)
     265             : {
     266             :         /* Make sure uninitialized memory isn't copied to devices. */
     267           0 :         kmsan_check_memory(&val, sizeof(val));
     268           0 :         IO_COND(addr, pio_write64_lo_hi(val, port),
     269             :                 writeq(val, addr));
     270           0 : }
     271             : 
     272           0 : void iowrite64_hi_lo(u64 val, void __iomem *addr)
     273             : {
     274             :         /* Make sure uninitialized memory isn't copied to devices. */
     275           0 :         kmsan_check_memory(&val, sizeof(val));
     276           0 :         IO_COND(addr, pio_write64_hi_lo(val, port),
     277             :                 writeq(val, addr));
     278           0 : }
     279             : 
     280           0 : void iowrite64be_lo_hi(u64 val, void __iomem *addr)
     281             : {
     282             :         /* Make sure uninitialized memory isn't copied to devices. */
     283           0 :         kmsan_check_memory(&val, sizeof(val));
     284           0 :         IO_COND(addr, pio_write64be_lo_hi(val, port),
     285             :                 mmio_write64be(val, addr));
     286           0 : }
     287             : 
     288           0 : void iowrite64be_hi_lo(u64 val, void __iomem *addr)
     289             : {
     290             :         /* Make sure uninitialized memory isn't copied to devices. */
     291           0 :         kmsan_check_memory(&val, sizeof(val));
     292           0 :         IO_COND(addr, pio_write64be_hi_lo(val, port),
     293             :                 mmio_write64be(val, addr));
     294           0 : }
     295             : 
     296             : EXPORT_SYMBOL(iowrite64_lo_hi);
     297             : EXPORT_SYMBOL(iowrite64_hi_lo);
     298             : EXPORT_SYMBOL(iowrite64be_lo_hi);
     299             : EXPORT_SYMBOL(iowrite64be_hi_lo);
     300             : 
     301             : #endif /* readq */
     302             : 
     303             : /*
     304             :  * These are the "repeat MMIO read/write" functions.
     305             :  * Note the "__raw" accesses, since we don't want to
     306             :  * convert to CPU byte order. We write in "IO byte
     307             :  * order" (we also don't have IO barriers).
     308             :  */
     309             : #ifndef mmio_insb
     310             : static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count)
     311             : {
     312           0 :         while (--count >= 0) {
     313           0 :                 u8 data = __raw_readb(addr);
     314           0 :                 *dst = data;
     315           0 :                 dst++;
     316             :         }
     317             : }
     318             : static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count)
     319             : {
     320           0 :         while (--count >= 0) {
     321           0 :                 u16 data = __raw_readw(addr);
     322           0 :                 *dst = data;
     323           0 :                 dst++;
     324             :         }
     325             : }
     326             : static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count)
     327             : {
     328           0 :         while (--count >= 0) {
     329           0 :                 u32 data = __raw_readl(addr);
     330           0 :                 *dst = data;
     331           0 :                 dst++;
     332             :         }
     333             : }
     334             : #endif
     335             : 
     336             : #ifndef mmio_outsb
     337             : static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count)
     338             : {
     339           0 :         while (--count >= 0) {
     340           0 :                 __raw_writeb(*src, addr);
     341           0 :                 src++;
     342             :         }
     343             : }
     344             : static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count)
     345             : {
     346           0 :         while (--count >= 0) {
     347           0 :                 __raw_writew(*src, addr);
     348           0 :                 src++;
     349             :         }
     350             : }
     351             : static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count)
     352             : {
     353           0 :         while (--count >= 0) {
     354           0 :                 __raw_writel(*src, addr);
     355           0 :                 src++;
     356             :         }
     357             : }
     358             : #endif
     359             : 
     360           0 : void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count)
     361             : {
     362           0 :         IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count));
     363             :         /* KMSAN must treat values read from devices as initialized. */
     364           0 :         kmsan_unpoison_memory(dst, count);
     365           0 : }
     366           0 : void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count)
     367             : {
     368           0 :         IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count));
     369             :         /* KMSAN must treat values read from devices as initialized. */
     370           0 :         kmsan_unpoison_memory(dst, count * 2);
     371           0 : }
     372           0 : void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count)
     373             : {
     374           0 :         IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count));
     375             :         /* KMSAN must treat values read from devices as initialized. */
     376           0 :         kmsan_unpoison_memory(dst, count * 4);
     377           0 : }
     378             : EXPORT_SYMBOL(ioread8_rep);
     379             : EXPORT_SYMBOL(ioread16_rep);
     380             : EXPORT_SYMBOL(ioread32_rep);
     381             : 
     382           0 : void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count)
     383             : {
     384             :         /* Make sure uninitialized memory isn't copied to devices. */
     385           0 :         kmsan_check_memory(src, count);
     386           0 :         IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count));
     387           0 : }
     388           0 : void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count)
     389             : {
     390             :         /* Make sure uninitialized memory isn't copied to devices. */
     391           0 :         kmsan_check_memory(src, count * 2);
     392           0 :         IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count));
     393           0 : }
     394           0 : void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count)
     395             : {
     396             :         /* Make sure uninitialized memory isn't copied to devices. */
     397           0 :         kmsan_check_memory(src, count * 4);
     398           0 :         IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count));
     399           0 : }
     400             : EXPORT_SYMBOL(iowrite8_rep);
     401             : EXPORT_SYMBOL(iowrite16_rep);
     402             : EXPORT_SYMBOL(iowrite32_rep);
     403             : 
     404             : #ifdef CONFIG_HAS_IOPORT_MAP
     405             : /* Create a virtual mapping cookie for an IO port range */
     406             : void __iomem *ioport_map(unsigned long port, unsigned int nr)
     407             : {
     408             :         if (port > PIO_MASK)
     409             :                 return NULL;
     410             :         return (void __iomem *) (unsigned long) (port + PIO_OFFSET);
     411             : }
     412             : 
     413             : void ioport_unmap(void __iomem *addr)
     414             : {
     415             :         /* Nothing to do */
     416             : }
     417             : EXPORT_SYMBOL(ioport_map);
     418             : EXPORT_SYMBOL(ioport_unmap);
     419             : #endif /* CONFIG_HAS_IOPORT_MAP */
     420             : 
     421             : #ifdef CONFIG_PCI
     422             : /* Hide the details if this is a MMIO or PIO address space and just do what
     423             :  * you expect in the correct way. */
     424           0 : void pci_iounmap(struct pci_dev *dev, void __iomem * addr)
     425             : {
     426           0 :         IO_COND(addr, /* nothing */, iounmap(addr));
     427           0 : }
     428             : EXPORT_SYMBOL(pci_iounmap);
     429             : #endif /* CONFIG_PCI */