Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : /*
3 : * This header is for implementations of dma_map_ops and related code.
4 : * It should not be included in drivers just using the DMA API.
5 : */
6 : #ifndef _LINUX_DMA_MAP_OPS_H
7 : #define _LINUX_DMA_MAP_OPS_H
8 :
9 : #include <linux/dma-mapping.h>
10 : #include <linux/pgtable.h>
11 :
12 : struct cma;
13 :
14 : /*
15 : * Values for struct dma_map_ops.flags:
16 : *
17 : * DMA_F_PCI_P2PDMA_SUPPORTED: Indicates the dma_map_ops implementation can
18 : * handle PCI P2PDMA pages in the map_sg/unmap_sg operation.
19 : */
20 : #define DMA_F_PCI_P2PDMA_SUPPORTED (1 << 0)
21 :
22 : struct dma_map_ops {
23 : unsigned int flags;
24 :
25 : void *(*alloc)(struct device *dev, size_t size,
26 : dma_addr_t *dma_handle, gfp_t gfp,
27 : unsigned long attrs);
28 : void (*free)(struct device *dev, size_t size, void *vaddr,
29 : dma_addr_t dma_handle, unsigned long attrs);
30 : struct page *(*alloc_pages)(struct device *dev, size_t size,
31 : dma_addr_t *dma_handle, enum dma_data_direction dir,
32 : gfp_t gfp);
33 : void (*free_pages)(struct device *dev, size_t size, struct page *vaddr,
34 : dma_addr_t dma_handle, enum dma_data_direction dir);
35 : struct sg_table *(*alloc_noncontiguous)(struct device *dev, size_t size,
36 : enum dma_data_direction dir, gfp_t gfp,
37 : unsigned long attrs);
38 : void (*free_noncontiguous)(struct device *dev, size_t size,
39 : struct sg_table *sgt, enum dma_data_direction dir);
40 : int (*mmap)(struct device *, struct vm_area_struct *,
41 : void *, dma_addr_t, size_t, unsigned long attrs);
42 :
43 : int (*get_sgtable)(struct device *dev, struct sg_table *sgt,
44 : void *cpu_addr, dma_addr_t dma_addr, size_t size,
45 : unsigned long attrs);
46 :
47 : dma_addr_t (*map_page)(struct device *dev, struct page *page,
48 : unsigned long offset, size_t size,
49 : enum dma_data_direction dir, unsigned long attrs);
50 : void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
51 : size_t size, enum dma_data_direction dir,
52 : unsigned long attrs);
53 : /*
54 : * map_sg should return a negative error code on error. See
55 : * dma_map_sgtable() for a list of appropriate error codes
56 : * and their meanings.
57 : */
58 : int (*map_sg)(struct device *dev, struct scatterlist *sg, int nents,
59 : enum dma_data_direction dir, unsigned long attrs);
60 : void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents,
61 : enum dma_data_direction dir, unsigned long attrs);
62 : dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
63 : size_t size, enum dma_data_direction dir,
64 : unsigned long attrs);
65 : void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
66 : size_t size, enum dma_data_direction dir,
67 : unsigned long attrs);
68 : void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle,
69 : size_t size, enum dma_data_direction dir);
70 : void (*sync_single_for_device)(struct device *dev,
71 : dma_addr_t dma_handle, size_t size,
72 : enum dma_data_direction dir);
73 : void (*sync_sg_for_cpu)(struct device *dev, struct scatterlist *sg,
74 : int nents, enum dma_data_direction dir);
75 : void (*sync_sg_for_device)(struct device *dev, struct scatterlist *sg,
76 : int nents, enum dma_data_direction dir);
77 : void (*cache_sync)(struct device *dev, void *vaddr, size_t size,
78 : enum dma_data_direction direction);
79 : int (*dma_supported)(struct device *dev, u64 mask);
80 : u64 (*get_required_mask)(struct device *dev);
81 : size_t (*max_mapping_size)(struct device *dev);
82 : size_t (*opt_mapping_size)(void);
83 : unsigned long (*get_merge_boundary)(struct device *dev);
84 : };
85 :
86 : #ifdef CONFIG_DMA_OPS
87 : #include <asm/dma-mapping.h>
88 :
89 : static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
90 : {
91 : if (dev->dma_ops)
92 : return dev->dma_ops;
93 : return get_arch_dma_ops();
94 : }
95 :
96 : static inline void set_dma_ops(struct device *dev,
97 : const struct dma_map_ops *dma_ops)
98 : {
99 : dev->dma_ops = dma_ops;
100 : }
101 : #else /* CONFIG_DMA_OPS */
102 : static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
103 : {
104 : return NULL;
105 : }
106 : static inline void set_dma_ops(struct device *dev,
107 : const struct dma_map_ops *dma_ops)
108 : {
109 : }
110 : #endif /* CONFIG_DMA_OPS */
111 :
112 : #ifdef CONFIG_DMA_CMA
113 : extern struct cma *dma_contiguous_default_area;
114 :
115 : static inline struct cma *dev_get_cma_area(struct device *dev)
116 : {
117 : if (dev && dev->cma_area)
118 : return dev->cma_area;
119 : return dma_contiguous_default_area;
120 : }
121 :
122 : void dma_contiguous_reserve(phys_addr_t addr_limit);
123 : int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base,
124 : phys_addr_t limit, struct cma **res_cma, bool fixed);
125 :
126 : struct page *dma_alloc_from_contiguous(struct device *dev, size_t count,
127 : unsigned int order, bool no_warn);
128 : bool dma_release_from_contiguous(struct device *dev, struct page *pages,
129 : int count);
130 : struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp);
131 : void dma_free_contiguous(struct device *dev, struct page *page, size_t size);
132 :
133 : void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size);
134 : #else /* CONFIG_DMA_CMA */
135 : static inline struct cma *dev_get_cma_area(struct device *dev)
136 : {
137 : return NULL;
138 : }
139 : static inline void dma_contiguous_reserve(phys_addr_t limit)
140 : {
141 : }
142 : static inline int dma_contiguous_reserve_area(phys_addr_t size,
143 : phys_addr_t base, phys_addr_t limit, struct cma **res_cma,
144 : bool fixed)
145 : {
146 : return -ENOSYS;
147 : }
148 : static inline struct page *dma_alloc_from_contiguous(struct device *dev,
149 : size_t count, unsigned int order, bool no_warn)
150 : {
151 : return NULL;
152 : }
153 : static inline bool dma_release_from_contiguous(struct device *dev,
154 : struct page *pages, int count)
155 : {
156 : return false;
157 : }
158 : /* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */
159 : static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size,
160 : gfp_t gfp)
161 : {
162 : return NULL;
163 : }
164 0 : static inline void dma_free_contiguous(struct device *dev, struct page *page,
165 : size_t size)
166 : {
167 0 : __free_pages(page, get_order(size));
168 0 : }
169 : #endif /* CONFIG_DMA_CMA*/
170 :
171 : #ifdef CONFIG_DMA_PERNUMA_CMA
172 : void dma_pernuma_cma_reserve(void);
173 : #else
174 : static inline void dma_pernuma_cma_reserve(void) { }
175 : #endif /* CONFIG_DMA_PERNUMA_CMA */
176 :
177 : #ifdef CONFIG_DMA_DECLARE_COHERENT
178 : int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
179 : dma_addr_t device_addr, size_t size);
180 : void dma_release_coherent_memory(struct device *dev);
181 : int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
182 : dma_addr_t *dma_handle, void **ret);
183 : int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
184 : int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
185 : void *cpu_addr, size_t size, int *ret);
186 : #else
187 : static inline int dma_declare_coherent_memory(struct device *dev,
188 : phys_addr_t phys_addr, dma_addr_t device_addr, size_t size)
189 : {
190 : return -ENOSYS;
191 : }
192 :
193 : #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
194 : #define dma_release_from_dev_coherent(dev, order, vaddr) (0)
195 : #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
196 : static inline void dma_release_coherent_memory(struct device *dev) { }
197 : #endif /* CONFIG_DMA_DECLARE_COHERENT */
198 :
199 : #ifdef CONFIG_DMA_GLOBAL_POOL
200 : void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
201 : dma_addr_t *dma_handle);
202 : int dma_release_from_global_coherent(int order, void *vaddr);
203 : int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
204 : size_t size, int *ret);
205 : int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
206 : #else
207 : static inline void *dma_alloc_from_global_coherent(struct device *dev,
208 : ssize_t size, dma_addr_t *dma_handle)
209 : {
210 : return NULL;
211 : }
212 : static inline int dma_release_from_global_coherent(int order, void *vaddr)
213 : {
214 : return 0;
215 : }
216 : static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
217 : void *cpu_addr, size_t size, int *ret)
218 : {
219 : return 0;
220 : }
221 : #endif /* CONFIG_DMA_GLOBAL_POOL */
222 :
223 : /*
224 : * This is the actual return value from the ->alloc_noncontiguous method.
225 : * The users of the DMA API should only care about the sg_table, but to make
226 : * the DMA-API internal vmaping and freeing easier we stash away the page
227 : * array as well (except for the fallback case). This can go away any time,
228 : * e.g. when a vmap-variant that takes a scatterlist comes along.
229 : */
230 : struct dma_sgt_handle {
231 : struct sg_table sgt;
232 : struct page **pages;
233 : };
234 : #define sgt_handle(sgt) \
235 : container_of((sgt), struct dma_sgt_handle, sgt)
236 :
237 : int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
238 : void *cpu_addr, dma_addr_t dma_addr, size_t size,
239 : unsigned long attrs);
240 : int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
241 : void *cpu_addr, dma_addr_t dma_addr, size_t size,
242 : unsigned long attrs);
243 : struct page *dma_common_alloc_pages(struct device *dev, size_t size,
244 : dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
245 : void dma_common_free_pages(struct device *dev, size_t size, struct page *vaddr,
246 : dma_addr_t dma_handle, enum dma_data_direction dir);
247 :
248 : struct page **dma_common_find_pages(void *cpu_addr);
249 : void *dma_common_contiguous_remap(struct page *page, size_t size, pgprot_t prot,
250 : const void *caller);
251 : void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
252 : const void *caller);
253 : void dma_common_free_remap(void *cpu_addr, size_t size);
254 :
255 : struct page *dma_alloc_from_pool(struct device *dev, size_t size,
256 : void **cpu_addr, gfp_t flags,
257 : bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
258 : bool dma_free_from_pool(struct device *dev, void *start, size_t size);
259 :
260 : int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
261 : dma_addr_t dma_start, u64 size);
262 :
263 : #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
264 : defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
265 : defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
266 : extern bool dma_default_coherent;
267 : static inline bool dev_is_dma_coherent(struct device *dev)
268 : {
269 : return dev->dma_coherent;
270 : }
271 : #else
272 : static inline bool dev_is_dma_coherent(struct device *dev)
273 : {
274 : return true;
275 : }
276 : #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
277 :
278 : void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
279 : gfp_t gfp, unsigned long attrs);
280 : void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
281 : dma_addr_t dma_addr, unsigned long attrs);
282 :
283 : #ifdef CONFIG_MMU
284 : /*
285 : * Page protection so that devices that can't snoop CPU caches can use the
286 : * memory coherently. We default to pgprot_noncached which is usually used
287 : * for ioremap as a safe bet, but architectures can override this with less
288 : * strict semantics if possible.
289 : */
290 : #ifndef pgprot_dmacoherent
291 : #define pgprot_dmacoherent(prot) pgprot_noncached(prot)
292 : #endif
293 :
294 : pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs);
295 : #else
296 : static inline pgprot_t dma_pgprot(struct device *dev, pgprot_t prot,
297 : unsigned long attrs)
298 : {
299 : return prot; /* no protection bits supported without page tables */
300 : }
301 : #endif /* CONFIG_MMU */
302 :
303 : #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE
304 : void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
305 : enum dma_data_direction dir);
306 : #else
307 : static inline void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
308 : enum dma_data_direction dir)
309 : {
310 : }
311 : #endif /* ARCH_HAS_SYNC_DMA_FOR_DEVICE */
312 :
313 : #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU
314 : void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
315 : enum dma_data_direction dir);
316 : #else
317 : static inline void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
318 : enum dma_data_direction dir)
319 : {
320 : }
321 : #endif /* ARCH_HAS_SYNC_DMA_FOR_CPU */
322 :
323 : #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
324 : void arch_sync_dma_for_cpu_all(void);
325 : #else
326 : static inline void arch_sync_dma_for_cpu_all(void)
327 : {
328 : }
329 : #endif /* CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL */
330 :
331 : #ifdef CONFIG_ARCH_HAS_DMA_PREP_COHERENT
332 : void arch_dma_prep_coherent(struct page *page, size_t size);
333 : #else
334 : static inline void arch_dma_prep_coherent(struct page *page, size_t size)
335 : {
336 : }
337 : #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
338 :
339 : #ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
340 : void arch_dma_mark_clean(phys_addr_t paddr, size_t size);
341 : #else
342 : static inline void arch_dma_mark_clean(phys_addr_t paddr, size_t size)
343 : {
344 : }
345 : #endif /* ARCH_HAS_DMA_MARK_CLEAN */
346 :
347 : void *arch_dma_set_uncached(void *addr, size_t size);
348 : void arch_dma_clear_uncached(void *addr, size_t size);
349 :
350 : #ifdef CONFIG_ARCH_HAS_DMA_MAP_DIRECT
351 : bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr);
352 : bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle);
353 : bool arch_dma_map_sg_direct(struct device *dev, struct scatterlist *sg,
354 : int nents);
355 : bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,
356 : int nents);
357 : #else
358 : #define arch_dma_map_page_direct(d, a) (false)
359 : #define arch_dma_unmap_page_direct(d, a) (false)
360 : #define arch_dma_map_sg_direct(d, s, n) (false)
361 : #define arch_dma_unmap_sg_direct(d, s, n) (false)
362 : #endif
363 :
364 : #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
365 : void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
366 : const struct iommu_ops *iommu, bool coherent);
367 : #else
368 : static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
369 : u64 size, const struct iommu_ops *iommu, bool coherent)
370 : {
371 : }
372 : #endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */
373 :
374 : #ifdef CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS
375 : void arch_teardown_dma_ops(struct device *dev);
376 : #else
377 : static inline void arch_teardown_dma_ops(struct device *dev)
378 : {
379 : }
380 : #endif /* CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS */
381 :
382 : #ifdef CONFIG_DMA_API_DEBUG
383 : void dma_debug_add_bus(struct bus_type *bus);
384 : void debug_dma_dump_mappings(struct device *dev);
385 : #else
386 : static inline void dma_debug_add_bus(struct bus_type *bus)
387 : {
388 : }
389 : static inline void debug_dma_dump_mappings(struct device *dev)
390 : {
391 : }
392 : #endif /* CONFIG_DMA_API_DEBUG */
393 :
394 : extern const struct dma_map_ops dma_dummy_ops;
395 :
396 : enum pci_p2pdma_map_type {
397 : /*
398 : * PCI_P2PDMA_MAP_UNKNOWN: Used internally for indicating the mapping
399 : * type hasn't been calculated yet. Functions that return this enum
400 : * never return this value.
401 : */
402 : PCI_P2PDMA_MAP_UNKNOWN = 0,
403 :
404 : /*
405 : * PCI_P2PDMA_MAP_NOT_SUPPORTED: Indicates the transaction will
406 : * traverse the host bridge and the host bridge is not in the
407 : * allowlist. DMA Mapping routines should return an error when
408 : * this is returned.
409 : */
410 : PCI_P2PDMA_MAP_NOT_SUPPORTED,
411 :
412 : /*
413 : * PCI_P2PDMA_BUS_ADDR: Indicates that two devices can talk to
414 : * each other directly through a PCI switch and the transaction will
415 : * not traverse the host bridge. Such a mapping should program
416 : * the DMA engine with PCI bus addresses.
417 : */
418 : PCI_P2PDMA_MAP_BUS_ADDR,
419 :
420 : /*
421 : * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: Indicates two devices can talk
422 : * to each other, but the transaction traverses a host bridge on the
423 : * allowlist. In this case, a normal mapping either with CPU physical
424 : * addresses (in the case of dma-direct) or IOVA addresses (in the
425 : * case of IOMMUs) should be used to program the DMA engine.
426 : */
427 : PCI_P2PDMA_MAP_THRU_HOST_BRIDGE,
428 : };
429 :
430 : struct pci_p2pdma_map_state {
431 : struct dev_pagemap *pgmap;
432 : int map;
433 : u64 bus_off;
434 : };
435 :
436 : #ifdef CONFIG_PCI_P2PDMA
437 : enum pci_p2pdma_map_type
438 : pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
439 : struct scatterlist *sg);
440 : #else /* CONFIG_PCI_P2PDMA */
441 : static inline enum pci_p2pdma_map_type
442 : pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
443 : struct scatterlist *sg)
444 : {
445 : return PCI_P2PDMA_MAP_NOT_SUPPORTED;
446 : }
447 : #endif /* CONFIG_PCI_P2PDMA */
448 :
449 : #endif /* _LINUX_DMA_MAP_OPS_H */
|
