From ea8c64ace86647260ec4255f483e5844d62af2df Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 10 Jan 2018 16:21:13 +0100 Subject: dma-mapping: move swiotlb arch helpers to a new header phys_to_dma, dma_to_phys and dma_capable are helpers published by architecture code for use of swiotlb and xen-swiotlb only. Drivers are not supposed to use these directly, but use the DMA API instead. Move these to a new asm/dma-direct.h helper, included by a linux/dma-direct.h wrapper that provides the default linear mapping unless the architecture wants to override it. In the MIPS case the existing dma-coherent.h is reused for now as untangling it will take a bit of work. Signed-off-by: Christoph Hellwig Acked-by: Robin Murphy --- lib/swiotlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/swiotlb.c') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index cea19aaf303c..6583f3512386 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -18,7 +18,7 @@ */ #include -#include +#include #include #include #include -- cgit v1.2.3-55-g7522 From d0bc0c2a31c95002d37c3cc511ffdcab851b3256 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 4 Jan 2018 14:24:19 +0100 Subject: swiotlb: suppress warning when __GFP_NOWARN is set TTM tries to allocate coherent memory in chunks of 2MB first to improve TLB efficiency and falls back to allocating 4K pages if that fails. Suppress the warning when the 2MB allocations fails since there is a valid fall back path. Signed-off-by: Christian König Reported-by: Mike Galbraith Acked-by: Konrad Rzeszutek Wilk Bug: https://bugs.freedesktop.org/show_bug.cgi?id=104082 CC: stable@vger.kernel.org Signed-off-by: Christoph Hellwig --- lib/swiotlb.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'lib/swiotlb.c') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 6583f3512386..125c1062119f 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -586,7 +586,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, not_found: spin_unlock_irqrestore(&io_tlb_lock, flags); - if (printk_ratelimit()) + if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size); return SWIOTLB_MAP_ERROR; found: @@ -713,6 +713,7 @@ void * swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags) { + bool warn = !(flags & __GFP_NOWARN); dma_addr_t dev_addr; void *ret; int order = get_order(size); @@ -738,8 +739,8 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, * GFP_DMA memory; fall back on map_single(), which * will grab memory from the lowest available address range. */ - phys_addr_t paddr = map_single(hwdev, 0, size, - DMA_FROM_DEVICE, 0); + phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE, + warn ? 0 : DMA_ATTR_NO_WARN); if (paddr == SWIOTLB_MAP_ERROR) goto err_warn; @@ -769,9 +770,11 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, return ret; err_warn: - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", - dev_name(hwdev), size); - dump_stack(); + if (warn && printk_ratelimit()) { + pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", + dev_name(hwdev), size); + dump_stack(); + } return NULL; } -- cgit v1.2.3-55-g7522 From 7f2c8bbd321f18e4ccfd262748bd58fb7d4bb1db Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 14:14:54 +0100 Subject: swiotlb: rename swiotlb_free to swiotlb_exit Signed-off-by: Christoph Hellwig Acked-by: Christian König Reviewed-by: Konrad Rzeszutek Wilk --- arch/powerpc/kernel/dma-swiotlb.c | 2 +- arch/x86/kernel/pci-swiotlb.c | 2 +- include/linux/swiotlb.h | 4 ++-- lib/swiotlb.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/swiotlb.c') diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c index 506ac4fafac5..88f3963ca30f 100644 --- a/arch/powerpc/kernel/dma-swiotlb.c +++ b/arch/powerpc/kernel/dma-swiotlb.c @@ -121,7 +121,7 @@ static int __init check_swiotlb_enabled(void) if (ppc_swiotlb_enable) swiotlb_print_info(); else - swiotlb_free(); + swiotlb_exit(); return 0; } diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index 0d77603c2f50..0ee0f8f34251 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -120,7 +120,7 @@ void __init pci_swiotlb_late_init(void) { /* An IOMMU turned us off. */ if (!swiotlb) - swiotlb_free(); + swiotlb_exit(); else { printk(KERN_INFO "PCI-DMA: " "Using software bounce buffering for IO (SWIOTLB)\n"); diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 24ed817082ee..606375e35d87 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -115,10 +115,10 @@ extern int swiotlb_dma_supported(struct device *hwdev, u64 mask); #ifdef CONFIG_SWIOTLB -extern void __init swiotlb_free(void); +extern void __init swiotlb_exit(void); unsigned int swiotlb_max_segment(void); #else -static inline void swiotlb_free(void) { } +static inline void swiotlb_exit(void) { } static inline unsigned int swiotlb_max_segment(void) { return 0; } #endif diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 125c1062119f..cf5311908fa9 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -417,7 +417,7 @@ cleanup2: return -ENOMEM; } -void __init swiotlb_free(void) +void __init swiotlb_exit(void) { if (!io_tlb_orig_addr) return; -- cgit v1.2.3-55-g7522 From 251533eb35b22f9e87a440b14c6f20e2626397b3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 16:44:16 +0100 Subject: swiotlb: add common swiotlb_map_ops Currently all architectures that want to use swiotlb have to implement their own dma_map_ops instances. Provide a generic one based on the x86 implementation which first calls into dma_direct to try a full blown direct mapping implementation (including e.g. CMA) before falling back allocating from the swiotlb buffer. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- include/linux/swiotlb.h | 8 ++++++++ lib/swiotlb.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) (limited to 'lib/swiotlb.c') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 606375e35d87..5b1f2a00491c 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -66,6 +66,12 @@ extern void swiotlb_tbl_sync_single(struct device *hwdev, enum dma_sync_target target); /* Accessory functions. */ + +void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle, + gfp_t flags, unsigned long attrs); +void swiotlb_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_addr, unsigned long attrs); + extern void *swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags); @@ -126,4 +132,6 @@ extern void swiotlb_print_info(void); extern int is_swiotlb_buffer(phys_addr_t paddr); extern void swiotlb_set_max_segment(unsigned int); +extern const struct dma_map_ops swiotlb_dma_ops; + #endif /* __LINUX_SWIOTLB_H */ diff --git a/lib/swiotlb.c b/lib/swiotlb.c index cf5311908fa9..0fae2f45c3c0 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -1087,3 +1087,46 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask) return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; } EXPORT_SYMBOL(swiotlb_dma_supported); + +#ifdef CONFIG_DMA_DIRECT_OPS +void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs) +{ + void *vaddr; + + /* + * Don't print a warning when the first allocation attempt fails. + * swiotlb_alloc_coherent() will print a warning when the DMA memory + * allocation ultimately failed. + */ + gfp |= __GFP_NOWARN; + + vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + if (!vaddr) + vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp); + return vaddr; +} + +void swiotlb_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_addr, unsigned long attrs) +{ + if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr))) + swiotlb_free_coherent(dev, size, vaddr, dma_addr); + else + dma_direct_free(dev, size, vaddr, dma_addr, attrs); +} + +const struct dma_map_ops swiotlb_dma_ops = { + .mapping_error = swiotlb_dma_mapping_error, + .alloc = swiotlb_alloc, + .free = swiotlb_free, + .sync_single_for_cpu = swiotlb_sync_single_for_cpu, + .sync_single_for_device = swiotlb_sync_single_for_device, + .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, + .sync_sg_for_device = swiotlb_sync_sg_for_device, + .map_sg = swiotlb_map_sg_attrs, + .unmap_sg = swiotlb_unmap_sg_attrs, + .map_page = swiotlb_map_page, + .unmap_page = swiotlb_unmap_page, +}; +#endif /* CONFIG_DMA_DIRECT_OPS */ -- cgit v1.2.3-55-g7522 From aaf796dc6e84e809e4c791b6517326b26312c972 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Dec 2017 15:07:34 +0100 Subject: swiotlb: wire up ->dma_supported in swiotlb_dma_ops To properly reject too small DMA masks based on the addressability of the bounce buffer. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/swiotlb.c') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 0fae2f45c3c0..539fd1099ba9 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -1128,5 +1128,6 @@ const struct dma_map_ops swiotlb_dma_ops = { .unmap_sg = swiotlb_unmap_sg_attrs, .map_page = swiotlb_map_page, .unmap_page = swiotlb_unmap_page, + .dma_supported = swiotlb_dma_supported, }; #endif /* CONFIG_DMA_DIRECT_OPS */ -- cgit v1.2.3-55-g7522 From a25381aa3ae60a2e028c95f1318649b13cbbce30 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 20:56:02 +0100 Subject: swiotlb: refactor coherent buffer freeing Factor out a new swiotlb_free_buffer helper that checks if an address is allocated from the swiotlb bounce buffer, and if yes frees it. This allows to simplify the swiotlb_free implemenation that uses dma_direct_free to free the non-bounce buffer allocations. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'lib/swiotlb.c') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 539fd1099ba9..1a147f1354a1 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -780,22 +780,31 @@ err_warn: } EXPORT_SYMBOL(swiotlb_alloc_coherent); +static bool swiotlb_free_buffer(struct device *dev, size_t size, + dma_addr_t dma_addr) +{ + phys_addr_t phys_addr = dma_to_phys(dev, dma_addr); + + WARN_ON_ONCE(irqs_disabled()); + + if (!is_swiotlb_buffer(phys_addr)) + return false; + + /* + * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single. + * DMA_ATTR_SKIP_CPU_SYNC is optional. + */ + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + return true; +} + void swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, dma_addr_t dev_addr) { - phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); - - WARN_ON(irqs_disabled()); - if (!is_swiotlb_buffer(paddr)) + if (!swiotlb_free_buffer(hwdev, size, dev_addr)) free_pages((unsigned long)vaddr, get_order(size)); - else - /* - * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single. - * DMA_ATTR_SKIP_CPU_SYNC is optional. - */ - swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE, - DMA_ATTR_SKIP_CPU_SYNC); } EXPORT_SYMBOL(swiotlb_free_coherent); @@ -1110,9 +1119,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, void swiotlb_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { - if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr))) - swiotlb_free_coherent(dev, size, vaddr, dma_addr); - else + if (!swiotlb_free_buffer(dev, size, dma_addr)) dma_direct_free(dev, size, vaddr, dma_addr, attrs); } -- cgit v1.2.3-55-g7522 From 0176adb004065d6815a8e67946752df4cd947c5b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 22:15:30 +0100 Subject: swiotlb: refactor coherent buffer allocation Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent memory from the swiotlb bounce buffer. This allows to simplify the swiotlb_alloc implemenation that uses dma_direct_alloc to try to allocate a reachable buffer first. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 122 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 65 insertions(+), 57 deletions(-) (limited to 'lib/swiotlb.c') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 1a147f1354a1..0039b7c5e690 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, } EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); -void * -swiotlb_alloc_coherent(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags) +static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, + size_t size) { - bool warn = !(flags & __GFP_NOWARN); - dma_addr_t dev_addr; - void *ret; - int order = get_order(size); - u64 dma_mask = DMA_BIT_MASK(32); + u64 mask = DMA_BIT_MASK(32); - if (hwdev && hwdev->coherent_dma_mask) - dma_mask = hwdev->coherent_dma_mask; + if (dev && dev->coherent_dma_mask) + mask = dev->coherent_dma_mask; + return addr + size - 1 <= mask; +} - ret = (void *)__get_free_pages(flags, order); - if (ret) { - dev_addr = swiotlb_virt_to_bus(hwdev, ret); - if (dev_addr + size - 1 > dma_mask) { - /* - * The allocated memory isn't reachable by the device. - */ - free_pages((unsigned long) ret, order); - ret = NULL; - } - } - if (!ret) { - /* - * We are either out of memory or the device can't DMA to - * GFP_DMA memory; fall back on map_single(), which - * will grab memory from the lowest available address range. - */ - phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE, - warn ? 0 : DMA_ATTR_NO_WARN); - if (paddr == SWIOTLB_MAP_ERROR) - goto err_warn; +static void * +swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, + unsigned long attrs) +{ + phys_addr_t phys_addr; - ret = phys_to_virt(paddr); - dev_addr = swiotlb_phys_to_dma(hwdev, paddr); + if (swiotlb_force == SWIOTLB_NO_FORCE) + goto out_warn; - /* Confirm address can be DMA'd by device */ - if (dev_addr + size - 1 > dma_mask) { - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", - (unsigned long long)dma_mask, - (unsigned long long)dev_addr); + phys_addr = swiotlb_tbl_map_single(dev, + swiotlb_phys_to_dma(dev, io_tlb_start), + 0, size, DMA_FROM_DEVICE, 0); + if (phys_addr == SWIOTLB_MAP_ERROR) + goto out_warn; - /* - * DMA_TO_DEVICE to avoid memcpy in unmap_single. - * The DMA_ATTR_SKIP_CPU_SYNC is optional. - */ - swiotlb_tbl_unmap_single(hwdev, paddr, - size, DMA_TO_DEVICE, - DMA_ATTR_SKIP_CPU_SYNC); - goto err_warn; - } - } + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr); + if (dma_coherent_ok(dev, *dma_handle, size)) + goto out_unmap; - *dma_handle = dev_addr; - memset(ret, 0, size); + memset(phys_to_virt(phys_addr), 0, size); + return phys_to_virt(phys_addr); - return ret; +out_unmap: + dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", + (unsigned long long)(dev ? dev->coherent_dma_mask : 0), + (unsigned long long)*dma_handle); -err_warn: - if (warn && printk_ratelimit()) { - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", - dev_name(hwdev), size); + /* + * DMA_TO_DEVICE to avoid memcpy in unmap_single. + * DMA_ATTR_SKIP_CPU_SYNC is optional. + */ + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); +out_warn: + if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) { + dev_warn(dev, + "swiotlb: coherent allocation failed, size=%zu\n", + size); dump_stack(); } - return NULL; } + +void * +swiotlb_alloc_coherent(struct device *hwdev, size_t size, + dma_addr_t *dma_handle, gfp_t flags) +{ + int order = get_order(size); + unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0; + void *ret; + + ret = (void *)__get_free_pages(flags, order); + if (ret) { + *dma_handle = swiotlb_virt_to_bus(hwdev, ret); + if (dma_coherent_ok(hwdev, *dma_handle, size)) { + memset(ret, 0, size); + return ret; + } + free_pages((unsigned long)ret, order); + } + + return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs); +} EXPORT_SYMBOL(swiotlb_alloc_coherent); static bool swiotlb_free_buffer(struct device *dev, size_t size, @@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, { void *vaddr; + /* temporary workaround: */ + if (gfp & __GFP_NOWARN) + attrs |= DMA_ATTR_NO_WARN; + /* * Don't print a warning when the first allocation attempt fails. * swiotlb_alloc_coherent() will print a warning when the DMA memory @@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); if (!vaddr) - vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp); + vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs); return vaddr; } -- cgit v1.2.3-55-g7522 From 4bd89ed39b2ab8dc4ac4b6c59b07d420b0213bec Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Dec 2017 14:25:05 +0100 Subject: swiotlb: remove various exports All these symbols are only used by arch dma_ops implementations or xen-swiotlb. None of which can be modular. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'lib/swiotlb.c') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 0039b7c5e690..c43ec2271469 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -605,7 +605,6 @@ found: return tlb_addr; } -EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single); /* * Allocates bounce buffer and returns its kernel virtual address. @@ -675,7 +674,6 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, } spin_unlock_irqrestore(&io_tlb_lock, flags); } -EXPORT_SYMBOL_GPL(swiotlb_tbl_unmap_single); void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, size_t size, enum dma_data_direction dir, @@ -707,7 +705,6 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, BUG(); } } -EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, size_t size) @@ -884,7 +881,6 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer); } -EXPORT_SYMBOL_GPL(swiotlb_map_page); /* * Unmap a single streaming mode DMA translation. The dma_addr and size must @@ -925,7 +921,6 @@ void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, { unmap_single(hwdev, dev_addr, size, dir, attrs); } -EXPORT_SYMBOL_GPL(swiotlb_unmap_page); /* * Make physical memory consistent for a single streaming mode DMA translation @@ -963,7 +958,6 @@ swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, { swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); } -EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); void swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, @@ -971,7 +965,6 @@ swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, { swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); } -EXPORT_SYMBOL(swiotlb_sync_single_for_device); /* * Map a set of buffers described by scatterlist in streaming mode for DMA. @@ -1023,7 +1016,6 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, } return nelems; } -EXPORT_SYMBOL(swiotlb_map_sg_attrs); /* * Unmap a set of streaming mode DMA translations. Again, cpu read rules @@ -1043,7 +1035,6 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs); } -EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); /* * Make physical memory consistent for a set of streaming mode DMA translations @@ -1071,7 +1062,6 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, { swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); } -EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); void swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, @@ -1079,14 +1069,12 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, { swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); } -EXPORT_SYMBOL(swiotlb_sync_sg_for_device); int swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) { return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer)); } -EXPORT_SYMBOL(swiotlb_dma_mapping_error); /* * Return whether the given device DMA address mask can be supported @@ -1099,7 +1087,6 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask) { return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; } -EXPORT_SYMBOL(swiotlb_dma_supported); #ifdef CONFIG_DMA_DIRECT_OPS void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, -- cgit v1.2.3-55-g7522