Patch largely based on the work of Ian Molton (spyro@f2s.com). Signed-off-by: Dmitry Baryshkov Index: linux-2.6.23/arch/arm/mm/consistent.c =================================================================== --- linux-2.6.23.orig/arch/arm/mm/consistent.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/arch/arm/mm/consistent.c 2007-11-13 01:20:58.281143408 +0300 @@ -3,6 +3,8 @@ * * Copyright (C) 2000-2004 Russell King * + * Device local coherent memory support added by Ian Molton (spyro@f2s.com) + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. @@ -20,6 +22,7 @@ #include #include +#include #include #include @@ -35,6 +38,13 @@ #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT) #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT) +struct dma_coherent_mem { + void *virt_base; + u32 device_base; + int size; + int flags; + unsigned long *bitmap; +}; /* * These are the page tables (2MB each) covering uncached, DMA consistent allocations @@ -153,6 +163,13 @@ unsigned long order; u64 mask = ISA_DMA_THRESHOLD, limit; + /* Following is a work-around (a.k.a. hack) to prevent pages + * with __GFP_COMP being passed to split_page() which cannot + * handle them. The real problem is that this flag probably + * should be 0 on ARM as it is not supported on this + * platform--see CONFIG_HUGETLB_PAGE. */ + gfp &= ~(__GFP_COMP); + if (!consistent_pte[0]) { printk(KERN_ERR "%s: not initialised\n", __func__); dump_stack(); @@ -160,6 +177,26 @@ } if (dev) { + + if (dev->dma_mem) { + unsigned long flags; + int page; + void *ret; + + spin_lock_irqsave(&consistent_lock, flags); + page = bitmap_find_free_region(dev->dma_mem->bitmap, + dev->dma_mem->size, + get_order(size)); + spin_unlock_irqrestore(&consistent_lock, flags); + + if (page >= 0) { + *handle = dev->dma_mem->device_base + (page << PAGE_SHIFT); + ret = dev->dma_mem->virt_base + (page << PAGE_SHIFT); + memset(ret, 0, size); + return ret; + } + } + mask = dev->coherent_dma_mask; /* @@ -177,6 +214,9 @@ mask, (unsigned long long)ISA_DMA_THRESHOLD); goto no_page; } + + if (dev->dma_mem && dev->dma_mem->flags & DMA_MEMORY_EXCLUSIVE) + return NULL; } /* @@ -360,6 +400,8 @@ pte_t *ptep; int idx; u32 off; + struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; + unsigned long order; WARN_ON(irqs_disabled()); @@ -369,6 +411,15 @@ } size = PAGE_ALIGN(size); + order = get_order(size); + + /* What if mem is valid and the range is not? */ + if (mem && cpu_addr >= mem->virt_base && cpu_addr < (mem->virt_base + (mem->size << PAGE_SHIFT))) { + int page = (cpu_addr - mem->virt_base) >> PAGE_SHIFT; + + bitmap_release_region(mem->bitmap, page, order); + return; + } spin_lock_irqsave(&consistent_lock, flags); c = vm_region_find(&consistent_head, (unsigned long)cpu_addr); @@ -438,6 +489,81 @@ } EXPORT_SYMBOL(dma_free_coherent); +int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags) +{ + void *mem_base; + int pages = size >> PAGE_SHIFT; + int bitmap_size = (pages + 31)/32; + + if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) + goto out; + if (!size) + goto out; + if (dev->dma_mem) + goto out; + + /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ + mem_base = ioremap_nocache(bus_addr, size); + if (!mem_base) + goto out; + + dev->dma_mem = kmalloc(GFP_KERNEL, sizeof(struct dma_coherent_mem)); + if (!dev->dma_mem) + goto out; + memset(dev->dma_mem, 0, sizeof(struct dma_coherent_mem)); + dev->dma_mem->bitmap = kmalloc(GFP_KERNEL, bitmap_size); + if (!dev->dma_mem->bitmap) + goto free1_out; + memset(dev->dma_mem->bitmap, 0, bitmap_size); + + dev->dma_mem->virt_base = mem_base; + dev->dma_mem->device_base = device_addr; + dev->dma_mem->size = pages; + dev->dma_mem->flags = flags; + + if (flags & DMA_MEMORY_MAP) + return DMA_MEMORY_MAP; + + return DMA_MEMORY_IO; + + free1_out: + kfree(dev->dma_mem->bitmap); + out: + return 0; +} +EXPORT_SYMBOL(dma_declare_coherent_memory); + +void dma_release_declared_memory(struct device *dev) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + + if (!mem) + return; + dev->dma_mem = NULL; + kfree(mem->bitmap); + kfree(mem); +} +EXPORT_SYMBOL(dma_release_declared_memory); + +void *dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + int pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + int pos, err; + + if (!mem) + return ERR_PTR(-EINVAL); + + pos = (device_addr - mem->device_base) >> PAGE_SHIFT; + err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages)); + if (err != 0) + return ERR_PTR(err); + return mem->virt_base + (pos << PAGE_SHIFT); +} +EXPORT_SYMBOL(dma_mark_declared_memory_occupied); + /* * Initialise the consistent memory allocation. */ Index: linux-2.6.23/arch/arm/common/dmabounce.c =================================================================== --- linux-2.6.23.orig/arch/arm/common/dmabounce.c 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/arch/arm/common/dmabounce.c 2007-11-13 01:23:17.452501736 +0300 @@ -16,6 +16,7 @@ * * Copyright (C) 2002 Hewlett Packard Company. * Copyright (C) 2004 MontaVista Software, Inc. + * Copyright (C) 2007 Dmitry Baryshkov * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -29,6 +30,7 @@ #include #include #include +#include #include @@ -79,6 +81,75 @@ rwlock_t lock; }; +struct dmabounce_check_entry { + struct list_head list; + dmabounce_check checker; + void *data; +}; + +static struct list_head checkers = LIST_HEAD_INIT(checkers); +static rwlock_t checkers_lock = RW_LOCK_UNLOCKED; + +int +dmabounce_register_checker(dmabounce_check function, void *data) +{ + unsigned long flags; + struct dmabounce_check_entry *entry = + kzalloc(sizeof(struct dmabounce_check_entry), GFP_ATOMIC); + + if (!entry) + return ENOMEM; + + INIT_LIST_HEAD(&entry->list); + entry->checker = function; + entry->data = data; + + write_lock_irqsave(checkers_lock, flags); + list_add(&entry->list, &checkers); + write_unlock_irqrestore(checkers_lock, flags); + + return 0; +} + +void +dmabounce_remove_checker(dmabounce_check function, void *data) +{ + unsigned long flags; + struct list_head *pos; + + write_lock_irqsave(checkers_lock, flags); + __list_for_each(pos, &checkers) { + struct dmabounce_check_entry *entry = container_of(pos, + struct dmabounce_check_entry, list); + if (entry->checker == function && entry->data == data) { + list_del(pos); + write_unlock_irqrestore(checkers_lock, flags); + kfree(entry); + return; + } + } + + printk(KERN_WARNING "dmabounce checker not found: %p\n", function); +} + +int dma_needs_bounce(struct device *dev, dma_addr_t dma, size_t size) +{ + unsigned long flags; + struct list_head *pos; + + read_lock_irqsave(checkers_lock, flags); + __list_for_each(pos, &checkers) { + struct dmabounce_check_entry *entry = container_of(pos, + struct dmabounce_check_entry, list); + if (entry->checker(dev, dma, size, entry->data)) { + read_unlock_irqrestore(checkers_lock, flags); + return 1; + } + } + + read_unlock_irqrestore(checkers_lock, flags); + return 0; +} #ifdef STATS static ssize_t dmabounce_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -642,7 +713,6 @@ dev->bus_id, dev->bus->name); } - EXPORT_SYMBOL(dma_map_single); EXPORT_SYMBOL(dma_unmap_single); EXPORT_SYMBOL(dma_map_sg); @@ -652,6 +722,9 @@ EXPORT_SYMBOL(dma_sync_sg); EXPORT_SYMBOL(dmabounce_register_dev); EXPORT_SYMBOL(dmabounce_unregister_dev); +EXPORT_SYMBOL(dmabounce_register_checker); +EXPORT_SYMBOL(dmabounce_remove_checker); + MODULE_AUTHOR("Christopher Hoover , Deepak Saxena "); MODULE_DESCRIPTION("Special dma_{map/unmap/dma_sync}_* routines for systems with limited DMA windows"); Index: linux-2.6.23/include/asm-arm/dma-mapping.h =================================================================== --- linux-2.6.23.orig/include/asm-arm/dma-mapping.h 2007-10-10 00:31:38.000000000 +0400 +++ linux-2.6.23/include/asm-arm/dma-mapping.h 2007-11-13 01:24:05.588500474 +0300 @@ -7,6 +7,18 @@ #include +#define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY +extern int +dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags); + +extern void +dma_release_declared_memory(struct device *dev); + +extern void * +dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size); + /* * DMA-consistent mapping functions. These allocate/free a region of * uncached, unwrite-buffered mapped memory space for use with DMA @@ -433,23 +445,10 @@ */ extern void dmabounce_unregister_dev(struct device *); -/** - * dma_needs_bounce - * - * @dev: valid struct device pointer - * @dma_handle: dma_handle of unbounced buffer - * @size: size of region being mapped - * - * Platforms that utilize the dmabounce mechanism must implement - * this function. - * - * The dmabounce routines call this function whenever a dma-mapping - * is requested to determine whether a given buffer needs to be bounced - * or not. The function must return 0 if the buffer is OK for - * DMA access and 1 if the buffer needs to be bounced. - * - */ -extern int dma_needs_bounce(struct device*, dma_addr_t, size_t); +typedef int (*dmabounce_check)(struct device *dev, dma_addr_t dma, size_t size, void *data); +extern int dmabounce_register_checker(dmabounce_check, void *data); +extern void dmabounce_remove_checker(dmabounce_check, void *data); + #endif /* CONFIG_DMABOUNCE */ #endif /* __KERNEL__ */