[ARM PATCH] 1255/1: [PATCH] SA-1111 PCI support for USB

Fixes several oopsen in the SA-1111 "fake" PCI support

Complete re-write of the SA-1111 DMA bug "bounce buffer" workaround.

Merge latest drivers/pci/pool.c into mach-sa1100/pcipool.c (pool
allocation debugging follows CONFIG_DEBUG_SLAB a la drivers/pci/pool.c)

/arch/arm/mach-sa1100/pcipool.h can be deleted
(unrelated: /arch/arm/mach-sa1100/sa1111-ohci.c can be deleted)

Applies to 2.5.30-rmk1.  Should be back ported to 2.4 as the existing
SA-1111 bounce buffer code is broken.

(This patch is *required* for the OHCI HCD driver that is part of 2.5.21+.)

arch/arm/mach-sa1100/Makefile

@@ -27,7 +27,7 @@ endif
 
 # Next, the SA1111 stuff.
 obj-$(CONFIG_SA1111)			+= sa1111.o
-obj-$(CONFIG_USB_OHCI_SA1111)	+= sa1111-pcibuf.o pcipool.o
+obj-$(CONFIG_USB_OHCI_HCD)		+= sa1111-pcibuf.o pcipool.o
 
 # Specific board support
 obj-$(CONFIG_SA1100_ADSBITSY)		+= adsbitsy.o

arch/arm/mach-sa1100/pcipool.c

@@ -20,8 +20,6 @@
 
 #include <asm/page.h>
 
-#include "pcipool.h"
-
 /*
  * Pool allocator ... wraps the pci_alloc_consistent page allocator, so
  * small blocks are easily used by drivers for bus mastering controllers.
@@ -33,7 +31,6 @@ struct pci_pool {	/* the pool */
 	spinlock_t		lock;
 	size_t			blocks_per_page;
 	size_t			size;
-	int			flags;
 	struct pci_dev		*dev;
 	size_t			allocation;
 	char			name [32];
@@ -52,6 +49,19 @@ struct pci_page {	/* cacheable header for 'allocation' bytes */
 
 // #define CONFIG_PCIPOOL_DEBUG
 
+static inline const char *slot_name(const struct pci_pool *pool)
+{
+	const struct pci_dev *pdev = pool->dev;
+
+	if (pdev == 0)
+		return "[0]";
+
+	else if (dev_is_sa1111(pdev))
+		return "[SA-1111]";
+	else
+		return pdev->slot_name;
+}
+
 
 /**
  * pci_pool_create - Creates a pool of pci consistent memory blocks, for dma.
@@ -60,7 +70,7 @@ struct pci_page {	/* cacheable header for 'allocation' bytes */
  * @size: size of the blocks in this pool.
  * @align: alignment requirement for blocks; must be a power of two
  * @allocation: returned blocks won't cross this boundary (or zero)
- * @flags: SLAB_* flags (not all are supported).
+ * @mem_flags: SLAB_* flags.
  *
  * Returns a pci allocation pool with the requested characteristics, or
  * null if one can't be created.  Given one of these pools, pci_pool_alloc()
@@ -76,7 +86,7 @@ struct pci_page {	/* cacheable header for 'allocation' bytes */
  */
 struct pci_pool *
 pci_pool_create (const char *name, struct pci_dev *pdev,
-	size_t size, size_t align, size_t allocation, int flags)
+	size_t size, size_t align, size_t allocation, int mem_flags)
 {
 	struct pci_pool		*retval;
 
@@ -100,13 +110,9 @@ pci_pool_create (const char *name, struct pci_dev *pdev,
 	} else if (allocation < size)
 		return 0;
 
-	if (!(retval = kmalloc (sizeof *retval, flags)))
+	if (!(retval = kmalloc (sizeof *retval, mem_flags)))
 		return retval;
 
-#ifdef	CONFIG_PCIPOOL_DEBUG
-	flags |= SLAB_POISON;
-#endif
-
 	strncpy (retval->name, name, sizeof retval->name);
 	retval->name [sizeof retval->name - 1] = 0;
 
@@ -114,14 +120,13 @@ pci_pool_create (const char *name, struct pci_dev *pdev,
 	INIT_LIST_HEAD (&retval->page_list);
 	spin_lock_init (&retval->lock);
 	retval->size = size;
-	retval->flags = flags;
 	retval->allocation = allocation;
 	retval->blocks_per_page = allocation / size;
 	init_waitqueue_head (&retval->waitq);
 
 #ifdef CONFIG_PCIPOOL_DEBUG
 	printk (KERN_DEBUG "pcipool create %s/%s size %d, %d/page (%d alloc)\n",
-		pdev ? pdev->slot_name : NULL, retval->name, size,
+		slot_name(retval), retval->name, size,
 		retval->blocks_per_page, allocation);
 #endif
 
@@ -143,11 +148,13 @@ pool_alloc_page (struct pci_pool *pool, int mem_flags)
 	if (!page)
 		return 0;
 	page->vaddr = pci_alloc_consistent (pool->dev,
-				pool->allocation, &page->dma);
+					    pool->allocation,
+					    &page->dma);
 	if (page->vaddr) {
 		memset (page->bitmap, 0xff, mapsize);	// bit set == free
-		if (pool->flags & SLAB_POISON)
+#ifdef	CONFIG_DEBUG_SLAB
 		memset (page->vaddr, POOL_POISON_BYTE, pool->allocation);
+#endif
 		list_add (&page->page_list, &pool->page_list);
 	} else {
 		kfree (page);
@@ -173,8 +180,9 @@ pool_free_page (struct pci_pool *pool, struct pci_page *page)
 {
 	dma_addr_t	dma = page->dma;
 
-	if (pool->flags & SLAB_POISON)
+#ifdef	CONFIG_DEBUG_SLAB
 	memset (page->vaddr, POOL_POISON_BYTE, pool->allocation);
+#endif
 	pci_free_consistent (pool->dev, pool->allocation, page->vaddr, dma);
 	list_del (&page->page_list);
 	kfree (page);
@@ -195,8 +203,7 @@ pci_pool_destroy (struct pci_pool *pool)
 
 #ifdef CONFIG_PCIPOOL_DEBUG
 	printk (KERN_DEBUG "pcipool destroy %s/%s\n",
-		pool->dev ? pool->dev->slot_name : NULL,
-		pool->name);
+		slot_name(pool), pool->name);
 #endif
 
 	spin_lock_irqsave (&pool->lock, flags);
@@ -206,8 +213,7 @@ pci_pool_destroy (struct pci_pool *pool)
 				struct pci_page, page_list);
 		if (is_page_busy (pool->blocks_per_page, page->bitmap)) {
 			printk (KERN_ERR "pci_pool_destroy %s/%s, %p busy\n",
-				pool->dev ? pool->dev->slot_name : NULL,
-				pool->name, page->vaddr);
+				slot_name(pool), pool->name, page->vaddr);
 			/* leak the still-in-use consistent memory */
 			list_del (&page->page_list);
 			kfree (page);
@@ -327,35 +333,32 @@ pci_pool_free (struct pci_pool *pool, void *vaddr, dma_addr_t dma)
 	int			map, block;
 
 	if ((page = pool_find_page (pool, dma)) == 0) {
-		printk (KERN_ERR "pci_pool_free %s/%s, %p/%x (bad dma)\n",
-			pool->dev ? pool->dev->slot_name : NULL,
-			pool->name, vaddr, dma);
-		return;
-	}
-#ifdef	CONFIG_PCIPOOL_DEBUG
-	if (((dma - page->dma) + (void *)page->vaddr) != vaddr) {
-		printk (KERN_ERR "pci_pool_free %s/%s, %p (bad vaddr)/%x\n",
+		printk (KERN_ERR "pci_pool_free %s/%s, %p/%lx (bad dma)\n",
 			pool->dev ? pool->dev->slot_name : NULL,
-			pool->name, vaddr, dma);
+			pool->name, vaddr, (unsigned long) dma);
 		return;
 	}
-#endif
 
 	block = dma - page->dma;
 	block /= pool->size;
 	map = block / BITS_PER_LONG;
 	block %= BITS_PER_LONG;
 
-#ifdef	CONFIG_PCIPOOL_DEBUG
+#ifdef	CONFIG_DEBUG_SLAB
+	if (((dma - page->dma) + (void *)page->vaddr) != vaddr) {
+		printk (KERN_ERR "pci_pool_free %s/%s, %p (bad vaddr)/%lx\n",
+			pool->dev ? pool->dev->slot_name : NULL,
+			pool->name, vaddr, (unsigned long) dma);
+		return;
+	}
 	if (page->bitmap [map] & (1UL << block)) {
 		printk (KERN_ERR "pci_pool_free %s/%s, dma %x already free\n",
 			pool->dev ? pool->dev->slot_name : NULL,
 			pool->name, dma);
 		return;
 	}
-#endif
-	if (pool->flags & SLAB_POISON)
 	memset (vaddr, POOL_POISON_BYTE, pool->size);
+#endif
 
 	spin_lock_irqsave (&pool->lock, flags);
 	set_bit (block, &page->bitmap [map]);
@@ -369,9 +372,19 @@ pci_pool_free (struct pci_pool *pool, void *vaddr, dma_addr_t dma)
 	spin_unlock_irqrestore (&pool->lock, flags);
 }
 
-
 EXPORT_SYMBOL (pci_pool_create);
 EXPORT_SYMBOL (pci_pool_destroy);
 EXPORT_SYMBOL (pci_pool_alloc);
 EXPORT_SYMBOL (pci_pool_free);
 
+/* **************************************** */
+
+static int __init pcipool_init(void)
+{
+	MOD_INC_USE_COUNT;	/* never unload */
+
+	return 0;
+}
+module_init(pcipool_init);
+
+MODULE_LICENSE("GPL");

arch/arm/mach-sa1100/pcipool.h

-
-struct pci_pool *pci_pool_create (const char *name, struct pci_dev *dev,
-		size_t size, size_t align, size_t allocation, int flags);
-void pci_pool_destroy (struct pci_pool *pool);
-
-void *pci_pool_alloc (struct pci_pool *pool, int flags, dma_addr_t *handle);
-void pci_pool_free (struct pci_pool *pool, void *vaddr, dma_addr_t addr);
-

arch/arm/mach-sa1100/sa1111-pcibuf.c

 /*
  *  linux/arch/arm/mach-sa1100/pci-sa1111.c
  *
- *  Special pci_map/unmap_single routines for SA-1111.   These functions
- *  compensate for a bug in the SA-1111 hardware which don't allow DMA
- *  to/from addresses above 1MB.
+ *  Special pci_{map/unmap/dma_sync}_* routines for SA-1111.
  *
- *  Brad Parker (brad@heeltoe.com)
+ *  These functions utilize bouncer buffers to compensate for a bug in
+ *  the SA-1111 hardware which don't allow DMA to/from addresses
+ *  certain addresses above 1MB.
  *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
+ *  Re-written by Christopher Hoover <ch@murgatroid.com>
+ *  Original version by Brad Parker (brad@heeltoe.com)
  *
- *  06/13/2001 - created.
- */
+ *  Copyright (C) 2002 Hewlett Packard Company.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  version 2 as published by the Free Software Foundation.
+ * */
+
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
+#include <linux/list.h>
+#include <asm/hardware/sa1111.h>
 
-#include "pcipool.h"
+//#define DEBUG
+#ifdef DEBUG
+#define DPRINTK(...) do { printk(KERN_DEBUG __VA_ARGS__); } while (0)
+#else
+#define DPRINTK(...) do { } while (0)
+#endif
 
-/*
- * simple buffer allocator for copying of unsafe to safe buffers
- * uses __alloc/__free for actual buffers
- * keeps track of safe buffers we've allocated so we can recover the
- * unsafe buffers.
- */
 
-#define MAX_SAFE	32
-#define SIZE_SMALL	1024
-#define SIZE_LARGE	(16*1024)
+struct safe_buffer {
+	struct list_head node;
+
+	/* original request */
+	void		*ptr;
+	size_t		size;
+	int		direction;
+
+	/* safe buffer info */
+	struct pci_pool *pool;
+	void		*safe;
+	dma_addr_t	safe_dma_addr;
+};
+
+LIST_HEAD(safe_buffers);
 
-static long mapped_alloc_size;
-static char *safe_buffers[MAX_SAFE][2];
 
+#define SIZE_SMALL	1024
+#define SIZE_LARGE	(16*1024)
 
-static struct pci_pool *small_buffer_cache, *large_buffer_cache;
+static struct pci_pool *small_buffer_pool, *large_buffer_pool;
 
-static int
-init_safe_buffers(struct pci_dev *dev)
+static int __init
+create_safe_buffer_pools(void)
 {
-	small_buffer_cache = pci_pool_create("pci_small_buffer",
-					    dev,
+	small_buffer_pool = pci_pool_create("sa1111_small_dma_buffer",
+					    SA1111_FAKE_PCIDEV,
 					    SIZE_SMALL,
 					    0 /* byte alignment */,
 					    0 /* no page-crossing issues */,
-					    GFP_KERNEL | GFP_DMA);
-
-	if (small_buffer_cache == 0)
+					    SLAB_KERNEL);
+	if (0 == small_buffer_pool) {
+		printk(KERN_ERR
+		       "sa1111_pcibuf: could not allocate small pci pool\n");
 		return -1;
+	}
 
-	large_buffer_cache = pci_pool_create("pci_large_buffer",
-					    dev,
+	large_buffer_pool = pci_pool_create("sa1111_large_dma_buffer",
+					    SA1111_FAKE_PCIDEV,
 					    SIZE_LARGE,
 					    0 /* byte alignment */,
 					    0 /* no page-crossing issues */,
-					    GFP_KERNEL | GFP_DMA);
-	if (large_buffer_cache == 0)
+					    SLAB_KERNEL);
+	if (0 == large_buffer_pool) {
+		printk(KERN_ERR
+		       "sa1111_pcibuf: could not allocate large pci pool\n");
+		pci_pool_destroy(small_buffer_pool);
+		small_buffer_pool = 0;
 		return -1;
+	}
 
 	return 0;
 }
 
+static void __exit
+destroy_safe_buffer_pools(void)
+{
+	if (small_buffer_pool)
+		pci_pool_destroy(small_buffer_pool);
+	if (large_buffer_pool)
+		pci_pool_destroy(large_buffer_pool);
+
+	small_buffer_pool = large_buffer_pool = 0;
+}
+
+
 /* allocate a 'safe' buffer and keep track of it */
-static char *
-alloc_safe_buffer(char *unsafe, int size, dma_addr_t *pbus)
+static struct safe_buffer *
+alloc_safe_buffer(struct pci_dev *hwdev, void *ptr, size_t size, int direction)
 {
-	char *safe;
-	dma_addr_t busptr;
+	struct safe_buffer *buf;
 	struct pci_pool *pool;
-	int i;
+	void *safe;
+	dma_addr_t safe_dma_addr;
 
-	if (0) printk("alloc_safe_buffer(size=%d)\n", size);
+	DPRINTK("%s(ptr=%p, size=%d, direction=%d)\n",
+		__func__, ptr, size, direction);
 
-	if (size <= SIZE_SMALL)
-		pool = small_buffer_cache;
-	else
-		if (size < SIZE_LARGE)
-			pool = large_buffer_cache;
-				else
-					return 0;
-
-	safe = pci_pool_alloc(pool, SLAB_ATOMIC, &busptr);
-	if (safe == 0)
+	buf = kmalloc(sizeof(struct safe_buffer), GFP_ATOMIC);
+	if (buf == 0) {
+		printk(KERN_WARNING "%s: kmalloc failed\n", __func__);
 		return 0;
-
-	for (i = 0; i < MAX_SAFE; i++)
-		if (safe_buffers[i][0] == 0) {
-			break;
 	}
 
-	if (i == MAX_SAFE) {
-		panic(__FILE__ ": exceeded MAX_SAFE buffers");
+	if (size <= SIZE_SMALL) {
+		pool = small_buffer_pool;
+		safe = pci_pool_alloc(pool, GFP_ATOMIC, &safe_dma_addr);
+	} else if (size <= SIZE_LARGE) {
+		pool = large_buffer_pool;
+		safe = pci_pool_alloc(pool, GFP_ATOMIC, &safe_dma_addr);
+	} else {
+		printk(KERN_DEBUG
+		       "sa111_pcibuf: resorting to pci_alloc_consistent\n");
+		pool = 0;
+		safe = pci_alloc_consistent(SA1111_FAKE_PCIDEV, size,
+					    &safe_dma_addr);
 	}
 
-	/* place the size index and the old buffer ptr in the first 8 bytes
-	 * and return a ptr + 12 to caller
-	 */
-	((int *)safe)[0] = i;
-	((char **)safe)[1] = (char *)pool;
-	((char **)safe)[2] = unsafe;
+	if (safe == 0) {
+		printk(KERN_WARNING
+		       "%s: could not alloc dma memory (size=%d)\n",
+		       __func__, size);
+		kfree(buf);
+		return 0;
+	}
 
-	busptr += sizeof(int) + sizeof(char *) + sizeof(char *);
+	BUG_ON(sa1111_check_dma_bug(safe_dma_addr));	// paranoia
 
-	safe_buffers[i][0] = (void *)busptr;
-	safe_buffers[i][1] = (void *)safe;
+	buf->ptr = ptr;
+	buf->size = size;
+	buf->direction = direction;
+	buf->pool = pool;
+	buf->safe = safe;
+	buf->safe_dma_addr = safe_dma_addr;
 
-	safe += sizeof(int) + sizeof(char *) + sizeof(char *);
+	MOD_INC_USE_COUNT;
+	list_add(&buf->node, &safe_buffers);
 
-	*pbus = busptr;
-	return safe;
+	return buf;
 }
 
 /* determine if a buffer is from our "safe" pool */
-static char *
-find_safe_buffer(char *busptr, char **unsafe)
+static struct safe_buffer *
+find_safe_buffer(dma_addr_t safe_dma_addr)
 {
-	int i;
-	char *buf;
-
-	for (i = 0; i < MAX_SAFE; i++) {
-		if (safe_buffers[i][0] == busptr) {
-			if (0) printk("find_safe_buffer(%p) found @ %d\n", busptr, i);
-			buf = safe_buffers[i][1];
-			*unsafe = ((char **)buf)[2];
-			return buf + sizeof(int) + sizeof(char *) + sizeof(char *);
+	struct list_head *entry;
+
+	list_for_each(entry, &safe_buffers) {
+		struct safe_buffer *b =
+			list_entry(entry, struct safe_buffer, node);
+
+		if (b->safe_dma_addr == safe_dma_addr) {
+			return b;
 		}
 	}
 
-	return (char *)0;
+	return 0;
 }
 
 static void
-free_safe_buffer(char *buf)
+free_safe_buffer(struct safe_buffer *buf)
 {
-	int index;
-	struct pci_pool *pool;
-	char *dma;
+	DPRINTK("%s(buf=%p)\n", __func__, buf);
 
-	if (0) printk("free_safe_buffer(buf=%p)\n", buf);
+	list_del(&buf->node);
 
-	/* retrieve the buffer size index */
-	buf -= sizeof(int) + sizeof(char*) + sizeof(char*);
-	index = ((int *)buf)[0];
-	pool = (struct pci_pool *)((char **)buf)[1];
+	if (buf->pool)
+		pci_pool_free(buf->pool, buf->safe, buf->safe_dma_addr);
+	else
+		pci_free_consistent(SA1111_FAKE_PCIDEV, buf->size, buf->safe,
+				    buf->safe_dma_addr);
+	kfree(buf);
 
-	if (0) printk("free_safe_buffer(%p) index %d\n",
-		      buf, index);
+	MOD_DEC_USE_COUNT;
+}
 
-	if (index < 0 || index >= MAX_SAFE) {
-		printk(__FILE__ ": free_safe_buffer() corrupt buffer\n");
-		return;
-	}
+static inline int
+dma_range_is_safe(dma_addr_t addr, size_t size)
+{
+	unsigned int physaddr = SA1111_DMA_ADDR((unsigned int) addr);
 
-	dma = safe_buffers[index][0];
-	safe_buffers[index][0] = 0;
+	/* Any address within one megabyte of the start of the target
+         * bank will be OK.  This is an overly conservative test:
+         * other addresses can be OK depending on the dram
+         * configuration.  (See sa1111.c:sa1111_check_dma_bug() * for
+         * details.)
+	 *
+	 * We take care to ensure the entire dma region is within
+	 * the safe range.
+	 */
 
-	pci_pool_free(pool, buf, (u32)dma);
+	return ((physaddr + size - 1) < (1<<20));
 }
 
-/*
-  NOTE:
-  replace pci_map/unmap_single with local routines which will
-  do buffer copies if buffer is above 1mb...
-*/
-
 /*
  * see if a buffer address is in an 'unsafe' range.  if it is
  * allocate a 'safe' buffer and copy the unsafe buffer into it.
  * substitute the safe buffer for the unsafe one.
  * (basically move the buffer from an unsafe area to a safe one)
- *
- * we assume calls to map_single are symmetric with calls to unmap_single...
  */
 dma_addr_t
-sa1111_map_single(struct pci_dev *hwdev, void *virtptr,
-	       size_t size, int direction)
+sa1111_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction)
 {
-	dma_addr_t busptr;
+	unsigned long flags;
+	dma_addr_t dma_addr;
 
-	mapped_alloc_size += size;
+	DPRINTK("%s(hwdev=%p,ptr=%p,size=%d,dir=%x)\n",
+	       __func__, hwdev, ptr, size, direction);
 
-	if (0) printk("pci_map_single(hwdev=%p,ptr=%p,size=%d,dir=%x) "
-		      "alloced=%ld\n",
-		      hwdev, virtptr, size, direction, mapped_alloc_size);
+	BUG_ON(hwdev != SA1111_FAKE_PCIDEV);
+	BUG_ON(direction == PCI_DMA_NONE);
 
-	busptr = virt_to_bus(virtptr);
+	local_irq_save(flags);
 
-	/* we assume here that a buffer will never be >=64k */
-	if ( (((unsigned long)busptr) & 0x100000) ||
-	     ((((unsigned long)busptr)+size) & 0x100000) )
-	{
-		char *safe;
+	dma_addr = virt_to_bus(ptr);
 
-		safe = alloc_safe_buffer(virtptr, size, &busptr);
-		if (safe == 0) {
-			printk("unable to map unsafe buffer %p!\n", virtptr);
+	if (!dma_range_is_safe(dma_addr, size)) {
+		struct safe_buffer *buf;
+
+		buf = alloc_safe_buffer(hwdev, ptr, size, direction);
+		if (buf == 0) {
+			printk(KERN_ERR
+			       "%s: unable to map unsafe buffer %p!\n",
+			       __func__, ptr);
+			local_irq_restore(flags);
 			return 0;
 		}
 
-		if (0) printk("unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
-			      virtptr, (void *)virt_to_bus(virtptr),
-			      safe, (void *)busptr);
+		DPRINTK("%s: unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
+			__func__,
+			buf->ptr, (void *) virt_to_bus(buf->ptr),
+			buf->safe, (void *) buf->safe_dma_addr);
 
-		memcpy(safe, virtptr, size);
-		consistent_sync(safe, size, direction);
+		if ((direction == PCI_DMA_TODEVICE) ||
+		    (direction == PCI_DMA_BIDIRECTIONAL)) {
+			DPRINTK("%s: copy out from unsafe %p, to safe %p, size %d\n",
+				__func__, ptr, buf->safe, size);
+			memcpy(buf->safe, ptr, size);
+		}
+		consistent_sync(buf->safe, size, direction);
 
-		return busptr;
+		dma_addr = buf->safe_dma_addr;
+	} else {
+		consistent_sync(ptr, size, direction);
 	}
 
-	consistent_sync(virtptr, size, direction);
-	return busptr;
+	local_irq_restore(flags);
+	return dma_addr;
 }
 
 /*
- * see if a mapped address was really a "safe" buffer and if so,
- * copy the data from the safe buffer back to the unsafe buffer
- * and free up the safe buffer.
- * (basically return things back to the way they should be)
+ * see if a mapped address was really a "safe" buffer and if so, copy
+ * the data from the safe buffer back to the unsafe buffer and free up
+ * the safe buffer.  (basically return things back to the way they
+ * should be)
  */
+
 void
 sa1111_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr,
 		    size_t size, int direction)
 {
-	char *safe, *unsafe;
-	void *buf;
+	unsigned long flags;
+	struct safe_buffer *buf;
+
+	DPRINTK("%s(hwdev=%p,ptr=%p,size=%d,dir=%x)\n",
+		__func__, hwdev, (void *) dma_addr, size, direction);
+
+	BUG_ON(hwdev != SA1111_FAKE_PCIDEV);
+	BUG_ON(direction == PCI_DMA_NONE);
+
+	local_irq_save(flags);
 
-	/* hack; usb-ohci.c never sends hwdev==NULL, all others do */
-	if (hwdev == NULL) {
-		return;
+	buf = find_safe_buffer(dma_addr);
+	if (buf) {
+		BUG_ON(buf->size != size);
+		BUG_ON(buf->direction != direction);
+
+		DPRINTK("%s: unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
+			__func__,
+			buf->ptr, (void *) virt_to_bus(buf->ptr),
+			buf->safe, (void *) buf->safe_dma_addr);
+
+		if ((direction == PCI_DMA_FROMDEVICE) ||
+		    (direction == PCI_DMA_BIDIRECTIONAL)) {
+			DPRINTK("%s: copy back from safe %p, to unsafe %p size %d\n",
+				__func__, buf->safe, buf->ptr, size);
+			memcpy(buf->ptr, buf->safe, size);
+		}
+		free_safe_buffer(buf);
 	}
 
-	mapped_alloc_size -= size;
+	local_irq_restore(flags);
+}
 
-	if (0) printk("pci_unmap_single(hwdev=%p,ptr=%p,size=%d,dir=%x) "
-		      "alloced=%ld\n",
-		      hwdev, (void *)dma_addr, size, direction,
-		      mapped_alloc_size);
+int
+sa1111_map_sg(struct pci_dev *hwdev, struct scatterlist *sg,
+	      int nents, int direction)
+{
+	BUG();			/* Not implemented. */
+}
 
-	if ((safe = find_safe_buffer((void *)dma_addr, &unsafe))) {
-		if (0) printk("copyback unsafe %p, safe %p, size %d\n",
-			      unsafe, safe, size);
+void
+sa1111_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents,
+	     int direction)
+{
+	BUG();			/* Not implemented. */
+}
 
-		consistent_sync(safe, size, PCI_DMA_FROMDEVICE);
-		memcpy(unsafe, safe, size);
-		free_safe_buffer(safe);
+void
+sa1111_dma_sync_single(struct pci_dev *hwdev, dma_addr_t dma_addr,
+		       size_t size, int direction)
+{
+	unsigned long flags;
+	struct safe_buffer *buf;
+
+	DPRINTK("%s(hwdev=%p,ptr=%p,size=%d,dir=%x)\n",
+		__func__, hwdev, (void *) dma_addr, size, direction);
+
+	BUG_ON(hwdev != SA1111_FAKE_PCIDEV);
+
+	local_irq_save(flags);
+
+	buf = find_safe_buffer(dma_addr);
+	if (buf) {
+		BUG_ON(buf->size != size);
+		BUG_ON(buf->direction != direction);
+
+		DPRINTK("%s: unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
+			__func__,
+			buf->ptr, (void *) virt_to_bus(buf->ptr),
+			buf->safe, (void *) buf->safe_dma_addr);
+
+		switch (direction) {
+		case PCI_DMA_FROMDEVICE:
+			DPRINTK("%s: copy back from safe %p, to unsafe %p size %d\n",
+				__func__, buf->safe, buf->ptr, size);
+			memcpy(buf->ptr, buf->safe, size);
+			break;
+		case PCI_DMA_TODEVICE:
+			DPRINTK("%s: copy out from unsafe %p, to safe %p, size %d\n",
+				__func__,buf->ptr, buf->safe, size);
+			memcpy(buf->safe, buf->ptr, size);
+			break;
+		case PCI_DMA_BIDIRECTIONAL:
+			BUG();	/* is this allowed?  what does it mean? */
+		default:
+			BUG();
+		}
+		consistent_sync(buf->safe, size, direction);
 	} else {
-		/* assume this is normal memory */
-		buf = bus_to_virt(dma_addr);
-		consistent_sync(buf, size, PCI_DMA_FROMDEVICE);
+		consistent_sync(bus_to_virt(dma_addr), size, direction);
 	}
+
+	local_irq_restore(flags);
+}
+
+void
+sa1111_dma_sync_sg(struct pci_dev *hwdev, struct scatterlist *sg,
+		   int nelems, int direction)
+{
+	BUG();			/* Not implemented. */
 }
 
 EXPORT_SYMBOL(sa1111_map_single);
 EXPORT_SYMBOL(sa1111_unmap_single);
+EXPORT_SYMBOL(sa1111_map_sg);
+EXPORT_SYMBOL(sa1111_unmap_sg);
+EXPORT_SYMBOL(sa1111_dma_sync_single);
+EXPORT_SYMBOL(sa1111_dma_sync_sg);
+
+/* **************************************** */
 
-static int __init sa1111_init_safe_buffers(void)
+static int __init sa1111_pcibuf_init(void)
 {
-	printk("Initializing SA1111 buffer pool for DMA workaround\n");
-	init_safe_buffers(NULL);
-	return 0;
+	int ret;
+
+	printk(KERN_DEBUG
+	       "sa1111_pcibuf: initializing SA-1111 DMA workaround\n");
+
+	ret = create_safe_buffer_pools();
+
+	return ret;
 }
+module_init(sa1111_pcibuf_init);
 
-static void free_safe_buffers(void)
+static void __exit sa1111_pcibuf_exit(void)
 {
-	pci_pool_destroy(small_buffer_cache);
-	pci_pool_destroy(large_buffer_cache);
+	BUG_ON(!list_empty(&safe_buffers));
+
+	destroy_safe_buffer_pools();
 }
+module_exit(sa1111_pcibuf_exit);
 
-module_init(sa1111_init_safe_buffers);
-module_exit(free_safe_buffers);
+MODULE_AUTHOR("Christopher Hoover <ch@hpl.hp.com>");
+MODULE_DESCRIPTION("Special pci_{map/unmap/dma_sync}_* routines for SA-1111.");
+MODULE_LICENSE("GPL");

arch/arm/mm/consistent.c

@@ -98,7 +98,8 @@ void *pci_alloc_consistent(struct pci_dev *hwdev, size_t size, dma_addr_t *handl
 {
 	int gfp = GFP_KERNEL;
 
-	if (hwdev == NULL || hwdev->dma_mask != 0xffffffff)
+	if (hwdev == NULL || dev_is_sa1111(hwdev) ||
+	    hwdev->dma_mask != 0xffffffff)
 		gfp |= GFP_DMA;
 
 	return consistent_alloc(gfp, size, handle);

include/asm-arm/hardware/sa1111.h

@@ -611,10 +611,13 @@ struct sa1111_device {
 
 extern struct sa1111_device *sa1111;
 
+int sa1111_check_dma_bug(dma_addr_t addr);
+
 /*
  * These frob the SKPCR register.
  */
 void sa1111_enable_device(unsigned int mask);
 void sa1111_disable_device(unsigned int mask);
 
+
 #endif  /* _ASM_ARCH_SA1111 */

include/asm-arm/pci.h

@@ -12,6 +12,34 @@
 
 struct pci_dev;
 
+/*
+ * For SA-1111 these functions are "magic" and utilize bounce
+ * buffers as need to workaround SA-1111 DMA bugs.  They are called in
+ * place of their pci_* counterparts when dev_is_sa1111() returns true.
+ */
+dma_addr_t sa1111_map_single(struct pci_dev *, void *, size_t, int);
+void sa1111_unmap_single(struct pci_dev *, dma_addr_t, size_t, int);
+int sa1111_map_sg(struct pci_dev *, struct scatterlist *, int, int);
+void sa1111_unmap_sg(struct pci_dev *, struct scatterlist *, int, int);
+void sa1111_dma_sync_single(struct pci_dev *, dma_addr_t, size_t, int);
+void sa1111_dma_sync_sg(struct pci_dev *, struct scatterlist *, int, int);
+
+#ifdef CONFIG_SA1111
+
+#define SA1111_FAKE_PCIDEV ((struct pci_dev *) 1111)
+
+static inline int dev_is_sa1111(const struct pci_dev *dev)
+{
+	return (dev == SA1111_FAKE_PCIDEV);
+}
+
+#else
+
+static inline int dev_is_sa1111(const struct pci_dev *dev) { return 0; }
+
+#endif
+
+
 static inline void pcibios_set_master(struct pci_dev *dev)
 {
 	/* No special bus mastering setup handling */
@@ -61,17 +89,9 @@ pci_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr,
 static inline dma_addr_t
 pci_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction)
 {
-#ifdef CONFIG_SA1111
-	extern dma_addr_t sa1111_map_single(struct pci_dev *, void *, size_t, int);
-
-	/*
-	 * for SA1111 these functions are "magic" and relocate buffers.  We
-	 * only need to do these if hwdev is non-null; otherwise we expect
-	 * the buffer to already be suitable for DMA.
-	 */
-	if (hwdev != NULL)
+	if (dev_is_sa1111(hwdev))
 		return sa1111_map_single(hwdev, ptr, size, direction);
-#endif
+
 	consistent_sync(ptr, size, direction);
 	return virt_to_bus(ptr);
 }
@@ -86,12 +106,9 @@ pci_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction)
 static inline void
 pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr, size_t size, int direction)
 {
-#ifdef CONFIG_SA1111
-	extern void sa1111_unmap_single(struct pci_dev *, dma_addr_t, size_t, int);
-
-	if (hwdev != NULL)
+	if (dev_is_sa1111(hwdev))
 		sa1111_unmap_single(hwdev, dma_addr, size, direction);
-#endif
+
 	/* nothing to do */
 }
 
@@ -99,7 +116,7 @@ pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr, size_t size, int di
  * Whether pci_unmap_{single,page} is a nop depends upon the
  * configuration.
  */
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) || defined(CONFIG_SA1111)
 #define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)	dma_addr_t ADDR_NAME;
 #define DECLARE_PCI_UNMAP_LEN(LEN_NAME)		__u32 LEN_NAME;
 #define pci_unmap_addr(PTR, ADDR_NAME)		((PTR)->ADDR_NAME)
@@ -135,6 +152,9 @@ pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int directi
 {
 	int i;
 
+	if (dev_is_sa1111(hwdev))
+		return sa1111_map_sg(hwdev, sg, nents, direction);
+
 	for (i = 0; i < nents; i++, sg++) {
 		char *virt;
 
@@ -153,6 +173,11 @@ pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int directi
 static inline void
 pci_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction)
 {
+	if (dev_is_sa1111(hwdev)) {
+		sa1111_unmap_sg(hwdev, sg, nents, direction);
+		return;
+	}
+
 	/* nothing to do */
 }
 
@@ -168,6 +193,11 @@ pci_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direc
 static inline void
 pci_dma_sync_single(struct pci_dev *hwdev, dma_addr_t dma_handle, size_t size, int direction)
 {
+	if (dev_is_sa1111(hwdev)) {
+	  	sa1111_dma_sync_single(hwdev, dma_handle, size, direction);
+		return;
+	}
+
 	consistent_sync(bus_to_virt(dma_handle), size, direction);
 }
 
@@ -182,6 +212,11 @@ pci_dma_sync_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nelems, int d
 {
 	int i;
 
+	if (dev_is_sa1111(hwdev)) {
+	  	sa1111_dma_sync_sg(hwdev, sg, nelems, direction);
+		return;
+	}
+
 	for (i = 0; i < nelems; i++, sg++) {
 		char *virt = page_address(sg->page) + sg->offset;
 		consistent_sync(virt, sg->length, direction);
@@ -204,6 +239,19 @@ static inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask)
 /* Return the index of the PCI controller for device PDEV. */
 #define pci_controller_num(PDEV)	(0)
 
+
+#if defined(CONFIG_SA1111) && !defined(CONFIG_PCI)
+/* SA-1111 needs these prototypes even when !defined(CONFIG_PCI) */
+
+/* kmem_cache style wrapper around pci_alloc_consistent() */
+struct pci_pool *pci_pool_create (const char *name, struct pci_dev *dev,
+		size_t size, size_t align, size_t allocation, int flags);
+void pci_pool_destroy (struct pci_pool *pool);
+
+void *pci_pool_alloc (struct pci_pool *pool, int flags, dma_addr_t *handle);
+void pci_pool_free (struct pci_pool *pool, void *vaddr, dma_addr_t addr);
+#endif
+
 #endif /* __KERNEL__ */
  
 #endif