Commit 281dd25c authored by Yasunori Goto's avatar Yasunori Goto Committed by Linus Torvalds

[PATCH] swiotlb: make sure initial DMA allocations really are in DMA memory

This introduces a limit parameter to the core bootmem allocator; The new
parameter indicates that physical memory allocated by the bootmem
allocator should be within the requested limit.

We also introduce alloc_bootmem_low_pages_limit, alloc_bootmem_node_limit,
alloc_bootmem_low_pages_node_limit apis, but alloc_bootmem_low_pages_limit
is the only api used for swiotlb.

The existing alloc_bootmem_low_pages() api could instead have been
changed and made to pass right limit to the core allocator.  But that
would make the patch more intrusive for 2.6.14, as other arches use
alloc_bootmem_low_pages().  We may be done that post 2.6.14 as a
cleanup.

With this, swiotlb gets memory within 4G for both x86_64 and ia64
arches.
Signed-off-by: default avatarYasunori Goto <y-goto@jp.fujitsu.com>
Cc: Ravikiran G Thirumalai <kiran@scalex86.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 51b190b3
...@@ -123,8 +123,8 @@ swiotlb_init_with_default_size (size_t default_size) ...@@ -123,8 +123,8 @@ swiotlb_init_with_default_size (size_t default_size)
/* /*
* Get IO TLB memory from the low pages * Get IO TLB memory from the low pages
*/ */
io_tlb_start = alloc_bootmem_low_pages(io_tlb_nslabs * io_tlb_start = alloc_bootmem_low_pages_limit(io_tlb_nslabs *
(1 << IO_TLB_SHIFT)); (1 << IO_TLB_SHIFT), 0x100000000);
if (!io_tlb_start) if (!io_tlb_start)
panic("Cannot allocate SWIOTLB buffer"); panic("Cannot allocate SWIOTLB buffer");
io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT); io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT);
......
...@@ -43,7 +43,7 @@ typedef struct bootmem_data { ...@@ -43,7 +43,7 @@ typedef struct bootmem_data {
extern unsigned long __init bootmem_bootmap_pages (unsigned long); extern unsigned long __init bootmem_bootmap_pages (unsigned long);
extern unsigned long __init init_bootmem (unsigned long addr, unsigned long memend); extern unsigned long __init init_bootmem (unsigned long addr, unsigned long memend);
extern void __init free_bootmem (unsigned long addr, unsigned long size); extern void __init free_bootmem (unsigned long addr, unsigned long size);
extern void * __init __alloc_bootmem (unsigned long size, unsigned long align, unsigned long goal); extern void * __init __alloc_bootmem_limit (unsigned long size, unsigned long align, unsigned long goal, unsigned long limit);
#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
extern void __init reserve_bootmem (unsigned long addr, unsigned long size); extern void __init reserve_bootmem (unsigned long addr, unsigned long size);
#define alloc_bootmem(x) \ #define alloc_bootmem(x) \
...@@ -54,6 +54,16 @@ extern void __init reserve_bootmem (unsigned long addr, unsigned long size); ...@@ -54,6 +54,16 @@ extern void __init reserve_bootmem (unsigned long addr, unsigned long size);
__alloc_bootmem((x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) __alloc_bootmem((x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS))
#define alloc_bootmem_low_pages(x) \ #define alloc_bootmem_low_pages(x) \
__alloc_bootmem((x), PAGE_SIZE, 0) __alloc_bootmem((x), PAGE_SIZE, 0)
#define alloc_bootmem_limit(x, limit) \
__alloc_bootmem_limit((x), SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS), (limit))
#define alloc_bootmem_low_limit(x, limit) \
__alloc_bootmem_limit((x), SMP_CACHE_BYTES, 0, (limit))
#define alloc_bootmem_pages_limit(x, limit) \
__alloc_bootmem_limit((x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS), (limit))
#define alloc_bootmem_low_pages_limit(x, limit) \
__alloc_bootmem_limit((x), PAGE_SIZE, 0, (limit))
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
extern unsigned long __init free_all_bootmem (void); extern unsigned long __init free_all_bootmem (void);
...@@ -61,7 +71,7 @@ extern unsigned long __init init_bootmem_node (pg_data_t *pgdat, unsigned long f ...@@ -61,7 +71,7 @@ extern unsigned long __init init_bootmem_node (pg_data_t *pgdat, unsigned long f
extern void __init reserve_bootmem_node (pg_data_t *pgdat, unsigned long physaddr, unsigned long size); extern void __init reserve_bootmem_node (pg_data_t *pgdat, unsigned long physaddr, unsigned long size);
extern void __init free_bootmem_node (pg_data_t *pgdat, unsigned long addr, unsigned long size); extern void __init free_bootmem_node (pg_data_t *pgdat, unsigned long addr, unsigned long size);
extern unsigned long __init free_all_bootmem_node (pg_data_t *pgdat); extern unsigned long __init free_all_bootmem_node (pg_data_t *pgdat);
extern void * __init __alloc_bootmem_node (pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal); extern void * __init __alloc_bootmem_node_limit (pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal, unsigned long limit);
#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
#define alloc_bootmem_node(pgdat, x) \ #define alloc_bootmem_node(pgdat, x) \
__alloc_bootmem_node((pgdat), (x), SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS)) __alloc_bootmem_node((pgdat), (x), SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS))
...@@ -69,6 +79,14 @@ extern void * __init __alloc_bootmem_node (pg_data_t *pgdat, unsigned long size, ...@@ -69,6 +79,14 @@ extern void * __init __alloc_bootmem_node (pg_data_t *pgdat, unsigned long size,
__alloc_bootmem_node((pgdat), (x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) __alloc_bootmem_node((pgdat), (x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS))
#define alloc_bootmem_low_pages_node(pgdat, x) \ #define alloc_bootmem_low_pages_node(pgdat, x) \
__alloc_bootmem_node((pgdat), (x), PAGE_SIZE, 0) __alloc_bootmem_node((pgdat), (x), PAGE_SIZE, 0)
#define alloc_bootmem_node_limit(pgdat, x, limit) \
__alloc_bootmem_node_limit((pgdat), (x), SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS), (limit))
#define alloc_bootmem_pages_node_limit(pgdat, x, limit) \
__alloc_bootmem_node_limit((pgdat), (x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS), (limit))
#define alloc_bootmem_low_pages_node_limit(pgdat, x, limit) \
__alloc_bootmem_node_limit((pgdat), (x), PAGE_SIZE, 0, (limit))
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
#ifdef CONFIG_HAVE_ARCH_ALLOC_REMAP #ifdef CONFIG_HAVE_ARCH_ALLOC_REMAP
...@@ -105,5 +123,15 @@ extern void *__init alloc_large_system_hash(const char *tablename, ...@@ -105,5 +123,15 @@ extern void *__init alloc_large_system_hash(const char *tablename,
#endif #endif
extern int __initdata hashdist; /* Distribute hashes across NUMA nodes? */ extern int __initdata hashdist; /* Distribute hashes across NUMA nodes? */
static inline void *__alloc_bootmem (unsigned long size, unsigned long align, unsigned long goal)
{
return __alloc_bootmem_limit(size, align, goal, 0);
}
static inline void *__alloc_bootmem_node (pg_data_t *pgdat, unsigned long size, unsigned long align,
unsigned long goal)
{
return __alloc_bootmem_node_limit(pgdat, size, align, goal, 0);
}
#endif /* _LINUX_BOOTMEM_H */ #endif /* _LINUX_BOOTMEM_H */
...@@ -154,10 +154,10 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, ...@@ -154,10 +154,10 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
*/ */
static void * __init static void * __init
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
unsigned long align, unsigned long goal) unsigned long align, unsigned long goal, unsigned long limit)
{ {
unsigned long offset, remaining_size, areasize, preferred; unsigned long offset, remaining_size, areasize, preferred;
unsigned long i, start = 0, incr, eidx; unsigned long i, start = 0, incr, eidx, end_pfn = bdata->node_low_pfn;
void *ret; void *ret;
if(!size) { if(!size) {
...@@ -166,7 +166,14 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, ...@@ -166,7 +166,14 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
} }
BUG_ON(align & (align-1)); BUG_ON(align & (align-1));
eidx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT); if (limit && bdata->node_boot_start >= limit)
return NULL;
limit >>=PAGE_SHIFT;
if (limit && end_pfn > limit)
end_pfn = limit;
eidx = end_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
offset = 0; offset = 0;
if (align && if (align &&
(bdata->node_boot_start & (align - 1UL)) != 0) (bdata->node_boot_start & (align - 1UL)) != 0)
...@@ -178,11 +185,12 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, ...@@ -178,11 +185,12 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
* first, then we try to allocate lower pages. * first, then we try to allocate lower pages.
*/ */
if (goal && (goal >= bdata->node_boot_start) && if (goal && (goal >= bdata->node_boot_start) &&
((goal >> PAGE_SHIFT) < bdata->node_low_pfn)) { ((goal >> PAGE_SHIFT) < end_pfn)) {
preferred = goal - bdata->node_boot_start; preferred = goal - bdata->node_boot_start;
if (bdata->last_success >= preferred) if (bdata->last_success >= preferred)
preferred = bdata->last_success; if (!limit || (limit && limit > bdata->last_success))
preferred = bdata->last_success;
} else } else
preferred = 0; preferred = 0;
...@@ -382,14 +390,15 @@ unsigned long __init free_all_bootmem (void) ...@@ -382,14 +390,15 @@ unsigned long __init free_all_bootmem (void)
return(free_all_bootmem_core(NODE_DATA(0))); return(free_all_bootmem_core(NODE_DATA(0)));
} }
void * __init __alloc_bootmem (unsigned long size, unsigned long align, unsigned long goal) void * __init __alloc_bootmem_limit (unsigned long size, unsigned long align, unsigned long goal,
unsigned long limit)
{ {
pg_data_t *pgdat = pgdat_list; pg_data_t *pgdat = pgdat_list;
void *ptr; void *ptr;
for_each_pgdat(pgdat) for_each_pgdat(pgdat)
if ((ptr = __alloc_bootmem_core(pgdat->bdata, size, if ((ptr = __alloc_bootmem_core(pgdat->bdata, size,
align, goal))) align, goal, limit)))
return(ptr); return(ptr);
/* /*
...@@ -400,14 +409,16 @@ void * __init __alloc_bootmem (unsigned long size, unsigned long align, unsigned ...@@ -400,14 +409,16 @@ void * __init __alloc_bootmem (unsigned long size, unsigned long align, unsigned
return NULL; return NULL;
} }
void * __init __alloc_bootmem_node (pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal)
void * __init __alloc_bootmem_node_limit (pg_data_t *pgdat, unsigned long size, unsigned long align,
unsigned long goal, unsigned long limit)
{ {
void *ptr; void *ptr;
ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal); ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, limit);
if (ptr) if (ptr)
return (ptr); return (ptr);
return __alloc_bootmem(size, align, goal); return __alloc_bootmem_limit(size, align, goal, limit);
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment