Commit 481ebd0d authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

bootmem: fix aligning of node-relative indexes and offsets

Absolute alignment requirements may never be applied to node-relative
offsets.  Andreas Herrmann spotted this flaw when a bootmem allocation on
an unaligned node was itself not aligned because the combination of an
unaligned node with an aligned offset into that node is not garuanteed to
be aligned itself.

This patch introduces two helper functions that align a node-relative
index or offset with respect to the node's starting address so that the
absolute PFN or virtual address that results from combining the two
satisfies the requested alignment.

Then all the broken ALIGN()s in alloc_bootmem_core() are replaced by these
helpers.
Signed-off-by: default avatarJohannes Weiner <hannes@saeurebad.de>
Reported-by: default avatarAndreas Herrmann <andreas.herrmann3@amd.com>
Debugged-by: default avatarAndreas Herrmann <andreas.herrmann3@amd.com>
Reviewed-by: default avatarAndreas Herrmann <andreas.herrmann3@amd.com>
Tested-by: default avatarAndreas Herrmann <andreas.herrmann3@amd.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a09f4855
...@@ -405,6 +405,29 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size, ...@@ -405,6 +405,29 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size,
} }
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx,
unsigned long step)
{
unsigned long base = bdata->node_min_pfn;
/*
* Align the index with respect to the node start so that the
* combination of both satisfies the requested alignment.
*/
return ALIGN(base + idx, step) - base;
}
static unsigned long align_off(struct bootmem_data *bdata, unsigned long off,
unsigned long align)
{
unsigned long base = PFN_PHYS(bdata->node_min_pfn);
/* Same as align_idx for byte offsets */
return ALIGN(base + off, align) - base;
}
static void * __init alloc_bootmem_core(struct bootmem_data *bdata, static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
unsigned long size, unsigned long align, unsigned long size, unsigned long align,
unsigned long goal, unsigned long limit) unsigned long goal, unsigned long limit)
...@@ -441,7 +464,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -441,7 +464,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
else else
start = ALIGN(min, step); start = ALIGN(min, step);
sidx = start - bdata->node_min_pfn;; sidx = start - bdata->node_min_pfn;
midx = max - bdata->node_min_pfn; midx = max - bdata->node_min_pfn;
if (bdata->hint_idx > sidx) { if (bdata->hint_idx > sidx) {
...@@ -450,7 +473,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -450,7 +473,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
* catch the fallback below. * catch the fallback below.
*/ */
fallback = sidx + 1; fallback = sidx + 1;
sidx = ALIGN(bdata->hint_idx, step); sidx = align_idx(bdata, bdata->hint_idx, step);
} }
while (1) { while (1) {
...@@ -459,7 +482,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -459,7 +482,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
unsigned long eidx, i, start_off, end_off; unsigned long eidx, i, start_off, end_off;
find_block: find_block:
sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx); sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
sidx = ALIGN(sidx, step); sidx = align_idx(bdata, sidx, step);
eidx = sidx + PFN_UP(size); eidx = sidx + PFN_UP(size);
if (sidx >= midx || eidx > midx) if (sidx >= midx || eidx > midx)
...@@ -467,7 +490,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -467,7 +490,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
for (i = sidx; i < eidx; i++) for (i = sidx; i < eidx; i++)
if (test_bit(i, bdata->node_bootmem_map)) { if (test_bit(i, bdata->node_bootmem_map)) {
sidx = ALIGN(i, step); sidx = align_idx(bdata, i, step);
if (sidx == i) if (sidx == i)
sidx += step; sidx += step;
goto find_block; goto find_block;
...@@ -475,7 +498,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -475,7 +498,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
if (bdata->last_end_off & (PAGE_SIZE - 1) && if (bdata->last_end_off & (PAGE_SIZE - 1) &&
PFN_DOWN(bdata->last_end_off) + 1 == sidx) PFN_DOWN(bdata->last_end_off) + 1 == sidx)
start_off = ALIGN(bdata->last_end_off, align); start_off = align_off(bdata, bdata->last_end_off, align);
else else
start_off = PFN_PHYS(sidx); start_off = PFN_PHYS(sidx);
...@@ -499,7 +522,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -499,7 +522,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
} }
if (fallback) { if (fallback) {
sidx = ALIGN(fallback - 1, step); sidx = align_idx(bdata, fallback - 1, step);
fallback = 0; fallback = 0;
goto find_block; goto find_block;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment