Commit 793e6612 authored by Matthew Auld's avatar Matthew Auld Committed by Rodrigo Vivi

drm/xe/buddy: add visible tracking

Replace the allocation code with the i915 version. This simplifies the
code a little, and importantly we get the accounting at the mgr level,
which is useful for debug (and maybe userspace), plus per resource
tracking so we can easily check if a resource is using one or pages in
the mappable part of vram (useful for eviction), or if the resource is
completely within the mappable portion (useful for checking if the
resource can be safely CPU mapped).

v2: Fix missing PAGE_SHIFT
v3: (Gwan-gyeong Mun)
  - Fix incorrect usage of ilog2(mm.chunk_size).
  - Fix calculation when checking for impossible allocation sizes, also
    check much earlier.
v4: (Gwan-gyeong Mun)
  - Fix calculation when extending the [fpfn, lpfn] range due to the
    roundup_pow_of_two().
v5: (Gwan-gyeong Mun)
  - Move the check for running out of mappable VRAM to before doing any of
    the roundup_pow_of_two().
v6: (Jani)
  - Stop abusing BUG_ON(). We can easily just use WARN_ON() here and
    return a proper error to the caller, which is much nicer if we ever
    trigger these.
Signed-off-by: default avatarMatthew Auld <matthew.auld@intel.com>
Cc: Gwan-gyeong Mun <gwan-gyeong.mun@intel.com>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: Lucas De Marchi <lucas.demarchi@intel.com>
Cc: Jani Nikula <jani.nikula@linux.intel.com>
Reviewed-by: default avatarGwan-gyeong Mun <gwan-gyeong.mun@intel.com>
Signed-off-by: default avatarRodrigo Vivi <rodrigo.vivi@intel.com>
parent 11a2407e
...@@ -135,7 +135,7 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe) ...@@ -135,7 +135,7 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe)
{ {
struct xe_ttm_stolen_mgr *mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL); struct xe_ttm_stolen_mgr *mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL);
struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
u64 stolen_size, pgsize; u64 stolen_size, io_size, pgsize;
int err; int err;
if (IS_DGFX(xe)) if (IS_DGFX(xe))
...@@ -154,7 +154,17 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe) ...@@ -154,7 +154,17 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe)
if (pgsize < PAGE_SIZE) if (pgsize < PAGE_SIZE)
pgsize = PAGE_SIZE; pgsize = PAGE_SIZE;
err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size, pgsize); /*
* We don't try to attempt partial visible support for stolen vram,
* since stolen is always at the end of vram, and the BAR size is pretty
* much always 256M, with small-bar.
*/
io_size = 0;
if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe))
io_size = stolen_size;
err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size,
io_size, pgsize);
if (err) { if (err) {
drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err); drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err);
return; return;
...@@ -163,8 +173,8 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe) ...@@ -163,8 +173,8 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe)
drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n", drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n",
stolen_size); stolen_size);
if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe)) if (io_size)
mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, stolen_size); mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, io_size);
} }
u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset) u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset)
......
...@@ -49,45 +49,29 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, ...@@ -49,45 +49,29 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
const struct ttm_place *place, const struct ttm_place *place,
struct ttm_resource **res) struct ttm_resource **res)
{ {
u64 max_bytes, cur_size, min_block_size;
struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
struct xe_ttm_vram_mgr_resource *vres; struct xe_ttm_vram_mgr_resource *vres;
u64 size, remaining_size, lpfn, fpfn;
struct drm_buddy *mm = &mgr->mm; struct drm_buddy *mm = &mgr->mm;
unsigned long pages_per_block; u64 size, remaining_size, min_page_size;
int r; unsigned long lpfn;
int err;
lpfn = (u64)place->lpfn << PAGE_SHIFT;
if (!lpfn || lpfn > man->size)
lpfn = man->size;
fpfn = (u64)place->fpfn << PAGE_SHIFT; lpfn = place->lpfn;
if (!lpfn || lpfn > man->size >> PAGE_SHIFT)
lpfn = man->size >> PAGE_SHIFT;
max_bytes = mgr->manager.size; if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn))
if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { return -E2BIG; /* don't trigger eviction for the impossible */
pages_per_block = ~0ul;
} else {
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pages_per_block = HPAGE_PMD_NR;
#else
/* default to 2MB */
pages_per_block = 2UL << (20UL - PAGE_SHIFT);
#endif
pages_per_block = max_t(uint32_t, pages_per_block,
tbo->page_alignment);
}
vres = kzalloc(sizeof(*vres), GFP_KERNEL); vres = kzalloc(sizeof(*vres), GFP_KERNEL);
if (!vres) if (!vres)
return -ENOMEM; return -ENOMEM;
ttm_resource_init(tbo, place, &vres->base); ttm_resource_init(tbo, place, &vres->base);
remaining_size = vres->base.size;
/* bail out quickly if there's likely not enough VRAM for this BO */ /* bail out quickly if there's likely not enough VRAM for this BO */
if (ttm_resource_manager_usage(man) > max_bytes) { if (ttm_resource_manager_usage(man) > man->size) {
r = -ENOSPC; err = -ENOSPC;
goto error_fini; goto error_fini;
} }
...@@ -96,95 +80,100 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, ...@@ -96,95 +80,100 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
if (place->flags & TTM_PL_FLAG_TOPDOWN) if (place->flags & TTM_PL_FLAG_TOPDOWN)
vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
if (fpfn || lpfn != man->size) if (place->fpfn || lpfn != man->size >> PAGE_SHIFT)
/* Allocate blocks in desired range */
vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
mutex_lock(&mgr->lock); if (WARN_ON(!vres->base.size)) {
while (remaining_size) { err = -EINVAL;
if (tbo->page_alignment) goto error_fini;
min_block_size = tbo->page_alignment << PAGE_SHIFT; }
else size = vres->base.size;
min_block_size = mgr->default_page_size;
XE_BUG_ON(min_block_size < mm->chunk_size);
/* Limit maximum size to 2GiB due to SG table limitations */
size = min(remaining_size, 2ULL << 30);
if (size >= pages_per_block << PAGE_SHIFT)
min_block_size = pages_per_block << PAGE_SHIFT;
cur_size = size;
if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT) {
/*
* Except for actual range allocation, modify the size and
* min_block_size conforming to continuous flag enablement
*/
if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
size = roundup_pow_of_two(size);
min_block_size = size;
/*
* Modify the size value if size is not
* aligned with min_block_size
*/
} else if (!IS_ALIGNED(size, min_block_size)) {
size = round_up(size, min_block_size);
}
}
r = drm_buddy_alloc_blocks(mm, fpfn, min_page_size = mgr->default_page_size;
lpfn, if (tbo->page_alignment)
size, min_page_size = tbo->page_alignment << PAGE_SHIFT;
min_block_size,
&vres->blocks, if (WARN_ON(min_page_size < mm->chunk_size)) {
vres->flags); err = -EINVAL;
if (unlikely(r)) goto error_fini;
goto error_free_blocks; }
if (size > remaining_size) if (WARN_ON(min_page_size > SZ_2G)) { /* FIXME: sg limit */
remaining_size = 0; err = -EINVAL;
else goto error_fini;
remaining_size -= size;
} }
mutex_unlock(&mgr->lock);
if (cur_size != size) { if (WARN_ON((size > SZ_2G &&
struct drm_buddy_block *block; (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS)))) {
struct list_head *trim_list; err = -EINVAL;
u64 original_size; goto error_fini;
LIST_HEAD(temp); }
trim_list = &vres->blocks; if (WARN_ON(!IS_ALIGNED(size, min_page_size))) {
original_size = vres->base.size; err = -EINVAL;
goto error_fini;
}
mutex_lock(&mgr->lock);
if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) {
mutex_unlock(&mgr->lock);
err = -ENOSPC;
goto error_fini;
}
if (place->fpfn + (size >> PAGE_SHIFT) != place->lpfn &&
place->flags & TTM_PL_FLAG_CONTIGUOUS) {
size = roundup_pow_of_two(size);
min_page_size = size;
lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn);
}
remaining_size = size;
do {
/* /*
* If size value is rounded up to min_block_size, trim the last * Limit maximum size to 2GiB due to SG table limitations.
* block to the required size * FIXME: Should maybe be handled as part of sg construction.
*/ */
if (!list_is_singular(&vres->blocks)) { u64 alloc_size = min_t(u64, remaining_size, SZ_2G);
block = list_last_entry(&vres->blocks, typeof(*block), link);
list_move_tail(&block->link, &temp); err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
trim_list = &temp; (u64)lpfn << PAGE_SHIFT,
/* alloc_size,
* Compute the original_size value by subtracting the min_page_size,
* last block size with (aligned size - original size) &vres->blocks,
*/ vres->flags);
original_size = drm_buddy_block_size(mm, block) - if (err)
(size - cur_size); goto error_free_blocks;
}
mutex_lock(&mgr->lock); remaining_size -= alloc_size;
drm_buddy_block_trim(mm, } while (remaining_size);
original_size,
trim_list);
mutex_unlock(&mgr->lock);
if (!list_empty(&temp)) if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
list_splice_tail(trim_list, &vres->blocks); if (!drm_buddy_block_trim(mm, vres->base.size, &vres->blocks))
size = vres->base.size;
} }
if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {
vres->used_visible_size = size;
} else {
struct drm_buddy_block *block;
list_for_each_entry(block, &vres->blocks, link) {
u64 start = drm_buddy_block_offset(block);
if (start < mgr->visible_size) {
u64 end = start + drm_buddy_block_size(mm, block);
vres->used_visible_size +=
min(end, mgr->visible_size) - start;
}
}
}
mgr->visible_avail -= vres->used_visible_size;
mutex_unlock(&mgr->lock);
if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) &&
xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks)) xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks))
vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
...@@ -213,7 +202,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, ...@@ -213,7 +202,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
ttm_resource_fini(man, &vres->base); ttm_resource_fini(man, &vres->base);
kfree(vres); kfree(vres);
return r; return err;
} }
static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
...@@ -226,6 +215,7 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, ...@@ -226,6 +215,7 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
mutex_lock(&mgr->lock); mutex_lock(&mgr->lock);
drm_buddy_free_list(mm, &vres->blocks); drm_buddy_free_list(mm, &vres->blocks);
mgr->visible_avail += vres->used_visible_size;
mutex_unlock(&mgr->lock); mutex_unlock(&mgr->lock);
ttm_resource_fini(man, res); ttm_resource_fini(man, res);
...@@ -240,6 +230,13 @@ static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, ...@@ -240,6 +230,13 @@ static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,
struct drm_buddy *mm = &mgr->mm; struct drm_buddy *mm = &mgr->mm;
mutex_lock(&mgr->lock); mutex_lock(&mgr->lock);
drm_printf(printer, "default_page_size: %lluKiB\n",
mgr->default_page_size >> 10);
drm_printf(printer, "visible_avail: %lluMiB\n",
(u64)mgr->visible_avail >> 20);
drm_printf(printer, "visible_size: %lluMiB\n",
(u64)mgr->visible_size >> 20);
drm_buddy_print(mm, printer); drm_buddy_print(mm, printer);
mutex_unlock(&mgr->lock); mutex_unlock(&mgr->lock);
drm_printf(printer, "man size:%llu\n", man->size); drm_printf(printer, "man size:%llu\n", man->size);
...@@ -262,6 +259,8 @@ static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg) ...@@ -262,6 +259,8 @@ static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
if (ttm_resource_manager_evict_all(&xe->ttm, man)) if (ttm_resource_manager_evict_all(&xe->ttm, man))
return; return;
WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size);
drm_buddy_fini(&mgr->mm); drm_buddy_fini(&mgr->mm);
ttm_resource_manager_cleanup(&mgr->manager); ttm_resource_manager_cleanup(&mgr->manager);
...@@ -270,7 +269,8 @@ static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg) ...@@ -270,7 +269,8 @@ static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
} }
int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
u32 mem_type, u64 size, u64 default_page_size) u32 mem_type, u64 size, u64 io_size,
u64 default_page_size)
{ {
struct ttm_resource_manager *man = &mgr->manager; struct ttm_resource_manager *man = &mgr->manager;
int err; int err;
...@@ -279,6 +279,8 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, ...@@ -279,6 +279,8 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
mgr->mem_type = mem_type; mgr->mem_type = mem_type;
mutex_init(&mgr->lock); mutex_init(&mgr->lock);
mgr->default_page_size = default_page_size; mgr->default_page_size = default_page_size;
mgr->visible_size = io_size;
mgr->visible_avail = io_size;
ttm_resource_manager_init(man, &xe->ttm, size); ttm_resource_manager_init(man, &xe->ttm, size);
err = drm_buddy_init(&mgr->mm, man->size, default_page_size); err = drm_buddy_init(&mgr->mm, man->size, default_page_size);
...@@ -298,7 +300,8 @@ int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr) ...@@ -298,7 +300,8 @@ int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr)
mgr->gt = gt; mgr->gt = gt;
return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + gt->info.vram_id, return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + gt->info.vram_id,
gt->mem.vram.size, PAGE_SIZE); gt->mem.vram.size, gt->mem.vram.io_size,
PAGE_SIZE);
} }
int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
......
...@@ -13,7 +13,8 @@ struct xe_device; ...@@ -13,7 +13,8 @@ struct xe_device;
struct xe_gt; struct xe_gt;
int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
u32 mem_type, u64 size, u64 default_page_size); u32 mem_type, u64 size, u64 io_size,
u64 default_page_size);
int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr); int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr);
int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
struct ttm_resource *res, struct ttm_resource *res,
......
...@@ -23,6 +23,10 @@ struct xe_ttm_vram_mgr { ...@@ -23,6 +23,10 @@ struct xe_ttm_vram_mgr {
struct ttm_resource_manager manager; struct ttm_resource_manager manager;
/** @mm: DRM buddy allocator which manages the VRAM */ /** @mm: DRM buddy allocator which manages the VRAM */
struct drm_buddy mm; struct drm_buddy mm;
/** @visible_size: Proped size of the CPU visible portion */
u64 visible_size;
/** @visible_avail: CPU visible portion still unallocated */
u64 visible_avail;
/** @default_page_size: default page size */ /** @default_page_size: default page size */
u64 default_page_size; u64 default_page_size;
/** @lock: protects allocations of VRAM */ /** @lock: protects allocations of VRAM */
...@@ -39,6 +43,8 @@ struct xe_ttm_vram_mgr_resource { ...@@ -39,6 +43,8 @@ struct xe_ttm_vram_mgr_resource {
struct ttm_resource base; struct ttm_resource base;
/** @blocks: list of DRM buddy blocks */ /** @blocks: list of DRM buddy blocks */
struct list_head blocks; struct list_head blocks;
/** @used_visible_size: How many CPU visible bytes this resource is using */
u64 used_visible_size;
/** @flags: flags associated with the resource */ /** @flags: flags associated with the resource */
unsigned long flags; unsigned long flags;
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment