Commit a5ff307f authored by Ben Skeggs's avatar Ben Skeggs

drm/nouveau/mmu: add a privileged method to directly manage PTEs

This provides a somewhat more direct method of manipulating the GPU page
tables, which will be required to support SVM.
Signed-off-by: default avatarBen Skeggs <bskeggs@redhat.com>
parent 8e68271d
......@@ -15,6 +15,8 @@ struct nvif_vmm_v0 {
#define NVIF_VMM_V0_PUT 0x02
#define NVIF_VMM_V0_MAP 0x03
#define NVIF_VMM_V0_UNMAP 0x04
#define NVIF_VMM_V0_PFNMAP 0x05
#define NVIF_VMM_V0_PFNCLR 0x06
struct nvif_vmm_page_v0 {
__u8 version;
......@@ -62,4 +64,28 @@ struct nvif_vmm_unmap_v0 {
__u8 pad01[7];
__u64 addr;
};
struct nvif_vmm_pfnmap_v0 {
__u8 version;
__u8 page;
__u8 pad02[6];
__u64 addr;
__u64 size;
#define NVIF_VMM_PFNMAP_V0_ADDR 0xfffffffffffff000ULL
#define NVIF_VMM_PFNMAP_V0_ADDR_SHIFT 12
#define NVIF_VMM_PFNMAP_V0_APER 0x00000000000000f0ULL
#define NVIF_VMM_PFNMAP_V0_HOST 0x0000000000000000ULL
#define NVIF_VMM_PFNMAP_V0_VRAM 0x0000000000000010ULL
#define NVIF_VMM_PFNMAP_V0_W 0x0000000000000002ULL
#define NVIF_VMM_PFNMAP_V0_V 0x0000000000000001ULL
#define NVIF_VMM_PFNMAP_V0_NONE 0x0000000000000000ULL
__u64 phys[];
};
struct nvif_vmm_pfnclr_v0 {
__u8 version;
__u8 pad01[7];
__u64 addr;
__u64 size;
};
#endif
......@@ -64,6 +64,7 @@ struct nvkm_vmm_map {
struct nvkm_mm_node *mem;
struct scatterlist *sgl;
dma_addr_t *dma;
u64 *pfn;
u64 off;
const struct nvkm_vmm_page *page;
......
......@@ -42,6 +42,69 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle)
return nvkm_uvmm(object)->vmm;
}
static int
nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
{
struct nvkm_client *client = uvmm->object.client;
union {
struct nvif_vmm_pfnclr_v0 v0;
} *args = argv;
struct nvkm_vmm *vmm = uvmm->vmm;
int ret = -ENOSYS;
u64 addr, size;
if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
addr = args->v0.addr;
size = args->v0.size;
} else
return ret;
if (!client->super)
return -ENOENT;
if (size) {
mutex_lock(&vmm->mutex);
ret = nvkm_vmm_pfn_unmap(vmm, addr, size);
mutex_unlock(&vmm->mutex);
}
return ret;
}
static int
nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
{
struct nvkm_client *client = uvmm->object.client;
union {
struct nvif_vmm_pfnmap_v0 v0;
} *args = argv;
struct nvkm_vmm *vmm = uvmm->vmm;
int ret = -ENOSYS;
u64 addr, size, *phys;
u8 page;
if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) {
page = args->v0.page;
addr = args->v0.addr;
size = args->v0.size;
phys = args->v0.phys;
if (argc != (size >> page) * sizeof(args->v0.phys[0]))
return -EINVAL;
} else
return ret;
if (!client->super)
return -ENOENT;
if (size) {
mutex_lock(&vmm->mutex);
ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys);
mutex_unlock(&vmm->mutex);
}
return ret;
}
static int
nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
{
......@@ -78,7 +141,7 @@ nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
goto done;
}
nvkm_vmm_unmap_locked(vmm, vma);
nvkm_vmm_unmap_locked(vmm, vma, false);
ret = 0;
done:
mutex_unlock(&vmm->mutex);
......@@ -124,6 +187,11 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
goto fail;
}
if (ret = -EINVAL, vma->mapped && !vma->memory) {
VMM_DEBUG(vmm, "pfnmap %016llx", addr);
goto fail;
}
if (ret = -EINVAL, vma->addr != addr || vma->size != size) {
if (addr + size > vma->addr + vma->size || vma->memory ||
(vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) {
......@@ -271,6 +339,8 @@ nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc)
case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc);
case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc);
case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc);
case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc);
case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc);
default:
break;
}
......
......@@ -255,11 +255,23 @@ nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
}
static bool
nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
{
const struct nvkm_vmm_desc *desc = it->desc;
const int type = desc->type == SPT;
struct nvkm_vmm_pt *pgt = it->pt[0];
bool dma;
if (pfn) {
/* Need to clear PTE valid bits before we dma_unmap_page(). */
dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
if (dma) {
/* GPU may have cached the PT, flush before unmap. */
nvkm_vmm_flush_mark(it);
nvkm_vmm_flush(it);
desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
}
}
/* Drop PTE references. */
pgt->refs[type] -= ptes;
......@@ -349,7 +361,7 @@ nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
}
static bool
nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
{
const struct nvkm_vmm_desc *desc = it->desc;
const int type = desc->type == SPT;
......@@ -379,7 +391,7 @@ nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
}
static bool
nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
{
struct nvkm_vmm_pt *pt = it->pt[0];
if (it->desc->type == PGD)
......@@ -387,14 +399,14 @@ nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
else
if (it->desc->type == LPT)
memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
return nvkm_vmm_unref_ptes(it, ptei, ptes);
return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
}
static bool
nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
{
nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
return nvkm_vmm_ref_ptes(it, ptei, ptes);
return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
}
static bool
......@@ -487,8 +499,8 @@ nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
static inline u64
nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size, const char *name, bool ref,
bool (*REF_PTES)(struct nvkm_vmm_iter *, u32, u32),
u64 addr, u64 size, const char *name, bool ref, bool pfn,
bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
nvkm_vmm_pxe_func CLR_PTES)
{
......@@ -548,7 +560,7 @@ nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
}
/* Handle PTE updates. */
if (!REF_PTES || REF_PTES(&it, ptei, ptes)) {
if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
struct nvkm_mmu_pt *pt = pgt->pt[type];
if (MAP_PTES || CLR_PTES) {
if (MAP_PTES)
......@@ -590,7 +602,7 @@ static void
nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size)
{
nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false,
nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
nvkm_vmm_sparse_unref_ptes, NULL, NULL,
page->desc->func->invalid ?
page->desc->func->invalid : page->desc->func->unmap);
......@@ -602,8 +614,8 @@ nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
{
if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
true, nvkm_vmm_sparse_ref_ptes, NULL,
NULL, page->desc->func->sparse);
true, false, nvkm_vmm_sparse_ref_ptes,
NULL, NULL, page->desc->func->sparse);
if (fail != ~0ULL) {
if ((size = fail - addr))
nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
......@@ -666,11 +678,11 @@ nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
static void
nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size, bool sparse)
u64 addr, u64 size, bool sparse, bool pfn)
{
const struct nvkm_vmm_desc_func *func = page->desc->func;
nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
false, nvkm_vmm_unref_ptes, NULL, NULL,
false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
sparse ? func->sparse : func->invalid ? func->invalid :
func->unmap);
}
......@@ -681,10 +693,10 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
nvkm_vmm_pte_func func)
{
u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
nvkm_vmm_ref_ptes, func, map, NULL);
false, nvkm_vmm_ref_ptes, func, map, NULL);
if (fail != ~0ULL) {
if ((size = fail - addr))
nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false);
nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
return -ENOMEM;
}
return 0;
......@@ -692,10 +704,11 @@ nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
static void
nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size, bool sparse)
u64 addr, u64 size, bool sparse, bool pfn)
{
const struct nvkm_vmm_desc_func *func = page->desc->func;
nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, NULL, NULL, NULL,
nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
NULL, NULL, NULL,
sparse ? func->sparse : func->invalid ? func->invalid :
func->unmap);
}
......@@ -705,7 +718,7 @@ nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size, struct nvkm_vmm_map *map,
nvkm_vmm_pte_func func)
{
nvkm_vmm_iter(vmm, page, addr, size, "map", false,
nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
NULL, func, map, NULL);
}
......@@ -713,7 +726,7 @@ static void
nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size)
{
nvkm_vmm_iter(vmm, page, addr, size, "unref", false,
nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
nvkm_vmm_unref_ptes, NULL, NULL, NULL);
}
......@@ -721,7 +734,7 @@ static int
nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
u64 addr, u64 size)
{
u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true,
u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
nvkm_vmm_ref_ptes, NULL, NULL, NULL);
if (fail != ~0ULL) {
if (fail != addr)
......@@ -935,12 +948,41 @@ nvkm_vmm_node_split(struct nvkm_vmm *vmm,
return vma;
}
static void
nvkm_vma_dump(struct nvkm_vma *vma)
{
printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n",
vma->addr, (u64)vma->size,
vma->used ? '-' : 'F',
vma->mapref ? 'R' : '-',
vma->sparse ? 'S' : '-',
vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
vma->part ? 'P' : '-',
vma->user ? 'U' : '-',
vma->busy ? 'B' : '-',
vma->mapped ? 'M' : '-',
vma->memory);
}
static void
nvkm_vmm_dump(struct nvkm_vmm *vmm)
{
struct nvkm_vma *vma;
list_for_each_entry(vma, &vmm->list, head) {
nvkm_vma_dump(vma);
}
}
static void
nvkm_vmm_dtor(struct nvkm_vmm *vmm)
{
struct nvkm_vma *vma;
struct rb_node *node;
if (0)
nvkm_vmm_dump(vmm);
while ((node = rb_first(&vmm->root))) {
struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
nvkm_vmm_put(vmm, &vma);
......@@ -1105,33 +1147,216 @@ nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
}
static struct nvkm_vma *
nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
u64 addr, u64 size, u8 page, bool map)
{
struct nvkm_vma *prev = NULL;
struct nvkm_vma *next = NULL;
if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
if (prev->memory || prev->mapped != map)
prev = NULL;
}
if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
if (!next->part ||
next->memory || next->mapped != map)
next = NULL;
}
if (prev || next)
return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
return nvkm_vmm_node_split(vmm, vma, addr, size);
}
int
nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
{
struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
struct nvkm_vma *next;
u64 limit = addr + size;
u64 start = addr;
if (!vma)
return -EINVAL;
do {
if (!vma->mapped || vma->memory)
continue;
size = min(limit - start, vma->size - (start - vma->addr));
nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
start, size, false, true);
next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
if (!WARN_ON(!next)) {
vma = next;
vma->refd = NVKM_VMA_PAGE_NONE;
vma->mapped = false;
}
} while ((vma = node(vma, next)) && (start = vma->addr) < limit);
return 0;
}
/*TODO:
* - Avoid PT readback (for dma_unmap etc), this might end up being dealt
* with inside HMM, which would be a lot nicer for us to deal with.
* - Multiple page sizes (particularly for huge page support).
* - Support for systems without a 4KiB page size.
*/
int
nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
{
const struct nvkm_vmm_page *page = vmm->func->page;
struct nvkm_vma *vma, *tmp;
u64 limit = addr + size;
u64 start = addr;
int pm = size >> shift;
int pi = 0;
/* Only support mapping where the page size of the incoming page
* array matches a page size available for direct mapping.
*/
while (page->shift && page->shift != shift &&
page->desc->func->pfn == NULL)
page++;
if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
!IS_ALIGNED(size, 1ULL << shift) ||
addr + size < addr || addr + size > vmm->limit) {
VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
shift, page->shift, addr, size);
return -EINVAL;
}
if (!(vma = nvkm_vmm_node_search(vmm, addr)))
return -ENOENT;
do {
bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
bool mapped = vma->mapped;
u64 size = limit - start;
u64 addr = start;
int pn, ret = 0;
/* Narrow the operation window to cover a single action (page
* should be mapped or not) within a single VMA.
*/
for (pn = 0; pi + pn < pm; pn++) {
if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
break;
}
size = min_t(u64, size, pn << page->shift);
size = min_t(u64, size, vma->size + vma->addr - addr);
/* Reject any operation to unmanaged regions, and areas that
* have nvkm_memory objects mapped in them already.
*/
if (!vma->mapref || vma->memory) {
ret = -EINVAL;
goto next;
}
/* In order to both properly refcount GPU page tables, and
* prevent "normal" mappings and these direct mappings from
* interfering with each other, we need to track contiguous
* ranges that have been mapped with this interface.
*
* Here we attempt to either split an existing VMA so we're
* able to flag the region as either unmapped/mapped, or to
* merge with adjacent VMAs that are already compatible.
*
* If the region is already compatible, nothing is required.
*/
if (map != mapped) {
tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
page -
vmm->func->page, map);
if (WARN_ON(!tmp)) {
ret = -ENOMEM;
goto next;
}
if ((tmp->mapped = map))
tmp->refd = page - vmm->func->page;
else
tmp->refd = NVKM_VMA_PAGE_NONE;
vma = tmp;
}
/* Update HW page tables. */
if (map) {
struct nvkm_vmm_map args;
args.page = page;
args.pfn = &pfn[pi];
if (!mapped) {
ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
size, &args, page->
desc->func->pfn);
} else {
nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
page->desc->func->pfn);
}
} else {
if (mapped) {
nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
false, true);
}
}
next:
/* Iterate to next operation. */
if (vma->addr + vma->size == addr + size)
vma = node(vma, next);
start += size;
if (ret) {
/* Failure is signalled by clearing the valid bit on
* any PFN that couldn't be modified as requested.
*/
while (size) {
pfn[pi++] = NVKM_VMM_PFN_NONE;
size -= 1 << page->shift;
}
} else {
pi += size >> page->shift;
}
} while (vma && start < limit);
return 0;
}
void
nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
{
struct nvkm_vma *next = node(vma, next);
struct nvkm_vma *prev = NULL;
struct nvkm_vma *next;
nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
nvkm_memory_unref(&vma->memory);
vma->mapped = false;
if (!vma->part || ((prev = node(vma, prev)), prev->mapped))
if (vma->part && (prev = node(vma, prev)) && prev->mapped)
prev = NULL;
if (!next->part || next->mapped)
if ((next = node(vma, next)) && (!next->part || next->mapped))
next = NULL;
nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
}
void
nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
{
const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
if (vma->mapref) {
nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse);
nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
vma->refd = NVKM_VMA_PAGE_NONE;
} else {
nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse);
nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
}
nvkm_vmm_unmap_region(vmm, vma);
......@@ -1142,7 +1367,7 @@ nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
{
if (vma->memory) {
mutex_lock(&vmm->mutex);
nvkm_vmm_unmap_locked(vmm, vma);
nvkm_vmm_unmap_locked(vmm, vma, false);
mutex_unlock(&vmm->mutex);
}
}
......@@ -1341,7 +1566,8 @@ nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
* the page tree.
*/
nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
size, vma->sparse);
size, vma->sparse,
!mem);
} else
if (refd != NVKM_VMA_PAGE_NONE) {
/* Drop allocation-time PTE references. */
......@@ -1577,7 +1803,7 @@ nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
}
static bool
nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, u32 ptei, u32 ptes)
nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
{
const struct nvkm_vmm_desc *desc = it->desc;
const int type = desc->type == SPT;
......@@ -1599,7 +1825,7 @@ nvkm_vmm_boot(struct nvkm_vmm *vmm)
if (ret)
return ret;
nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false,
nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
nvkm_vmm_boot_ptes, NULL, NULL, NULL);
vmm->bootstrapped = true;
return 0;
......
......@@ -67,6 +67,10 @@ struct nvkm_vmm_desc_func {
nvkm_vmm_pte_func mem;
nvkm_vmm_pte_func dma;
nvkm_vmm_pte_func sgl;
nvkm_vmm_pte_func pfn;
bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
nvkm_vmm_pxe_func pfn_unmap;
};
extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
......@@ -166,8 +170,20 @@ int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
bool sparse, u8 page, u8 align, u64 size,
struct nvkm_vma **pvma);
void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn);
void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *);
#define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL
#define NVKM_VMM_PFN_ADDR_SHIFT 12
#define NVKM_VMM_PFN_APER 0x00000000000000f0ULL
#define NVKM_VMM_PFN_HOST 0x0000000000000000ULL
#define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL
#define NVKM_VMM_PFN_W 0x0000000000000002ULL
#define NVKM_VMM_PFN_V 0x0000000000000001ULL
#define NVKM_VMM_PFN_NONE 0x0000000000000000ULL
int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn);
int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size);
struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
......
......@@ -27,6 +27,81 @@
#include <nvif/ifc00d.h>
#include <nvif/unpack.h>
static void
gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm,
struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
{
struct device *dev = vmm->mmu->subdev.device->dev;
dma_addr_t addr;
nvkm_kmap(pt->memory);
while (ptes--) {
u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
u64 data = (u64)datahi << 32 | datalo;
if ((data & (3ULL << 1)) != 0) {
addr = (data >> 8) << 12;
dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
}
ptei++;
}
nvkm_done(pt->memory);
}
static bool
gp100_vmm_pfn_clear(struct nvkm_vmm *vmm,
struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
{
bool dma = false;
nvkm_kmap(pt->memory);
while (ptes--) {
u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
u64 data = (u64)datahi << 32 | datalo;
if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0));
dma = true;
}
ptei++;
}
nvkm_done(pt->memory);
return dma;
}
static void
gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
{
struct device *dev = vmm->mmu->subdev.device->dev;
dma_addr_t addr;
nvkm_kmap(pt->memory);
while (ptes--) {
u64 data = 0;
if (!(*map->pfn & NVKM_VMM_PFN_W))
data |= BIT_ULL(6); /* RO. */
if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
addr = dma_map_page(dev, pfn_to_page(addr), 0,
PAGE_SIZE, DMA_BIDIRECTIONAL);
if (!WARN_ON(dma_mapping_error(dev, addr))) {
data |= addr >> 4;
data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
data |= BIT_ULL(3); /* VOL. */
data |= BIT_ULL(0); /* VALID. */
}
} else {
data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
data |= BIT_ULL(0); /* VALID. */
}
VMM_WO064(pt, vmm, ptei++ * 8, data);
map->pfn++;
}
nvkm_done(pt->memory);
}
static inline void
gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
......@@ -89,6 +164,9 @@ gp100_vmm_desc_spt = {
.mem = gp100_vmm_pgt_mem,
.dma = gp100_vmm_pgt_dma,
.sgl = gp100_vmm_pgt_sgl,
.pfn = gp100_vmm_pgt_pfn,
.pfn_clear = gp100_vmm_pfn_clear,
.pfn_unmap = gp100_vmm_pfn_unmap,
};
static void
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment