Commit 9476df7d authored by Dan Williams's avatar Dan Williams Committed by Linus Torvalds

mm: introduce find_dev_pagemap()

There are several scenarios where we need to retrieve and update
metadata associated with a given devm_memremap_pages() mapping, and the
only lookup key available is a pfn in the range:

1/ We want to augment vmemmap_populate() (called via arch_add_memory())
   to allocate memmap storage from pre-allocated pages reserved by the
   device driver.  At vmemmap_alloc_block_buf() time it grabs device pages
   rather than page allocator pages.  This is in support of
   devm_memremap_pages() mappings where the memmap is too large to fit in
   main memory (i.e. large persistent memory devices).

2/ Taking a reference against the mapping when inserting device pages
   into the address_space radix of a given inode.  This facilitates
   unmap_mapping_range() and truncate_inode_pages() operations when the
   driver is tearing down the mapping.

3/ get_user_pages() operations on ZONE_DEVICE memory require taking a
   reference against the mapping so that the driver teardown path can
   revoke and drain usage of device pages.
Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
Tested-by: default avatarLogan Gunthorpe <logang@deltatee.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 260ae3f7
...@@ -21,9 +21,9 @@ ...@@ -21,9 +21,9 @@
#include <linux/init.h> #include <linux/init.h>
#include <linux/platform_device.h> #include <linux/platform_device.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/memory_hotplug.h>
#include <linux/moduleparam.h> #include <linux/moduleparam.h>
#include <linux/badblocks.h> #include <linux/badblocks.h>
#include <linux/memremap.h>
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <linux/pfn_t.h> #include <linux/pfn_t.h>
#include <linux/slab.h> #include <linux/slab.h>
......
...@@ -89,21 +89,6 @@ void devm_memunmap(struct device *dev, void *addr); ...@@ -89,21 +89,6 @@ void devm_memunmap(struct device *dev, void *addr);
void *__devm_memremap_pages(struct device *dev, struct resource *res); void *__devm_memremap_pages(struct device *dev, struct resource *res);
#ifdef CONFIG_ZONE_DEVICE
void *devm_memremap_pages(struct device *dev, struct resource *res);
#else
static inline void *devm_memremap_pages(struct device *dev, struct resource *res)
{
/*
* Fail attempts to call devm_memremap_pages() without
* ZONE_DEVICE support enabled, this requires callers to fall
* back to plain devm_memremap() based on config
*/
WARN_ON_ONCE(1);
return ERR_PTR(-ENXIO);
}
#endif
/* /*
* Some systems do not have legacy ISA devices. * Some systems do not have legacy ISA devices.
* /dev/port is not a valid interface on these systems. * /dev/port is not a valid interface on these systems.
......
#ifndef _LINUX_MEMREMAP_H_
#define _LINUX_MEMREMAP_H_
#include <linux/mm.h>
struct resource;
struct device;
/**
* struct dev_pagemap - metadata for ZONE_DEVICE mappings
* @dev: host device of the mapping for debug
*/
struct dev_pagemap {
/* TODO: vmem_altmap and percpu_ref count */
struct device *dev;
};
#ifdef CONFIG_ZONE_DEVICE
void *devm_memremap_pages(struct device *dev, struct resource *res);
struct dev_pagemap *find_dev_pagemap(resource_size_t phys);
#else
static inline void *devm_memremap_pages(struct device *dev,
struct resource *res)
{
/*
* Fail attempts to call devm_memremap_pages() without
* ZONE_DEVICE support enabled, this requires callers to fall
* back to plain devm_memremap() based on config
*/
WARN_ON_ONCE(1);
return ERR_PTR(-ENXIO);
}
static inline struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
{
return NULL;
}
#endif
#endif /* _LINUX_MEMREMAP_H_ */
...@@ -10,6 +10,8 @@ ...@@ -10,6 +10,8 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details. * General Public License for more details.
*/ */
#include <linux/radix-tree.h>
#include <linux/memremap.h>
#include <linux/device.h> #include <linux/device.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/pfn_t.h> #include <linux/pfn_t.h>
...@@ -155,22 +157,57 @@ pfn_t phys_to_pfn_t(dma_addr_t addr, unsigned long flags) ...@@ -155,22 +157,57 @@ pfn_t phys_to_pfn_t(dma_addr_t addr, unsigned long flags)
EXPORT_SYMBOL(phys_to_pfn_t); EXPORT_SYMBOL(phys_to_pfn_t);
#ifdef CONFIG_ZONE_DEVICE #ifdef CONFIG_ZONE_DEVICE
static DEFINE_MUTEX(pgmap_lock);
static RADIX_TREE(pgmap_radix, GFP_KERNEL);
#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
struct page_map { struct page_map {
struct resource res; struct resource res;
struct percpu_ref *ref;
struct dev_pagemap pgmap;
}; };
static void devm_memremap_pages_release(struct device *dev, void *res) static void pgmap_radix_release(struct resource *res)
{
resource_size_t key;
mutex_lock(&pgmap_lock);
for (key = res->start; key <= res->end; key += SECTION_SIZE)
radix_tree_delete(&pgmap_radix, key >> PA_SECTION_SHIFT);
mutex_unlock(&pgmap_lock);
}
static void devm_memremap_pages_release(struct device *dev, void *data)
{ {
struct page_map *page_map = res; struct page_map *page_map = data;
struct resource *res = &page_map->res;
resource_size_t align_start, align_size;
pgmap_radix_release(res);
/* pages are dead and unused, undo the arch mapping */ /* pages are dead and unused, undo the arch mapping */
arch_remove_memory(page_map->res.start, resource_size(&page_map->res)); align_start = res->start & ~(SECTION_SIZE - 1);
align_size = ALIGN(resource_size(res), SECTION_SIZE);
arch_remove_memory(align_start, align_size);
}
/* assumes rcu_read_lock() held at entry */
struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
{
struct page_map *page_map;
WARN_ON_ONCE(!rcu_read_lock_held());
page_map = radix_tree_lookup(&pgmap_radix, phys >> PA_SECTION_SHIFT);
return page_map ? &page_map->pgmap : NULL;
} }
void *devm_memremap_pages(struct device *dev, struct resource *res) void *devm_memremap_pages(struct device *dev, struct resource *res)
{ {
int is_ram = region_intersects(res->start, resource_size(res), int is_ram = region_intersects(res->start, resource_size(res),
"System RAM"); "System RAM");
resource_size_t key, align_start, align_size;
struct page_map *page_map; struct page_map *page_map;
int error, nid; int error, nid;
...@@ -190,18 +227,50 @@ void *devm_memremap_pages(struct device *dev, struct resource *res) ...@@ -190,18 +227,50 @@ void *devm_memremap_pages(struct device *dev, struct resource *res)
memcpy(&page_map->res, res, sizeof(*res)); memcpy(&page_map->res, res, sizeof(*res));
page_map->pgmap.dev = dev;
mutex_lock(&pgmap_lock);
error = 0;
for (key = res->start; key <= res->end; key += SECTION_SIZE) {
struct dev_pagemap *dup;
rcu_read_lock();
dup = find_dev_pagemap(key);
rcu_read_unlock();
if (dup) {
dev_err(dev, "%s: %pr collides with mapping for %s\n",
__func__, res, dev_name(dup->dev));
error = -EBUSY;
break;
}
error = radix_tree_insert(&pgmap_radix, key >> PA_SECTION_SHIFT,
page_map);
if (error) {
dev_err(dev, "%s: failed: %d\n", __func__, error);
break;
}
}
mutex_unlock(&pgmap_lock);
if (error)
goto err_radix;
nid = dev_to_node(dev); nid = dev_to_node(dev);
if (nid < 0) if (nid < 0)
nid = numa_mem_id(); nid = numa_mem_id();
error = arch_add_memory(nid, res->start, resource_size(res), true); align_start = res->start & ~(SECTION_SIZE - 1);
if (error) { align_size = ALIGN(resource_size(res), SECTION_SIZE);
devres_free(page_map); error = arch_add_memory(nid, align_start, align_size, true);
return ERR_PTR(error); if (error)
} goto err_add_memory;
devres_add(dev, page_map); devres_add(dev, page_map);
return __va(res->start); return __va(res->start);
err_add_memory:
err_radix:
pgmap_radix_release(res);
devres_free(page_map);
return ERR_PTR(error);
} }
EXPORT_SYMBOL(devm_memremap_pages); EXPORT_SYMBOL(devm_memremap_pages);
#endif /* CONFIG_ZONE_DEVICE */ #endif /* CONFIG_ZONE_DEVICE */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment