Commit f56f821f authored by Daniel Vetter's avatar Daniel Vetter

mm: extend prefault helpers to fault in more than PAGE_SIZE

drm/i915 wants to read/write more than one page in its fastpath
and hence needs to prefault more than PAGE_SIZE bytes.

Add new functions in filemap.h to make that possible.

Also kill a copy&pasted spurious space in both functions while at it.

v2: As suggested by Andrew Morton, add a multipage parameter to both
functions to avoid the additional branch for the pagemap.c hotpath.
My gcc 4.6 here seems to dtrt and indeed reap these branches where not
needed.

v3: Becaus I couldn't find a way around adding a uaddr += PAGE_SIZE to
the filemap.c hotpaths (that the compiler couldn't remove again),
let's go with separate new functions for the multipage use-case.

v4: Adjust comment to CodingStlye and fix spelling.
Acked-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarDaniel Vetter <daniel.vetter@ffwll.ch>
parent d174bd64
...@@ -416,7 +416,7 @@ i915_gem_shmem_pread(struct drm_device *dev, ...@@ -416,7 +416,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
mutex_unlock(&dev->struct_mutex); mutex_unlock(&dev->struct_mutex);
if (!prefaulted) { if (!prefaulted) {
ret = fault_in_pages_writeable(user_data, remain); ret = fault_in_multipages_writeable(user_data, remain);
/* Userspace is tricking us, but we've already clobbered /* Userspace is tricking us, but we've already clobbered
* its pages with the prefault and promised to write the * its pages with the prefault and promised to write the
* data up to the first fault. Hence ignore any errors * data up to the first fault. Hence ignore any errors
...@@ -809,8 +809,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, ...@@ -809,8 +809,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
args->size)) args->size))
return -EFAULT; return -EFAULT;
ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr, ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
args->size); args->size);
if (ret) if (ret)
return -EFAULT; return -EFAULT;
......
...@@ -997,7 +997,7 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec, ...@@ -997,7 +997,7 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
if (!access_ok(VERIFY_WRITE, ptr, length)) if (!access_ok(VERIFY_WRITE, ptr, length))
return -EFAULT; return -EFAULT;
if (fault_in_pages_readable(ptr, length)) if (fault_in_multipages_readable(ptr, length))
return -EFAULT; return -EFAULT;
} }
......
...@@ -426,7 +426,7 @@ static inline int fault_in_pages_writeable(char __user *uaddr, int size) ...@@ -426,7 +426,7 @@ static inline int fault_in_pages_writeable(char __user *uaddr, int size)
*/ */
if (((unsigned long)uaddr & PAGE_MASK) != if (((unsigned long)uaddr & PAGE_MASK) !=
((unsigned long)end & PAGE_MASK)) ((unsigned long)end & PAGE_MASK))
ret = __put_user(0, end); ret = __put_user(0, end);
} }
return ret; return ret;
} }
...@@ -445,13 +445,73 @@ static inline int fault_in_pages_readable(const char __user *uaddr, int size) ...@@ -445,13 +445,73 @@ static inline int fault_in_pages_readable(const char __user *uaddr, int size)
if (((unsigned long)uaddr & PAGE_MASK) != if (((unsigned long)uaddr & PAGE_MASK) !=
((unsigned long)end & PAGE_MASK)) { ((unsigned long)end & PAGE_MASK)) {
ret = __get_user(c, end); ret = __get_user(c, end);
(void)c; (void)c;
} }
} }
return ret; return ret;
} }
/*
* Multipage variants of the above prefault helpers, useful if more than
* PAGE_SIZE of data needs to be prefaulted. These are separate from the above
* functions (which only handle up to PAGE_SIZE) to avoid clobbering the
* filemap.c hotpaths.
*/
static inline int fault_in_multipages_writeable(char __user *uaddr, int size)
{
int ret;
const char __user *end = uaddr + size - 1;
if (unlikely(size == 0))
return 0;
/*
* Writing zeroes into userspace here is OK, because we know that if
* the zero gets there, we'll be overwriting it.
*/
while (uaddr <= end) {
ret = __put_user(0, uaddr);
if (ret != 0)
return ret;
uaddr += PAGE_SIZE;
}
/* Check whether the range spilled into the next page. */
if (((unsigned long)uaddr & PAGE_MASK) ==
((unsigned long)end & PAGE_MASK))
ret = __put_user(0, end);
return ret;
}
static inline int fault_in_multipages_readable(const char __user *uaddr,
int size)
{
volatile char c;
int ret;
const char __user *end = uaddr + size - 1;
if (unlikely(size == 0))
return 0;
while (uaddr <= end) {
ret = __get_user(c, uaddr);
if (ret != 0)
return ret;
uaddr += PAGE_SIZE;
}
/* Check whether the range spilled into the next page. */
if (((unsigned long)uaddr & PAGE_MASK) ==
((unsigned long)end & PAGE_MASK)) {
ret = __get_user(c, end);
(void)c;
}
return ret;
}
int add_to_page_cache_locked(struct page *page, struct address_space *mapping, int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask); pgoff_t index, gfp_t gfp_mask);
int add_to_page_cache_lru(struct page *page, struct address_space *mapping, int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment