Commit 27df704d authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'akpm' (patches from Andrew)

Merge misc fixes from Andrew Morton:
 "21 fixes"

* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (21 commits)
  userfaultfd: replace ENOSPC with ESRCH in case mm has gone during copy/zeropage
  zram: rework copy of compressor name in comp_algorithm_store()
  rmap: do not call mmu_notifier_invalidate_page() under ptl
  mm: fix list corruptions on shmem shrinklist
  mm/balloon_compaction.c: don't zero ballooned pages
  MAINTAINERS: copy virtio on balloon_compaction.c
  mm: fix KSM data corruption
  mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem
  mm: make tlb_flush_pending global
  mm: refactor TLB gathering API
  Revert "mm: numa: defer TLB flush for THP migration as long as possible"
  mm: migrate: fix barriers around tlb_flush_pending
  mm: migrate: prevent racy access to tlb_flush_pending
  fault-inject: fix wrong should_fail() decision in task context
  test_kmod: fix small memory leak on filesystem tests
  test_kmod: fix the lock in register_test_dev_kmod()
  test_kmod: fix bug which allows negative values on two config options
  test_kmod: fix spelling mistake: "EMTPY" -> "EMPTY"
  userfaultfd: hugetlbfs: remove superfluous page unlock in VM_SHARED case
  mm: ratelimit PFNs busy info message
  ...
parents 4e082e9b e86b298b
...@@ -14004,6 +14004,7 @@ F: drivers/block/virtio_blk.c ...@@ -14004,6 +14004,7 @@ F: drivers/block/virtio_blk.c
F: include/linux/virtio*.h F: include/linux/virtio*.h
F: include/uapi/linux/virtio_*.h F: include/uapi/linux/virtio_*.h
F: drivers/crypto/virtio/ F: drivers/crypto/virtio/
F: mm/balloon_compaction.c
VIRTIO CRYPTO DRIVER VIRTIO CRYPTO DRIVER
M: Gonglei <arei.gonglei@huawei.com> M: Gonglei <arei.gonglei@huawei.com>
......
...@@ -148,7 +148,8 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) ...@@ -148,7 +148,8 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb)
} }
static inline void static inline void
tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long end)
{ {
tlb->mm = mm; tlb->mm = mm;
tlb->fullmm = !(start | (end+1)); tlb->fullmm = !(start | (end+1));
...@@ -166,8 +167,14 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start ...@@ -166,8 +167,14 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start
} }
static inline void static inline void
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end, bool force)
{ {
if (force) {
tlb->range_start = start;
tlb->range_end = end;
}
tlb_flush_mmu(tlb); tlb_flush_mmu(tlb);
/* keep the page table cache within bounds */ /* keep the page table cache within bounds */
......
...@@ -168,7 +168,8 @@ static inline void __tlb_alloc_page(struct mmu_gather *tlb) ...@@ -168,7 +168,8 @@ static inline void __tlb_alloc_page(struct mmu_gather *tlb)
static inline void static inline void
tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long end)
{ {
tlb->mm = mm; tlb->mm = mm;
tlb->max = ARRAY_SIZE(tlb->local); tlb->max = ARRAY_SIZE(tlb->local);
...@@ -185,8 +186,11 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start ...@@ -185,8 +186,11 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start
* collected. * collected.
*/ */
static inline void static inline void
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end, bool force)
{ {
if (force)
tlb->need_flush = 1;
/* /*
* Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and
* tlb->end_addr. * tlb->end_addr.
......
...@@ -47,10 +47,9 @@ struct mmu_table_batch { ...@@ -47,10 +47,9 @@ struct mmu_table_batch {
extern void tlb_table_flush(struct mmu_gather *tlb); extern void tlb_table_flush(struct mmu_gather *tlb);
extern void tlb_remove_table(struct mmu_gather *tlb, void *table); extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
static inline void tlb_gather_mmu(struct mmu_gather *tlb, static inline void
struct mm_struct *mm, arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long start, unsigned long end)
unsigned long end)
{ {
tlb->mm = mm; tlb->mm = mm;
tlb->start = start; tlb->start = start;
...@@ -76,9 +75,15 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) ...@@ -76,9 +75,15 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb)
tlb_flush_mmu_free(tlb); tlb_flush_mmu_free(tlb);
} }
static inline void tlb_finish_mmu(struct mmu_gather *tlb, static inline void
unsigned long start, unsigned long end) arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end, bool force)
{ {
if (force) {
tlb->start = start;
tlb->end = end;
}
tlb_flush_mmu(tlb); tlb_flush_mmu(tlb);
} }
......
...@@ -36,7 +36,8 @@ static inline void init_tlb_gather(struct mmu_gather *tlb) ...@@ -36,7 +36,8 @@ static inline void init_tlb_gather(struct mmu_gather *tlb)
} }
static inline void static inline void
tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long end)
{ {
tlb->mm = mm; tlb->mm = mm;
tlb->start = start; tlb->start = start;
...@@ -47,9 +48,10 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start ...@@ -47,9 +48,10 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start
} }
static inline void static inline void
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end, bool force)
{ {
if (tlb->fullmm) if (tlb->fullmm || force)
flush_tlb_mm(tlb->mm); flush_tlb_mm(tlb->mm);
/* keep the page table cache within bounds */ /* keep the page table cache within bounds */
......
...@@ -45,7 +45,8 @@ static inline void init_tlb_gather(struct mmu_gather *tlb) ...@@ -45,7 +45,8 @@ static inline void init_tlb_gather(struct mmu_gather *tlb)
} }
static inline void static inline void
tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long end)
{ {
tlb->mm = mm; tlb->mm = mm;
tlb->start = start; tlb->start = start;
...@@ -80,13 +81,19 @@ tlb_flush_mmu(struct mmu_gather *tlb) ...@@ -80,13 +81,19 @@ tlb_flush_mmu(struct mmu_gather *tlb)
tlb_flush_mmu_free(tlb); tlb_flush_mmu_free(tlb);
} }
/* tlb_finish_mmu /* arch_tlb_finish_mmu
* Called at the end of the shootdown operation to free up any resources * Called at the end of the shootdown operation to free up any resources
* that were required. * that were required.
*/ */
static inline void static inline void
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end, bool force)
{ {
if (force) {
tlb->start = start;
tlb->end = end;
tlb->need_flush = 1;
}
tlb_flush_mmu(tlb); tlb_flush_mmu(tlb);
/* keep the page table cache within bounds */ /* keep the page table cache within bounds */
......
...@@ -308,7 +308,7 @@ static ssize_t comp_algorithm_store(struct device *dev, ...@@ -308,7 +308,7 @@ static ssize_t comp_algorithm_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len) struct device_attribute *attr, const char *buf, size_t len)
{ {
struct zram *zram = dev_to_zram(dev); struct zram *zram = dev_to_zram(dev);
char compressor[CRYPTO_MAX_ALG_NAME]; char compressor[ARRAY_SIZE(zram->compressor)];
size_t sz; size_t sz;
strlcpy(compressor, buf, sizeof(compressor)); strlcpy(compressor, buf, sizeof(compressor));
...@@ -327,7 +327,7 @@ static ssize_t comp_algorithm_store(struct device *dev, ...@@ -327,7 +327,7 @@ static ssize_t comp_algorithm_store(struct device *dev,
return -EBUSY; return -EBUSY;
} }
strlcpy(zram->compressor, compressor, sizeof(compressor)); strcpy(zram->compressor, compressor);
up_write(&zram->init_lock); up_write(&zram->init_lock);
return len; return len;
} }
......
...@@ -106,13 +106,13 @@ static int meminfo_proc_show(struct seq_file *m, void *v) ...@@ -106,13 +106,13 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
global_node_page_state(NR_FILE_MAPPED)); global_node_page_state(NR_FILE_MAPPED));
show_val_kb(m, "Shmem: ", i.sharedram); show_val_kb(m, "Shmem: ", i.sharedram);
show_val_kb(m, "Slab: ", show_val_kb(m, "Slab: ",
global_page_state(NR_SLAB_RECLAIMABLE) + global_node_page_state(NR_SLAB_RECLAIMABLE) +
global_page_state(NR_SLAB_UNRECLAIMABLE)); global_node_page_state(NR_SLAB_UNRECLAIMABLE));
show_val_kb(m, "SReclaimable: ", show_val_kb(m, "SReclaimable: ",
global_page_state(NR_SLAB_RECLAIMABLE)); global_node_page_state(NR_SLAB_RECLAIMABLE));
show_val_kb(m, "SUnreclaim: ", show_val_kb(m, "SUnreclaim: ",
global_page_state(NR_SLAB_UNRECLAIMABLE)); global_node_page_state(NR_SLAB_UNRECLAIMABLE));
seq_printf(m, "KernelStack: %8lu kB\n", seq_printf(m, "KernelStack: %8lu kB\n",
global_page_state(NR_KERNEL_STACK_KB)); global_page_state(NR_KERNEL_STACK_KB));
show_val_kb(m, "PageTables: ", show_val_kb(m, "PageTables: ",
......
...@@ -16,9 +16,10 @@ ...@@ -16,9 +16,10 @@
#include <linux/mmu_notifier.h> #include <linux/mmu_notifier.h>
#include <linux/page_idle.h> #include <linux/page_idle.h>
#include <linux/shmem_fs.h> #include <linux/shmem_fs.h>
#include <linux/uaccess.h>
#include <asm/elf.h> #include <asm/elf.h>
#include <linux/uaccess.h> #include <asm/tlb.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include "internal.h" #include "internal.h"
...@@ -1008,6 +1009,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, ...@@ -1008,6 +1009,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
struct mm_struct *mm; struct mm_struct *mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
enum clear_refs_types type; enum clear_refs_types type;
struct mmu_gather tlb;
int itype; int itype;
int rv; int rv;
...@@ -1054,6 +1056,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, ...@@ -1054,6 +1056,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
} }
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
tlb_gather_mmu(&tlb, mm, 0, -1);
if (type == CLEAR_REFS_SOFT_DIRTY) { if (type == CLEAR_REFS_SOFT_DIRTY) {
for (vma = mm->mmap; vma; vma = vma->vm_next) { for (vma = mm->mmap; vma; vma = vma->vm_next) {
if (!(vma->vm_flags & VM_SOFTDIRTY)) if (!(vma->vm_flags & VM_SOFTDIRTY))
...@@ -1075,7 +1078,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, ...@@ -1075,7 +1078,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
walk_page_range(0, mm->highest_vm_end, &clear_refs_walk); walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
if (type == CLEAR_REFS_SOFT_DIRTY) if (type == CLEAR_REFS_SOFT_DIRTY)
mmu_notifier_invalidate_range_end(mm, 0, -1); mmu_notifier_invalidate_range_end(mm, 0, -1);
flush_tlb_mm(mm); tlb_finish_mmu(&tlb, 0, -1);
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
out_mm: out_mm:
mmput(mm); mmput(mm);
......
...@@ -1600,7 +1600,7 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, ...@@ -1600,7 +1600,7 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
uffdio_copy.len); uffdio_copy.len);
mmput(ctx->mm); mmput(ctx->mm);
} else { } else {
return -ENOSPC; return -ESRCH;
} }
if (unlikely(put_user(ret, &user_uffdio_copy->copy))) if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
return -EFAULT; return -EFAULT;
...@@ -1647,7 +1647,7 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, ...@@ -1647,7 +1647,7 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
uffdio_zeropage.range.len); uffdio_zeropage.range.len);
mmput(ctx->mm); mmput(ctx->mm);
} else { } else {
return -ENOSPC; return -ESRCH;
} }
if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
return -EFAULT; return -EFAULT;
......
...@@ -112,10 +112,11 @@ struct mmu_gather { ...@@ -112,10 +112,11 @@ struct mmu_gather {
#define HAVE_GENERIC_MMU_GATHER #define HAVE_GENERIC_MMU_GATHER
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end); void arch_tlb_gather_mmu(struct mmu_gather *tlb,
struct mm_struct *mm, unsigned long start, unsigned long end);
void tlb_flush_mmu(struct mmu_gather *tlb); void tlb_flush_mmu(struct mmu_gather *tlb);
void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, void arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long end); unsigned long start, unsigned long end, bool force);
extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
int page_size); int page_size);
......
...@@ -487,14 +487,12 @@ struct mm_struct { ...@@ -487,14 +487,12 @@ struct mm_struct {
/* numa_scan_seq prevents two threads setting pte_numa */ /* numa_scan_seq prevents two threads setting pte_numa */
int numa_scan_seq; int numa_scan_seq;
#endif #endif
#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION)
/* /*
* An operation with batched TLB flushing is going on. Anything that * An operation with batched TLB flushing is going on. Anything that
* can move process memory needs to flush the TLB when moving a * can move process memory needs to flush the TLB when moving a
* PROT_NONE or PROT_NUMA mapped page. * PROT_NONE or PROT_NUMA mapped page.
*/ */
bool tlb_flush_pending; atomic_t tlb_flush_pending;
#endif
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
/* See flush_tlb_batched_pending() */ /* See flush_tlb_batched_pending() */
bool tlb_flush_batched; bool tlb_flush_batched;
...@@ -522,46 +520,60 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) ...@@ -522,46 +520,60 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
return mm->cpu_vm_mask_var; return mm->cpu_vm_mask_var;
} }
#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long end);
extern void tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end);
/* /*
* Memory barriers to keep this state in sync are graciously provided by * Memory barriers to keep this state in sync are graciously provided by
* the page table locks, outside of which no page table modifications happen. * the page table locks, outside of which no page table modifications happen.
* The barriers below prevent the compiler from re-ordering the instructions * The barriers are used to ensure the order between tlb_flush_pending updates,
* around the memory barriers that are already present in the code. * which happen while the lock is not taken, and the PTE updates, which happen
* while the lock is taken, are serialized.
*/ */
static inline bool mm_tlb_flush_pending(struct mm_struct *mm) static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
{ {
barrier(); return atomic_read(&mm->tlb_flush_pending) > 0;
return mm->tlb_flush_pending; }
/*
* Returns true if there are two above TLB batching threads in parallel.
*/
static inline bool mm_tlb_flush_nested(struct mm_struct *mm)
{
return atomic_read(&mm->tlb_flush_pending) > 1;
}
static inline void init_tlb_flush_pending(struct mm_struct *mm)
{
atomic_set(&mm->tlb_flush_pending, 0);
} }
static inline void set_tlb_flush_pending(struct mm_struct *mm)
static inline void inc_tlb_flush_pending(struct mm_struct *mm)
{ {
mm->tlb_flush_pending = true; atomic_inc(&mm->tlb_flush_pending);
/* /*
* Guarantee that the tlb_flush_pending store does not leak into the * Guarantee that the tlb_flush_pending increase does not leak into the
* critical section updating the page tables * critical section updating the page tables
*/ */
smp_mb__before_spinlock(); smp_mb__before_spinlock();
} }
/* Clearing is done after a TLB flush, which also provides a barrier. */ /* Clearing is done after a TLB flush, which also provides a barrier. */
static inline void clear_tlb_flush_pending(struct mm_struct *mm) static inline void dec_tlb_flush_pending(struct mm_struct *mm)
{
barrier();
mm->tlb_flush_pending = false;
}
#else
static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
{
return false;
}
static inline void set_tlb_flush_pending(struct mm_struct *mm)
{
}
static inline void clear_tlb_flush_pending(struct mm_struct *mm)
{ {
/*
* Guarantee that the tlb_flush_pending does not not leak into the
* critical section, since we must order the PTE change and changes to
* the pending TLB flush indication. We could have relied on TLB flush
* as a memory barrier, but this behavior is not clearly documented.
*/
smp_mb__before_atomic();
atomic_dec(&mm->tlb_flush_pending);
} }
#endif
struct vm_fault; struct vm_fault;
......
...@@ -807,7 +807,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, ...@@ -807,7 +807,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
mm_init_aio(mm); mm_init_aio(mm);
mm_init_owner(mm, p); mm_init_owner(mm, p);
mmu_notifier_mm_init(mm); mmu_notifier_mm_init(mm);
clear_tlb_flush_pending(mm); init_tlb_flush_pending(mm);
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
mm->pmd_huge_pte = NULL; mm->pmd_huge_pte = NULL;
#endif #endif
......
...@@ -1650,7 +1650,7 @@ static unsigned long minimum_image_size(unsigned long saveable) ...@@ -1650,7 +1650,7 @@ static unsigned long minimum_image_size(unsigned long saveable)
{ {
unsigned long size; unsigned long size;
size = global_page_state(NR_SLAB_RECLAIMABLE) size = global_node_page_state(NR_SLAB_RECLAIMABLE)
+ global_node_page_state(NR_ACTIVE_ANON) + global_node_page_state(NR_ACTIVE_ANON)
+ global_node_page_state(NR_INACTIVE_ANON) + global_node_page_state(NR_INACTIVE_ANON)
+ global_node_page_state(NR_ACTIVE_FILE) + global_node_page_state(NR_ACTIVE_FILE)
......
...@@ -110,10 +110,12 @@ bool should_fail(struct fault_attr *attr, ssize_t size) ...@@ -110,10 +110,12 @@ bool should_fail(struct fault_attr *attr, ssize_t size)
if (in_task()) { if (in_task()) {
unsigned int fail_nth = READ_ONCE(current->fail_nth); unsigned int fail_nth = READ_ONCE(current->fail_nth);
if (fail_nth && !WRITE_ONCE(current->fail_nth, fail_nth - 1)) if (fail_nth) {
goto fail; if (!WRITE_ONCE(current->fail_nth, fail_nth - 1))
goto fail;
return false; return false;
}
} }
/* No need to check any other properties if the probability is 0 */ /* No need to check any other properties if the probability is 0 */
......
...@@ -485,7 +485,7 @@ static ssize_t config_show(struct device *dev, ...@@ -485,7 +485,7 @@ static ssize_t config_show(struct device *dev,
config->test_driver); config->test_driver);
else else
len += snprintf(buf+len, PAGE_SIZE - len, len += snprintf(buf+len, PAGE_SIZE - len,
"driver:\tEMTPY\n"); "driver:\tEMPTY\n");
if (config->test_fs) if (config->test_fs)
len += snprintf(buf+len, PAGE_SIZE - len, len += snprintf(buf+len, PAGE_SIZE - len,
...@@ -493,7 +493,7 @@ static ssize_t config_show(struct device *dev, ...@@ -493,7 +493,7 @@ static ssize_t config_show(struct device *dev,
config->test_fs); config->test_fs);
else else
len += snprintf(buf+len, PAGE_SIZE - len, len += snprintf(buf+len, PAGE_SIZE - len,
"fs:\tEMTPY\n"); "fs:\tEMPTY\n");
mutex_unlock(&test_dev->config_mutex); mutex_unlock(&test_dev->config_mutex);
...@@ -746,11 +746,11 @@ static int trigger_config_run_type(struct kmod_test_device *test_dev, ...@@ -746,11 +746,11 @@ static int trigger_config_run_type(struct kmod_test_device *test_dev,
strlen(test_str)); strlen(test_str));
break; break;
case TEST_KMOD_FS_TYPE: case TEST_KMOD_FS_TYPE:
break;
kfree_const(config->test_fs); kfree_const(config->test_fs);
config->test_driver = NULL; config->test_driver = NULL;
copied = config_copy_test_fs(config, test_str, copied = config_copy_test_fs(config, test_str,
strlen(test_str)); strlen(test_str));
break;
default: default:
mutex_unlock(&test_dev->config_mutex); mutex_unlock(&test_dev->config_mutex);
return -EINVAL; return -EINVAL;
...@@ -880,10 +880,10 @@ static int test_dev_config_update_uint_sync(struct kmod_test_device *test_dev, ...@@ -880,10 +880,10 @@ static int test_dev_config_update_uint_sync(struct kmod_test_device *test_dev,
int (*test_sync)(struct kmod_test_device *test_dev)) int (*test_sync)(struct kmod_test_device *test_dev))
{ {
int ret; int ret;
long new; unsigned long new;
unsigned int old_val; unsigned int old_val;
ret = kstrtol(buf, 10, &new); ret = kstrtoul(buf, 10, &new);
if (ret) if (ret)
return ret; return ret;
...@@ -918,9 +918,9 @@ static int test_dev_config_update_uint_range(struct kmod_test_device *test_dev, ...@@ -918,9 +918,9 @@ static int test_dev_config_update_uint_range(struct kmod_test_device *test_dev,
unsigned int max) unsigned int max)
{ {
int ret; int ret;
long new; unsigned long new;
ret = kstrtol(buf, 10, &new); ret = kstrtoul(buf, 10, &new);
if (ret) if (ret)
return ret; return ret;
...@@ -1146,7 +1146,7 @@ static struct kmod_test_device *register_test_dev_kmod(void) ...@@ -1146,7 +1146,7 @@ static struct kmod_test_device *register_test_dev_kmod(void)
struct kmod_test_device *test_dev = NULL; struct kmod_test_device *test_dev = NULL;
int ret; int ret;
mutex_unlock(&reg_dev_mutex); mutex_lock(&reg_dev_mutex);
/* int should suffice for number of devices, test for wrap */ /* int should suffice for number of devices, test for wrap */
if (unlikely(num_test_devs + 1) < 0) { if (unlikely(num_test_devs + 1) < 0) {
......
...@@ -24,7 +24,7 @@ struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info) ...@@ -24,7 +24,7 @@ struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info)
{ {
unsigned long flags; unsigned long flags;
struct page *page = alloc_page(balloon_mapping_gfp_mask() | struct page *page = alloc_page(balloon_mapping_gfp_mask() |
__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_ZERO); __GFP_NOMEMALLOC | __GFP_NORETRY);
if (!page) if (!page)
return NULL; return NULL;
......
...@@ -124,9 +124,7 @@ void dump_mm(const struct mm_struct *mm) ...@@ -124,9 +124,7 @@ void dump_mm(const struct mm_struct *mm)
#ifdef CONFIG_NUMA_BALANCING #ifdef CONFIG_NUMA_BALANCING
"numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n" "numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n"
#endif #endif
#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION)
"tlb_flush_pending %d\n" "tlb_flush_pending %d\n"
#endif
"def_flags: %#lx(%pGv)\n", "def_flags: %#lx(%pGv)\n",
mm, mm->mmap, mm->vmacache_seqnum, mm->task_size, mm, mm->mmap, mm->vmacache_seqnum, mm->task_size,
...@@ -158,9 +156,7 @@ void dump_mm(const struct mm_struct *mm) ...@@ -158,9 +156,7 @@ void dump_mm(const struct mm_struct *mm)
#ifdef CONFIG_NUMA_BALANCING #ifdef CONFIG_NUMA_BALANCING
mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq, mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq,
#endif #endif
#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) atomic_read(&mm->tlb_flush_pending),
mm->tlb_flush_pending,
#endif
mm->def_flags, &mm->def_flags mm->def_flags, &mm->def_flags
); );
} }
......
...@@ -1495,6 +1495,13 @@ int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) ...@@ -1495,6 +1495,13 @@ int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
goto clear_pmdnuma; goto clear_pmdnuma;
} }
/*
* The page_table_lock above provides a memory barrier
* with change_protection_range.
*/
if (mm_tlb_flush_pending(vma->vm_mm))
flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
/* /*
* Migrate the THP to the requested node, returns with page unlocked * Migrate the THP to the requested node, returns with page unlocked
* and access rights restored. * and access rights restored.
......
...@@ -4062,9 +4062,9 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, ...@@ -4062,9 +4062,9 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
return ret; return ret;
out_release_unlock: out_release_unlock:
spin_unlock(ptl); spin_unlock(ptl);
out_release_nounlock:
if (vm_shared) if (vm_shared)
unlock_page(page); unlock_page(page);
out_release_nounlock:
put_page(page); put_page(page);
goto out; goto out;
} }
......
...@@ -1038,7 +1038,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, ...@@ -1038,7 +1038,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
goto out_unlock; goto out_unlock;
if (pte_write(*pvmw.pte) || pte_dirty(*pvmw.pte) || if (pte_write(*pvmw.pte) || pte_dirty(*pvmw.pte) ||
(pte_protnone(*pvmw.pte) && pte_savedwrite(*pvmw.pte))) { (pte_protnone(*pvmw.pte) && pte_savedwrite(*pvmw.pte)) ||
mm_tlb_flush_pending(mm)) {
pte_t entry; pte_t entry;
swapped = PageSwapCache(page); swapped = PageSwapCache(page);
......
...@@ -215,12 +215,8 @@ static bool tlb_next_batch(struct mmu_gather *tlb) ...@@ -215,12 +215,8 @@ static bool tlb_next_batch(struct mmu_gather *tlb)
return true; return true;
} }
/* tlb_gather_mmu void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
* Called to initialize an (on-stack) mmu_gather structure for page-table unsigned long start, unsigned long end)
* tear-down from @mm. The @fullmm argument is used when @mm is without
* users and we're going to destroy the full address space (exit/execve).
*/
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
{ {
tlb->mm = mm; tlb->mm = mm;
...@@ -275,10 +271,14 @@ void tlb_flush_mmu(struct mmu_gather *tlb) ...@@ -275,10 +271,14 @@ void tlb_flush_mmu(struct mmu_gather *tlb)
* Called at the end of the shootdown operation to free up any resources * Called at the end of the shootdown operation to free up any resources
* that were required. * that were required.
*/ */
void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) void arch_tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end, bool force)
{ {
struct mmu_gather_batch *batch, *next; struct mmu_gather_batch *batch, *next;
if (force)
__tlb_adjust_range(tlb, start, end - start);
tlb_flush_mmu(tlb); tlb_flush_mmu(tlb);
/* keep the page table cache within bounds */ /* keep the page table cache within bounds */
...@@ -398,6 +398,34 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table) ...@@ -398,6 +398,34 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
#endif /* CONFIG_HAVE_RCU_TABLE_FREE */ #endif /* CONFIG_HAVE_RCU_TABLE_FREE */
/* tlb_gather_mmu
* Called to initialize an (on-stack) mmu_gather structure for page-table
* tear-down from @mm. The @fullmm argument is used when @mm is without
* users and we're going to destroy the full address space (exit/execve).
*/
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
unsigned long start, unsigned long end)
{
arch_tlb_gather_mmu(tlb, mm, start, end);
inc_tlb_flush_pending(tlb->mm);
}
void tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end)
{
/*
* If there are parallel threads are doing PTE changes on same range
* under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB
* flush by batching, a thread has stable TLB entry can fail to flush
* the TLB by observing pte_none|!pte_dirty, for example so flush TLB
* forcefully if we detect parallel PTE batching threads.
*/
bool force = mm_tlb_flush_nested(tlb->mm);
arch_tlb_finish_mmu(tlb, start, end, force);
dec_tlb_flush_pending(tlb->mm);
}
/* /*
* Note: this doesn't free the actual pages themselves. That * Note: this doesn't free the actual pages themselves. That
* has been handled earlier when unmapping all the memory regions. * has been handled earlier when unmapping all the memory regions.
......
...@@ -1937,12 +1937,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, ...@@ -1937,12 +1937,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
put_page(new_page); put_page(new_page);
goto out_fail; goto out_fail;
} }
/*
* We are not sure a pending tlb flush here is for a huge page
* mapping or not. Hence use the tlb range variant
*/
if (mm_tlb_flush_pending(mm))
flush_tlb_range(vma, mmun_start, mmun_end);
/* Prepare a page as a migration target */ /* Prepare a page as a migration target */
__SetPageLocked(new_page); __SetPageLocked(new_page);
......
...@@ -244,7 +244,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma, ...@@ -244,7 +244,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma,
BUG_ON(addr >= end); BUG_ON(addr >= end);
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
flush_cache_range(vma, addr, end); flush_cache_range(vma, addr, end);
set_tlb_flush_pending(mm); inc_tlb_flush_pending(mm);
do { do {
next = pgd_addr_end(addr, end); next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd)) if (pgd_none_or_clear_bad(pgd))
...@@ -256,7 +256,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma, ...@@ -256,7 +256,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma,
/* Only flush the TLB if we actually modified any entries: */ /* Only flush the TLB if we actually modified any entries: */
if (pages) if (pages)
flush_tlb_range(vma, start, end); flush_tlb_range(vma, start, end);
clear_tlb_flush_pending(mm); dec_tlb_flush_pending(mm);
return pages; return pages;
} }
......
...@@ -4458,8 +4458,9 @@ long si_mem_available(void) ...@@ -4458,8 +4458,9 @@ long si_mem_available(void)
* Part of the reclaimable slab consists of items that are in use, * Part of the reclaimable slab consists of items that are in use,
* and cannot be freed. Cap this estimate at the low watermark. * and cannot be freed. Cap this estimate at the low watermark.
*/ */
available += global_page_state(NR_SLAB_RECLAIMABLE) - available += global_node_page_state(NR_SLAB_RECLAIMABLE) -
min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low); min(global_node_page_state(NR_SLAB_RECLAIMABLE) / 2,
wmark_low);
if (available < 0) if (available < 0)
available = 0; available = 0;
...@@ -4602,8 +4603,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) ...@@ -4602,8 +4603,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
global_node_page_state(NR_FILE_DIRTY), global_node_page_state(NR_FILE_DIRTY),
global_node_page_state(NR_WRITEBACK), global_node_page_state(NR_WRITEBACK),
global_node_page_state(NR_UNSTABLE_NFS), global_node_page_state(NR_UNSTABLE_NFS),
global_page_state(NR_SLAB_RECLAIMABLE), global_node_page_state(NR_SLAB_RECLAIMABLE),
global_page_state(NR_SLAB_UNRECLAIMABLE), global_node_page_state(NR_SLAB_UNRECLAIMABLE),
global_node_page_state(NR_FILE_MAPPED), global_node_page_state(NR_FILE_MAPPED),
global_node_page_state(NR_SHMEM), global_node_page_state(NR_SHMEM),
global_page_state(NR_PAGETABLE), global_page_state(NR_PAGETABLE),
...@@ -7668,7 +7669,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, ...@@ -7668,7 +7669,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
/* Make sure the range is really isolated. */ /* Make sure the range is really isolated. */
if (test_pages_isolated(outer_start, end, false)) { if (test_pages_isolated(outer_start, end, false)) {
pr_info("%s: [%lx, %lx) PFNs busy\n", pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
__func__, outer_start, end); __func__, outer_start, end);
ret = -EBUSY; ret = -EBUSY;
goto done; goto done;
......
...@@ -888,10 +888,10 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, ...@@ -888,10 +888,10 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
.flags = PVMW_SYNC, .flags = PVMW_SYNC,
}; };
int *cleaned = arg; int *cleaned = arg;
bool invalidation_needed = false;
while (page_vma_mapped_walk(&pvmw)) { while (page_vma_mapped_walk(&pvmw)) {
int ret = 0; int ret = 0;
address = pvmw.address;
if (pvmw.pte) { if (pvmw.pte) {
pte_t entry; pte_t entry;
pte_t *pte = pvmw.pte; pte_t *pte = pvmw.pte;
...@@ -899,11 +899,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, ...@@ -899,11 +899,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
if (!pte_dirty(*pte) && !pte_write(*pte)) if (!pte_dirty(*pte) && !pte_write(*pte))
continue; continue;
flush_cache_page(vma, address, pte_pfn(*pte)); flush_cache_page(vma, pvmw.address, pte_pfn(*pte));
entry = ptep_clear_flush(vma, address, pte); entry = ptep_clear_flush(vma, pvmw.address, pte);
entry = pte_wrprotect(entry); entry = pte_wrprotect(entry);
entry = pte_mkclean(entry); entry = pte_mkclean(entry);
set_pte_at(vma->vm_mm, address, pte, entry); set_pte_at(vma->vm_mm, pvmw.address, pte, entry);
ret = 1; ret = 1;
} else { } else {
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
...@@ -913,11 +913,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, ...@@ -913,11 +913,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
if (!pmd_dirty(*pmd) && !pmd_write(*pmd)) if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
continue; continue;
flush_cache_page(vma, address, page_to_pfn(page)); flush_cache_page(vma, pvmw.address, page_to_pfn(page));
entry = pmdp_huge_clear_flush(vma, address, pmd); entry = pmdp_huge_clear_flush(vma, pvmw.address, pmd);
entry = pmd_wrprotect(entry); entry = pmd_wrprotect(entry);
entry = pmd_mkclean(entry); entry = pmd_mkclean(entry);
set_pmd_at(vma->vm_mm, address, pmd, entry); set_pmd_at(vma->vm_mm, pvmw.address, pmd, entry);
ret = 1; ret = 1;
#else #else
/* unexpected pmd-mapped page? */ /* unexpected pmd-mapped page? */
...@@ -926,11 +926,16 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma, ...@@ -926,11 +926,16 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
} }
if (ret) { if (ret) {
mmu_notifier_invalidate_page(vma->vm_mm, address);
(*cleaned)++; (*cleaned)++;
invalidation_needed = true;
} }
} }
if (invalidation_needed) {
mmu_notifier_invalidate_range(vma->vm_mm, address,
address + (1UL << compound_order(page)));
}
return true; return true;
} }
...@@ -1323,7 +1328,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1323,7 +1328,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
}; };
pte_t pteval; pte_t pteval;
struct page *subpage; struct page *subpage;
bool ret = true; bool ret = true, invalidation_needed = false;
enum ttu_flags flags = (enum ttu_flags)arg; enum ttu_flags flags = (enum ttu_flags)arg;
/* munlock has nothing to gain from examining un-locked vmas */ /* munlock has nothing to gain from examining un-locked vmas */
...@@ -1363,11 +1368,9 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1363,11 +1368,9 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
VM_BUG_ON_PAGE(!pvmw.pte, page); VM_BUG_ON_PAGE(!pvmw.pte, page);
subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte); subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
address = pvmw.address;
if (!(flags & TTU_IGNORE_ACCESS)) { if (!(flags & TTU_IGNORE_ACCESS)) {
if (ptep_clear_flush_young_notify(vma, address, if (ptep_clear_flush_young_notify(vma, pvmw.address,
pvmw.pte)) { pvmw.pte)) {
ret = false; ret = false;
page_vma_mapped_walk_done(&pvmw); page_vma_mapped_walk_done(&pvmw);
...@@ -1376,7 +1379,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1376,7 +1379,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
} }
/* Nuke the page table entry. */ /* Nuke the page table entry. */
flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); flush_cache_page(vma, pvmw.address, pte_pfn(*pvmw.pte));
if (should_defer_flush(mm, flags)) { if (should_defer_flush(mm, flags)) {
/* /*
* We clear the PTE but do not flush so potentially * We clear the PTE but do not flush so potentially
...@@ -1386,11 +1389,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1386,11 +1389,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
* transition on a cached TLB entry is written through * transition on a cached TLB entry is written through
* and traps if the PTE is unmapped. * and traps if the PTE is unmapped.
*/ */
pteval = ptep_get_and_clear(mm, address, pvmw.pte); pteval = ptep_get_and_clear(mm, pvmw.address,
pvmw.pte);
set_tlb_ubc_flush_pending(mm, pte_dirty(pteval)); set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
} else { } else {
pteval = ptep_clear_flush(vma, address, pvmw.pte); pteval = ptep_clear_flush(vma, pvmw.address, pvmw.pte);
} }
/* Move the dirty bit to the page. Now the pte is gone. */ /* Move the dirty bit to the page. Now the pte is gone. */
...@@ -1405,12 +1409,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1405,12 +1409,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
if (PageHuge(page)) { if (PageHuge(page)) {
int nr = 1 << compound_order(page); int nr = 1 << compound_order(page);
hugetlb_count_sub(nr, mm); hugetlb_count_sub(nr, mm);
set_huge_swap_pte_at(mm, address, set_huge_swap_pte_at(mm, pvmw.address,
pvmw.pte, pteval, pvmw.pte, pteval,
vma_mmu_pagesize(vma)); vma_mmu_pagesize(vma));
} else { } else {
dec_mm_counter(mm, mm_counter(page)); dec_mm_counter(mm, mm_counter(page));
set_pte_at(mm, address, pvmw.pte, pteval); set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
} }
} else if (pte_unused(pteval)) { } else if (pte_unused(pteval)) {
...@@ -1434,7 +1438,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1434,7 +1438,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
swp_pte = swp_entry_to_pte(entry); swp_pte = swp_entry_to_pte(entry);
if (pte_soft_dirty(pteval)) if (pte_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte); swp_pte = pte_swp_mksoft_dirty(swp_pte);
set_pte_at(mm, address, pvmw.pte, swp_pte); set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
} else if (PageAnon(page)) { } else if (PageAnon(page)) {
swp_entry_t entry = { .val = page_private(subpage) }; swp_entry_t entry = { .val = page_private(subpage) };
pte_t swp_pte; pte_t swp_pte;
...@@ -1460,7 +1464,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1460,7 +1464,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
* If the page was redirtied, it cannot be * If the page was redirtied, it cannot be
* discarded. Remap the page to page table. * discarded. Remap the page to page table.
*/ */
set_pte_at(mm, address, pvmw.pte, pteval); set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
SetPageSwapBacked(page); SetPageSwapBacked(page);
ret = false; ret = false;
page_vma_mapped_walk_done(&pvmw); page_vma_mapped_walk_done(&pvmw);
...@@ -1468,7 +1472,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1468,7 +1472,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
} }
if (swap_duplicate(entry) < 0) { if (swap_duplicate(entry) < 0) {
set_pte_at(mm, address, pvmw.pte, pteval); set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
ret = false; ret = false;
page_vma_mapped_walk_done(&pvmw); page_vma_mapped_walk_done(&pvmw);
break; break;
...@@ -1484,14 +1488,18 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1484,14 +1488,18 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
swp_pte = swp_entry_to_pte(entry); swp_pte = swp_entry_to_pte(entry);
if (pte_soft_dirty(pteval)) if (pte_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte); swp_pte = pte_swp_mksoft_dirty(swp_pte);
set_pte_at(mm, address, pvmw.pte, swp_pte); set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
} else } else
dec_mm_counter(mm, mm_counter_file(page)); dec_mm_counter(mm, mm_counter_file(page));
discard: discard:
page_remove_rmap(subpage, PageHuge(page)); page_remove_rmap(subpage, PageHuge(page));
put_page(page); put_page(page);
mmu_notifier_invalidate_page(mm, address); invalidation_needed = true;
} }
if (invalidation_needed)
mmu_notifier_invalidate_range(mm, address,
address + (1UL << compound_order(page)));
return ret; return ret;
} }
......
...@@ -1022,7 +1022,11 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) ...@@ -1022,7 +1022,11 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
*/ */
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
spin_lock(&sbinfo->shrinklist_lock); spin_lock(&sbinfo->shrinklist_lock);
if (list_empty(&info->shrinklist)) { /*
* _careful to defend against unlocked access to
* ->shrink_list in shmem_unused_huge_shrink()
*/
if (list_empty_careful(&info->shrinklist)) {
list_add_tail(&info->shrinklist, list_add_tail(&info->shrinklist,
&sbinfo->shrinklist); &sbinfo->shrinklist);
sbinfo->shrinklist_len++; sbinfo->shrinklist_len++;
...@@ -1817,7 +1821,11 @@ alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, ...@@ -1817,7 +1821,11 @@ alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo,
* to shrink under memory pressure. * to shrink under memory pressure.
*/ */
spin_lock(&sbinfo->shrinklist_lock); spin_lock(&sbinfo->shrinklist_lock);
if (list_empty(&info->shrinklist)) { /*
* _careful to defend against unlocked access to
* ->shrink_list in shmem_unused_huge_shrink()
*/
if (list_empty_careful(&info->shrinklist)) {
list_add_tail(&info->shrinklist, list_add_tail(&info->shrinklist,
&sbinfo->shrinklist); &sbinfo->shrinklist);
sbinfo->shrinklist_len++; sbinfo->shrinklist_len++;
......
...@@ -633,7 +633,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) ...@@ -633,7 +633,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
* which are reclaimable, under pressure. The dentry * which are reclaimable, under pressure. The dentry
* cache and most inode caches should fall into this * cache and most inode caches should fall into this
*/ */
free += global_page_state(NR_SLAB_RECLAIMABLE); free += global_node_page_state(NR_SLAB_RECLAIMABLE);
/* /*
* Leave reserved pages. The pages are not for anonymous pages. * Leave reserved pages. The pages are not for anonymous pages.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment