mm/hmm: factor out pte and pmd handling to simplify hmm_vma_walk_pmd()

No functional change, just create one function to handle pmd and one to
handle pte (hmm_vma_handle_pmd() and hmm_vma_handle_pte()).

Link: http://lkml.kernel.org/r/20180323005527.758-14-jglisse@redhat.comSigned-off-by: Jérôme Glisse <jglisse@redhat.com>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Cc: Evgeny Baskakov <ebaskakov@nvidia.com>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Mark Hairgrove <mhairgrove@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm/hmm.c

@@ -375,74 +375,45 @@ static int hmm_vma_walk_hole(unsigned long addr,
 	return hmm_vma_walk->fault ? -EAGAIN : 0;
 }
 
-static int hmm_vma_walk_pmd(pmd_t *pmdp,
-			    unsigned long start,
+static int hmm_vma_handle_pmd(struct mm_walk *walk,
+			      unsigned long addr,
 			      unsigned long end,
-			    struct mm_walk *walk)
+			      uint64_t *pfns,
+			      pmd_t pmd)
 {
 	struct hmm_vma_walk *hmm_vma_walk = walk->private;
-	struct hmm_range *range = hmm_vma_walk->range;
-	struct vm_area_struct *vma = walk->vma;
-	uint64_t *pfns = range->pfns;
-	unsigned long addr = start, i;
-	bool write_fault;
-	pte_t *ptep;
-
-	i = (addr - range->start) >> PAGE_SHIFT;
-	write_fault = hmm_vma_walk->fault & hmm_vma_walk->write;
-
-again:
-	if (pmd_none(*pmdp))
-		return hmm_vma_walk_hole(start, end, walk);
-
-	if (pmd_huge(*pmdp) && vma->vm_flags & VM_HUGETLB)
-		return hmm_pfns_bad(start, end, walk);
-
-	if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
-		unsigned long pfn;
+	unsigned long pfn, i;
 	uint64_t flag = 0;
-		pmd_t pmd;
 
-		/*
-		 * No need to take pmd_lock here, even if some other threads
-		 * is splitting the huge pmd we will get that event through
-		 * mmu_notifier callback.
-		 *
-		 * So just read pmd value and check again its a transparent
-		 * huge or device mapping one and compute corresponding pfn
-		 * values.
-		 */
-		pmd = pmd_read_atomic(pmdp);
-		barrier();
-		if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
-			goto again;
 	if (pmd_protnone(pmd))
-			return hmm_vma_walk_hole(start, end, walk);
+		return hmm_vma_walk_hole(addr, end, walk);
 
-		if (write_fault && !pmd_write(pmd))
-			return hmm_vma_walk_hole(start, end, walk);
+	if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pmd_write(pmd))
+		return hmm_vma_walk_hole(addr, end, walk);
 
 	pfn = pmd_pfn(pmd) + pte_index(addr);
 	flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;
-		for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
+	for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
 		pfns[i] = hmm_pfn_from_pfn(pfn) | flag;
+	hmm_vma_walk->last = end;
 	return 0;
-	}
-
-	if (pmd_bad(*pmdp))
-		return hmm_pfns_bad(start, end, walk);
+}
 
-	ptep = pte_offset_map(pmdp, addr);
-	for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
+static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
+			      unsigned long end, pmd_t *pmdp, pte_t *ptep,
+			      uint64_t *pfn)
+{
+	struct hmm_vma_walk *hmm_vma_walk = walk->private;
+	struct vm_area_struct *vma = walk->vma;
 	pte_t pte = *ptep;
 
-		pfns[i] = 0;
+	*pfn = 0;
 
 	if (pte_none(pte)) {
-			pfns[i] = 0;
+		*pfn = 0;
 		if (hmm_vma_walk->fault)
 			goto fault;
-			continue;
+		return 0;
 	}
 
 	if (!pte_present(pte)) {
@@ -451,7 +422,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
 		if (!non_swap_entry(entry)) {
 			if (hmm_vma_walk->fault)
 				goto fault;
-				continue;
+			return 0;
 		}
 
 		/*
@@ -459,13 +430,16 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
 		 * device and report anything else as error.
 		 */
 		if (is_device_private_entry(entry)) {
-				pfns[i] = hmm_pfn_from_pfn(swp_offset(entry));
+			*pfn = hmm_pfn_from_pfn(swp_offset(entry));
 			if (is_write_device_private_entry(entry)) {
-					pfns[i] |= HMM_PFN_WRITE;
-				} else if (write_fault)
+				*pfn |= HMM_PFN_WRITE;
+			} else if ((hmm_vma_walk->fault & hmm_vma_walk->write))
 				goto fault;
-				pfns[i] |= HMM_PFN_DEVICE_PRIVATE;
-			} else if (is_migration_entry(entry)) {
+			*pfn |= HMM_PFN_DEVICE_PRIVATE;
+			return 0;
+		}
+
+		if (is_migration_entry(entry)) {
 			if (hmm_vma_walk->fault) {
 				pte_unmap(ptep);
 				hmm_vma_walk->last = addr;
@@ -473,28 +447,84 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
 						pmdp, addr);
 				return -EAGAIN;
 			}
-				continue;
-			} else {
-				/* Report error for everything else */
-				pfns[i] = HMM_PFN_ERROR;
+			return 0;
 		}
-			continue;
+
+		/* Report error for everything else */
+		*pfn = HMM_PFN_ERROR;
+		return -EFAULT;
 	}
 
-		if (write_fault && !pte_write(pte))
+	if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pte_write(pte))
 		goto fault;
 
-		pfns[i] = hmm_pfn_from_pfn(pte_pfn(pte));
-		pfns[i] |= pte_write(pte) ? HMM_PFN_WRITE : 0;
-		continue;
+	*pfn = hmm_pfn_from_pfn(pte_pfn(pte));
+	*pfn |= pte_write(pte) ? HMM_PFN_WRITE : 0;
+	return 0;
 
 fault:
 	pte_unmap(ptep);
 	/* Fault any virtual address we were asked to fault */
+	return hmm_vma_walk_hole(addr, end, walk);
+}
+
+static int hmm_vma_walk_pmd(pmd_t *pmdp,
+			    unsigned long start,
+			    unsigned long end,
+			    struct mm_walk *walk)
+{
+	struct hmm_vma_walk *hmm_vma_walk = walk->private;
+	struct hmm_range *range = hmm_vma_walk->range;
+	uint64_t *pfns = range->pfns;
+	unsigned long addr = start, i;
+	pte_t *ptep;
+
+	i = (addr - range->start) >> PAGE_SHIFT;
+
+again:
+	if (pmd_none(*pmdp))
 		return hmm_vma_walk_hole(start, end, walk);
+
+	if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))
+		return hmm_pfns_bad(start, end, walk);
+
+	if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
+		pmd_t pmd;
+
+		/*
+		 * No need to take pmd_lock here, even if some other threads
+		 * is splitting the huge pmd we will get that event through
+		 * mmu_notifier callback.
+		 *
+		 * So just read pmd value and check again its a transparent
+		 * huge or device mapping one and compute corresponding pfn
+		 * values.
+		 */
+		pmd = pmd_read_atomic(pmdp);
+		barrier();
+		if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
+			goto again;
+
+		return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
+	}
+
+	if (pmd_bad(*pmdp))
+		return hmm_pfns_bad(start, end, walk);
+
+	ptep = pte_offset_map(pmdp, addr);
+	for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
+		int r;
+
+		r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
+		if (r) {
+			/* hmm_vma_handle_pte() did unmap pte directory */
+			hmm_vma_walk->last = addr;
+			return r;
+		}
 	}
 	pte_unmap(ptep - 1);
 
+	hmm_vma_walk->last = addr;
 	return 0;
 }